diff --git a/CLAUDE.md b/CLAUDE.md index 73e32deb..5149c163 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -173,6 +173,19 @@ uses. Any future Android/Java tooltip work should target the normalized `Tooltip removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, `load_android_data.py`. +- **`scripts/hm_pages_to_templates.py`** - stores the Help+Manual topic pages under `i/` as JSON + plus one Pebble template derived from the pages themselves (`helpmanual-topic.peb`), after + rendering every page back byte for byte with the app's engine (`scripts/hm_render_check/`, + which needs `java` and the Pebble jars). With `--drop-jstopics` it also stores + `helpmanual-jstopic.peb` and drops the `i/jstopics/` rows the app then builds from the page JSON + (ADFA-6618; needs the app side). Run it after every step that edits `i/` pages: after + `remove_skin_font_comment.py`, `bundle_remote_images.py`, `remove_remote_loads.py`, + `share_skin_sprite.py`, `check_remote_loads.py` and the media steps (with their snapshot before + them), then `check_remote_loads.py` again, which reads Templates and JSON pages and so works on + either side. `share_skin_sprite.py` comes with PR #42 (ADFA-4195) and the other four with PR #41 + (ADFA-6589); this step ships after both. A media rollback (`--restore-originals`) cannot undo its + pages: start the rollback from this step's pre-run backup, keep that backup with the rollback + bundle, and run this step again afterwards. - **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and diff --git a/scripts/hm_pages_to_templates.py b/scripts/hm_pages_to_templates.py new file mode 100644 index 00000000..e0bd91f4 --- /dev/null +++ b/scripts/hm_pages_to_templates.py @@ -0,0 +1,1343 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# ] +# /// +""" +hm_pages_to_templates.py + +Stores each Help+Manual topic page under i/ as JSON plus one shared Pebble +template, instead of as a finished HTML page (ADFA-6618). + +Every topic's page, i/X.html, carries the same ~43 KB skin shell around a few +per-topic parts. This step derives that shell from the pages themselves - no +hand-written copy of the skin, so a skin change is picked up on the next run - +stores it once in Templates as a Pebble template, and turns each page row into +the topic's parts as JSON with templateId pointing at the template. The app +already renders such rows (DocumentationContentSource: a Content row with a +non-zero templateId holds JSON, which is rendered through that template before +it is served), so this needs no app change. + +How the template is derived: + + 1. A topic is a text/html page X.html whose JavaScript copy, + jstopics/X.js, exists. That file's hmLoadTopic({...}) fields + (hmTitle, hmBody, hmPrevLink, ...) are the export's own list of what is + per-topic. Pages without one (hmftsearch.html, the _hm*.htm templates, + ...) are not topics and are left alone, as is a page not named X.html: + the app builds jstopics/X.js only from X.html. + 2. Each page's body is located by its jstopics hmBody, with Help+Manual's + "\\n\\r" line breaks written as the page's "\\r\\n", and stood in for by a + marker, so the rest of the page is the skin plus a few small parts. + 3. The pages are aligned line by line. Lines every page shares, in the same + order, are the template's static text. Between them are the holes. + 4. Each hole is explained by the topic's fields: its text is rewritten as a + "shape" - static text with {{ field }} placeholders - and a shape is + accepted for a page only when rendering it with that page's values gives + the page's text back exactly. When pages need different shapes (a + disabled "Previous" link on the first topic, say), the template chooses + between them with {% if field is empty %}. The condition is whichever + one fits the data, not Help+Manual's own rule - in the 2026-10-13 + export the Top link is disabled exactly where hmPrevLink is empty - so + it fails safe: a page it does not fit fails the render check below. + 5. Where one shape fits every page but a page shows a different value in + it - a meta description written differently from its jstopics value, a + caption that is not the topic's title - that value is read out of the + page's text. It becomes the page's value of the field when no other hole + reads the field; otherwise it goes in a "part_" key, which the template + uses where a page has one. A hole none of this explains keeps each + page's own text in a "part_" key, so a page never depends on a guess. + +Each page's parts are stored in its JSON exactly as they appear in that page - +HTML-escaped where the page has them escaped (Help+Manual writes """ and +" " into meta descriptions) - and inserted with | raw, so nothing is +re-escaped on the way out. The JSON also holds the jstopics fields the page +itself does not show (hmHeader, hmTitlePath), and, under "js_" keys, a +field's jstopics value wherever it differs from the page's, so a later step +can build i/jstopics/X.js from the same row (ADFA-6618 phase 2). + +Before anything is written, every converted page is rendered with the real +Pebble engine (scripts/hm_render_check/Render.java, built as the app builds +its engine) and compared with the stored page byte for byte. A page that does +not render back exactly stays as HTML, is named, and is counted - as is any +topic that cannot be converted for another reason. Rendering needs `java` +(17 or later) and the Pebble 4.1.1, unbescape, slf4j-api and Gson jars; they +are looked for in the Gradle and Maven caches, or given with --classpath. + +Running it again changes nothing: pages already stored as JSON are rendered +back to HTML with the stored template, so a second run derives the same +template from the same pages, and a topic whose jstopics row is gone takes +its jstopics fields from its JSON. A page the newly derived template cannot +reproduce is written back as HTML rather than left pointing at a template +that no longer fits it. + +Phase 2, with --drop-jstopics: the jstopics rows go too. The same run +derives a second template, helpmanual-jstopic.peb, from the jstopics files' +shared frame (hmLoadTopic({...}) with each field's value JavaScript-escaped +from the page's JSON), and stores it in Templates beside the page template, +as plain UTF-8 like the page template. Every topic's stored jstopics file is +rebuilt from its page's JSON with the real engine first, and its row (chunks +included) is dropped only when the rebuilt file has the same field values +after JavaScript unescaping and the same text around them. A topic that does +not match, or whose page stays HTML for any reason, keeps its row, is named, +and makes the run exit 1. On later runs, +every topic whose row is gone is built again with the stored template and +checked against the fields its JSON holds; one that differs is named and +fails the run too. A page that goes back to HTML after its row was dropped - +it no longer fits the template, say - gets the row written back, built with +the stored jstopics template from the page's last JSON and checked the same +way, and the run exits 1 to say so; if the row cannot be built, the run +writes nothing. + +CodeOnTheGo's side of ADFA-6618 builds i/jstopics/X.js on request by +rendering helpmanual-jstopic.peb with i/X.html's JSON, so a database written +this way needs an app that does: with an older app, in-page navigation, +printing and topic popups break. Without --drop-jstopics the jstopics rows +are left alone, the run still reports how many it would rebuild, and the +database works with the app as it is. + +It is an option rather than a separate step because both templates come from +one derivation: the "js_" keys the jstopics template reads are written into +the page JSON by the same run, and the check before dropping a row has to use +exactly the JSON being stored. --jstopic-template FILE also writes the +derived jstopics template to FILE. + +Runs in one transaction (rolled back on any error), backs the database up +first, and VACUUMs afterwards - all three only when there is something to +change. --dry-run does all the work, the render check included, and reports +what would change without writing. + + uv run scripts/hm_pages_to_templates.py documentation.db --dry-run + uv run scripts/hm_pages_to_templates.py documentation.db + uv run scripts/hm_pages_to_templates.py documentation.db --drop-jstopics + +Where it goes in the pipeline: after every step that edits i/ pages, since +they work on HTML and after this step the topics' rows hold JSON and the skin +lives in Templates. With ADFA-6589's, ADFA-4195's and ADFA-6596's steps the +order is as below. Steps 1 to 3 and 5 come with PR #41 (ADFA-6589) and step 4 +with PR #42 (ADFA-4195), so this step ships after both: + + 1. remove_skin_font_comment.py + 2. bundle_remote_images.py + 3. remove_remote_loads.py + 4. share_skin_sprite.py - before this step: afterwards the sprite it moves + is in Templates, not in the pages, and it finds nothing to do + 5. check_remote_loads.py + 6. the media steps, with the snapshot update_media_references.py --before + diffs against taken before them + 7. this step + 8. check_remote_loads.py again. It reads Templates and JSON pages as well + as HTML, so it gives the same answer on either side of this step (90 + loads before and after on the untreated 2026-10-13 database). + +Rolling back media (optimize_db_media.py / update_media_references.py +--restore-originals) cannot undo i/ pages this step has converted: their rows +hold JSON that matches no recorded digest, the dropped jstopics rows named +images too (48 of the 359 on the 2026-10-13 database), and the skin's +own images, such as ./images/leftright.webp, are named only in Templates. A +media rollback has to start from this step's pre-run backup and then run this +step again, so keep that backup with the rollback bundle. + +Recovering: the timestamped backup taken before the first write is the whole +undo. +""" +from __future__ import annotations + +import argparse +import difflib +import json +import os +import re +import sqlite3 +import statistics +import subprocess +import sys +import tempfile +import threading +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from optimize_db_media import ( + BrotliCodec, backup_database, delete_media_row, human, is_continuation_path, load_dictionary, reassemble, + write_chunks, +) +from update_media_references import replace_row + +PAGE_TEMPLATE = "helpmanual-topic.peb" +JSTOPIC_TEMPLATE = "helpmanual-jstopic.peb" +CONTENT_TYPE = "text/html" +JSTOPIC_CONTENT_TYPE = "text/javascript" # what the jstopics rows are stored as, and what the app serves + +BODY_FIELD = "hmBody" +FILE_KEY = "file" +PART_PREFIX = "part_" +JS_PREFIX = "js_" + +# A page must share this much of its lines with the other topics' pages to be +# aligned with them; one that does not has a different shell and stays HTML. +MIN_SHARED_LINES = 0.9 + +# A hole whose pages need more distinct shapes than this is not explained by +# fields; each page keeps its own text instead. +MAX_SHAPES = 40 + +# Field values shorter than this are not looked for inside holes: a one- or +# two-character value matches too much by accident. A page whose short value +# does appear is still explained by a shape found on the other pages. +MIN_SUBSTITUTED = 3 + +# Stands in for the page body while pages are aligned. NUL never occurs in a +# page that is converted (checked), so it cannot collide with page text. +BODY_MARKER = "\x00body\x00" + +RENDER_JAVA = Path(__file__).resolve().parent / "hm_render_check" / "Render.java" + +# Pebble's newline trimming: the line break right after any tag is dropped. +# Same alternatives, same order, as io.pebbletemplates.pebble.lexer.Syntax. +_TRIMMED_NEWLINE = re.compile("\r\n|\n\r|\r|\n|\u0085|\u2028|\u2029") +_LINE = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+") +_JS_FIELD = re.compile(r'([A-Za-z_$][\w$]*)\s*:\s*"((?:[^"\\]|\\.)*)"', re.S) +_JS_FIELD_SLOT = re.compile(r'([A-Za-z_$][\w$]*)\s*:\s*""') +_JS_STRING = re.compile(r'"((?:[^"\\]|\\.)*)"', re.S) +_JS_ESCAPE = re.compile(r"\\(u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|\r\n|.)", re.S) +_JS_SIMPLE = {"n": "\n", "t": "\t", "r": "\r", "b": "\b", "f": "\f", "v": "\v", "0": "\0"} +_IDENTIFIER = re.compile(r"[A-Za-z][A-Za-z0-9_]*") + + +class BodyForm: + """How a jstopics hmBody relates to the body in the page.""" + + def __init__(self, name: str, to_page, to_jstopic, pebble_filter: str): + self.name = name + self.to_page = to_page + self.to_jstopic = to_jstopic + self.pebble_filter = pebble_filter + + +# Help+Manual writes a body's line breaks as "\n\r" in jstopics and "\r\n" in +# the page. The Pebble filter carries literal CR and LF characters: Pebble's +# string literals have no backslash escapes. +BODY_FORMS = ( + BodyForm("lfcr", lambda v: v.replace("\n\r", "\r\n"), lambda v: v.replace("\r\n", "\n\r"), + ' | replace({"\r\n": "\n\r"})'), + BodyForm("same", lambda v: v, lambda v: v, ""), +) + + +def js_unescape(text: str) -> str: + """The value of a JavaScript string literal's contents.""" + def one(match): + esc = match.group(1) + if esc[0] in "ux" and len(esc) > 1: + return chr(int(esc[1:], 16)) + if esc in ("\r\n", "\n", "\r", "\u2028", "\u2029"): + return "" # a line continuation + return _JS_SIMPLE.get(esc, esc) + # "\uD83D\uDE00" is one character in JavaScript; join such pairs here too. + return _JS_ESCAPE.sub(one, text).encode("utf-16", "surrogatepass").decode("utf-16", "surrogatepass") + + +def parse_jstopic(text: str): + """(fields, frame) for a jstopics file: its field values, JavaScript- + unescaped and in file order, and the file with every string literal + emptied, which is the same for every topic of one export. None when the + file holds no hmBody.""" + fields = {} + for name, value in _JS_FIELD.findall(text): + fields.setdefault(name, js_unescape(value)) + if BODY_FIELD not in fields: + return None + return fields, _JS_STRING.sub('""', text) + + +def hm_html(value: str) -> str: + """A value as Help+Manual writes it into page markup.""" + return (value.replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """).replace("\u00a0", " ")) + + +def pebble_empty(value) -> bool: + """Pebble's `is empty` test on a string: Java's trim() leaves nothing.""" + return value is None or all(ch <= " " for ch in value) + + +def split_lines(text: str) -> list: + """Lines with their endings, split on CR, LF and CRLF only.""" + return _LINE.findall(text) + + +class Topic: + def __init__(self, name: str, path: str, page: str, fields: dict, frame: str): + self.name = name # "debug-top" + self.path = path # "i/debug-top.html" + self.page = page # the page as text + self.fields = fields # jstopics fields, unescaped + self.frame = frame # the jstopics file with its strings emptied + + @property + def file(self) -> str: + return self.path.rsplit("/", 1)[-1] + + +# --- template text ----------------------------------------------------------- + +def text_seg(text: str) -> tuple: + return ("text", text) + + +def tag_seg(tag: str) -> tuple: + return ("tag", tag) + + +def var_tag(name: str) -> str: + return "{{ " + name + " | raw }}" + + +def emit(segments: list) -> str: + """Template source for a list of ("text", s) / ("tag", s) segments. + + Static text is made safe for Pebble's lexer: a "{" that would open a + delimiter ("{{", "{%", "{#") is written as {{ "{" }}, and a line break right + after a tag is doubled, since Pebble trims one there.""" + escaped = [] + for index, (kind, value) in enumerate(segments): + if kind == "tag": + escaped.append((kind, value)) + continue + following = next((v for _k, v in segments[index + 1:] if v), "") + start = 0 + for pos, ch in enumerate(value): + if ch != "{": + continue + nxt = value[pos + 1] if pos + 1 < len(value) else following[:1] + if nxt in ("{", "%", "#"): + escaped.append(("text", value[start:pos])) + escaped.append(("tag", '{{ "{" }}')) + start = pos + 1 + escaped.append(("text", value[start:])) + out = [] + after_tag = False + for kind, value in escaped: + if kind == "tag": + out.append(value) + after_tag = True + continue + if not value: + continue + if after_tag: + trimmed = _TRIMMED_NEWLINE.match(value) + if trimmed: + value = trimmed.group(0) + value + out.append(value) + after_tag = False + return "".join(out) + + +# --- derivation -------------------------------------------------------------- + +# A shape is a tuple of tokens: ("t", text) is static text, ("v", key) a +# page's value for key, and ("o", key, part) the value of part where a page +# has one and of key everywhere else. + +class Hole: + """A place where the pages differ, and how the template fills it.""" + + def __init__(self, values: dict, context: str): + self.values = values # {topic name: its text here, body marked} + self.context = context # static text before it, for naming + self.kind = None # "shape", "choice" or "part" + self.branches = [] # [(predicate or None, shape)] + self.part = None # key holding pages' own text, for "part" + self.part_pages = set() + + def keys(self) -> set: + """Every key the template reads to fill this hole.""" + out = {p[0] for p, _s in self.branches if p} + for _p, shape in self.branches: + for token in shape: + if token[0] != "t": + out.update(token[1:]) + if self.part: + out.add(self.part) + return out + + +class Derivation: + def __init__(self): + self.template = None + self.jstopic_template = None + self.contexts = {} # {topic name: JSON context} + self.unfit = {} # {topic name: reason} + self.holes = [] + self.fields = [] + self.body_form = {} + + +def _shape_of(value: str, lookup: dict, pattern) -> tuple: + """`value` as a shape: text with each field value found in it replaced by + a placeholder for that field.""" + if pattern is None: + return (("t", value),) if value else () + tokens, pos = [], 0 + for match in pattern.finditer(value): + if match.start() > pos: + tokens.append(("t", value[pos:match.start()])) + tokens.append(("v", lookup[match.group(0)])) + pos = match.end() + if pos < len(value): + tokens.append(("t", value[pos:])) + return tuple(tokens) + + +def _token_value(token: tuple, values: dict) -> str: + if token[0] == "t": + return token[1] + if token[0] == "o" and values.get(token[2]) is not None: + return values[token[2]] + return values.get(token[1], "") + + +def _render_shape(shape: tuple, values: dict) -> str: + return "".join(_token_value(token, values) for token in shape) + + +def _shape_segments(shape: tuple) -> list: + out = [] + for token in shape: + if token[0] == "t": + out.append(text_seg(token[1])) + elif token[0] == "v": + out.append(tag_seg(var_tag(token[1]))) + else: + out.append(tag_seg("{{ (" + token[2] + " is null ? " + token[1] + " : " + token[2] + ") | raw }}")) + return out + + +def _shape_regex(shape: tuple): + """A regular expression matching what `shape` renders to with any values, + capturing each key's value; None for a shape with no keys.""" + parts, groups, seen = [], {}, {} + for token in shape: + if token[0] == "t": + parts.append(re.escape(token[1])) + elif token[1] in seen: + parts.append(f"(?P={seen[token[1]]})") + else: + group = f"g{len(groups)}" + seen[token[1]] = group + groups[group] = token[1] + parts.append(f"(?P<{group}>.*?)") + return (re.compile("".join(parts), re.S), groups) if groups else (None, {}) + + +def _predicate_tag(predicate: tuple, first: bool) -> str: + name, negate = predicate + return "{% " + ("if " if first else "elseif ") + name + (" is not empty" if negate else " is empty") + " %}" + + +def _part_name(context: str, taken: set) -> str: + words = _IDENTIFIER.findall(context[-200:]) + stem = PART_PREFIX + (words[-1] if words else "text") + name, n = stem, 1 + while name in taken: + n += 1 + name = f"{stem}{n}" + taken.add(name) + return name + + +def _explain(hole: Hole, variables: dict, patterns: dict, names: list) -> None: + """Decides how the template fills `hole`; see the module docstring.""" + pages = list(hole.values) + shapes = {n: _shape_of(hole.values[n], *patterns[n]) for n in pages} + counts = Counter(shapes[n] for n in pages) + explained = {} + + def explains(shape): + if shape not in explained: + explained[shape] = {n for n in pages if _render_shape(shape, variables[n]) == hole.values[n]} + return explained[shape] + + chosen = [] + if len(counts) <= MAX_SHAPES: + remaining = set(pages) + while remaining: + candidates = {shapes[n] for n in remaining} + best = max(candidates, key=lambda s: (len(explains(s) & remaining), counts[s], repr(s))) + chosen.append(best) + remaining -= explains(best) + hole.kind, hole.branches, hole.part_pages = None, [], set() + if len(chosen) == 1: + hole.kind = "shape" + hole.branches = [(None, chosen[0])] + return + if chosen: + main = chosen[0] + uncovered = set(pages) - explains(main) + branches = [] + for shape in chosen[1:]: + must = uncovered & explains(shape) + if not must: + continue + # The condition is fitted to the data, not taken from Help+Manual's rules: in the + # 2026-10-13 export the Top link is disabled on exactly the pages whose hmPrevLink is + # empty, so that is what this finds. It fails safe - a later page it does not fit + # renders differently, fails the render check and stays HTML. + found = None + for name in names: + for negate in (False, True): + selected = {n for n in pages if pebble_empty(variables[n].get(name)) != negate} + if must <= selected <= explains(shape): + found = (name, negate) + break + if found: + break + if found is None: + branches = None + break + branches.append((found, shape)) + uncovered -= {n for n in pages if pebble_empty(variables[n].get(found[0])) != found[1]} + if branches is not None and not uncovered: + hole.kind = "choice" + hole.branches = branches + [(None, main)] + return + # Nothing chooses between the shapes: the pages the main shape does not + # explain keep their own text, unless _refine finds their values in it. + hole.kind = "part" + if chosen: + hole.branches = [(None, chosen[0])] + hole.part_pages = set(pages) - explains(chosen[0]) + else: + hole.part_pages = set(pages) + + +def _refine(holes: list, variables: dict, taken: set) -> None: + """For a hole whose main shape fails on some pages, reads those pages' + values out of their own text with the shape's pattern: the meta + description written differently from its jstopics value, say, or a page + whose caption is not its title. + + A key no other hole reads takes the page's value as it is - it is that + page's value of the key, in the form the page shows it. A key other holes + read keeps its value, and this hole reads the page's text from a "part_" + key on the pages that need one.""" + readers = Counter() + for hole in holes: + readers.update(hole.keys()) + for hole in holes: + if hole.kind != "part" or not hole.branches: + continue + main = hole.branches[0][1] + pattern, groups = _shape_regex(main) + if pattern is None: + continue + found = {} + for n in hole.part_pages: + match = pattern.fullmatch(hole.values[n]) + if match is None: + break + found[n] = {groups[g]: match.group(g) for g in groups} + else: + shape, pending = list(main), {} + for key in groups.values(): + differs = {n for n in found if found[n][key] != variables[n].get(key)} + if not differs: + continue + if readers[key] == 1 and key != BODY_FIELD: + for n in differs: + pending.setdefault(n, {})[key] = found[n][key] + continue + at = next(i for i, token in enumerate(shape) if token[0] != "t" and token[1] == key) + before = "".join(token[1] for token in shape[:at] if token[0] == "t") + part = _part_name(hole.context + before, taken) + shape = [("o", key, part) if token[0] == "v" and token[1] == key else token for token in shape] + for n in differs: + pending.setdefault(n, {})[part] = found[n][key] + trial = {n: dict(variables[n], **pending.get(n, {})) for n in hole.values} + if all(_render_shape(tuple(shape), trial[n]) == hole.values[n] for n in hole.values): + for n, values in pending.items(): + variables[n].update(values) + hole.kind, hole.branches, hole.part_pages = "shape", [(None, tuple(shape))], set() + + +def derive(topics: list) -> Derivation: + """Derives the page template, each topic's JSON and the jstopics template + from `topics` (sorted by path, so the result is deterministic).""" + result = Derivation() + unfit = result.unfit + + keysets = Counter(tuple(t.fields) for t in topics) + if not keysets: + return result + fields = list(keysets.most_common(1)[0][0]) + result.fields = fields + seeded, body_text = {}, {} + for t in topics: + if tuple(t.fields) != tuple(fields): + unfit[t.name] = (f"its jstopics file has fields {', '.join(t.fields)}; " + f"the other topics have {', '.join(fields)}") + continue + if "\x00" in t.page: + unfit[t.name] = "the page contains a NUL character" + continue + body = t.fields[BODY_FIELD] + for form in BODY_FORMS: + in_page = form.to_page(body) + # A form counts only when it maps the page's body back to the jstopics one exactly: + # a body with plain "\r\n" breaks would otherwise pass as "lfcr" and get them rewritten. + if in_page and form.to_jstopic(in_page) == body and t.page.count(in_page) == 1: + result.body_form[t.name] = form + body_text[t.name] = in_page + seeded[t.name] = t.page.replace(in_page, BODY_MARKER) + break + else: + unfit[t.name] = "its jstopics hmBody does not occur exactly once in the page" + by_name = {t.name: t for t in topics} + + lines = {n: split_lines(s) for n, s in seeded.items()} + if len(lines) < 2: + for n in lines: + unfit[n] = "fewer than two topics to derive a shared template from" + return result + sizes = Counter(len(v) for v in lines.values()) + mode = max(sizes, key=lambda size: (sizes[size], -size)) + base_name = next(n for n in lines if len(lines[n]) == mode) + base = lines[base_name] + maps = {} + for n, own in lines.items(): + matcher = difflib.SequenceMatcher(None, base, own, autojunk=False) + mapping = {} + for a, b, size in matcher.get_matching_blocks(): + for k in range(size): + mapping[a + k] = b + k + shared = len(mapping) / max(len(base), len(own)) + if shared < MIN_SHARED_LINES: + unfit[n] = f"shares only {shared:.0%} of its lines with the other topics' pages" + continue + maps[n] = mapping + names = sorted(maps) + if len(names) < 2: + for n in names: + unfit[n] = "fewer than two topics to derive a shared template from" + return result + anchors = [i for i in range(len(base)) if all(i in maps[n] for n in names)] + + # Static text and holes, in page order. + pieces = [] + bounds = [-1] + anchors + [len(base)] + context = "" + for k in range(len(bounds) - 1): + a, b = bounds[k], bounds[k + 1] + if k > 0: + pieces.append(base[a]) + context = (context + base[a])[-400:] + values = {} + for n in names: + start = maps[n][a] + 1 if a >= 0 else 0 + end = maps[n][b] if b < len(base) else len(lines[n]) + values[n] = "".join(lines[n][start:end]) + if len(set(values.values())) == 1: + pieces.append(values[names[0]]) + context = (context + values[names[0]])[-400:] + continue + split = {n: split_lines(v) for n, v in values.items()} + counts = {len(v) for v in split.values()} + subs = ([{n: split[n][j] for n in names} for j in range(len(split[names[0]]))] + if len(counts) == 1 and counts.pop() > 1 else [values]) + for sub in subs: + if len(set(sub.values())) == 1: + pieces.append(sub[names[0]]) + context = (context + sub[names[0]])[-400:] + else: + pieces.append(Hole(sub, context)) + holes = [p for p in pieces if isinstance(p, Hole)] + result.holes = holes + + # Each page's values, in the form its page shows them. + hole_text = {n: "".join(h.values[n] for h in holes) for n in names} + variables = {} + for n in names: + t = by_name[n] + own = {FILE_KEY: t.file, BODY_FIELD: BODY_MARKER} + for f in fields: + if f == BODY_FIELD: + continue + value = t.fields[f] + escaped = hm_html(value) + own[f] = escaped if escaped != value and escaped in hole_text[n] else value + variables[n] = own + patterns = {} + for n in names: + lookup = {} + for key, value in variables[n].items(): + if key == BODY_FIELD or (len(value) >= MIN_SUBSTITUTED and not pebble_empty(value)): + lookup.setdefault(value, key) + ordered = sorted(lookup, key=lambda v: (-len(v), v)) + patterns[n] = (lookup, re.compile("|".join(map(re.escape, ordered))) if ordered else None) + predicate_names = [FILE_KEY] + [f for f in fields if f != BODY_FIELD] + for hole in holes: + _explain(hole, variables, patterns, predicate_names) + taken = set(fields) | {FILE_KEY} + _refine(holes, variables, taken) + for hole in holes: + if hole.kind == "part": + hole.part = _part_name(hole.context, taken) + + segments = [tag_seg("{# Help+Manual topic page, derived from the export by OfflineDocumentationTools " + "scripts/hm_pages_to_templates.py. Generated: rerun the step, do not edit. #}")] + used = set() + for piece in pieces: + if not isinstance(piece, Hole): + for i, part in enumerate(piece.split(BODY_MARKER)): + if i: + segments.append(tag_seg(var_tag(BODY_FIELD))) + used.add(BODY_FIELD) + segments.append(text_seg(part)) + continue + hole = piece + used.update(hole.keys()) + if hole.kind == "part": + if hole.branches: + segments.append(tag_seg("{% if " + hole.part + " is null %}")) + segments.extend(_shape_segments(hole.branches[0][1])) + segments.append(tag_seg("{% else %}")) + segments.append(tag_seg(var_tag(hole.part))) + segments.append(tag_seg("{% endif %}")) + else: + segments.append(tag_seg(var_tag(hole.part))) + elif hole.kind == "choice": + for index, (predicate, shape) in enumerate(hole.branches): + if predicate is None: + segments.append(tag_seg("{% else %}")) + else: + segments.append(tag_seg(_predicate_tag(predicate, index == 0))) + segments.extend(_shape_segments(shape)) + segments.append(tag_seg("{% endif %}")) + else: + segments.extend(_shape_segments(hole.branches[0][1])) + result.template = emit(segments) + + # The JSON: every jstopics field (phase 2 builds jstopics/X.js from it) in + # the form the page shows it, the file name if the template uses it, and + # the "part_" keys a page needs. + for n in names: + values = dict(variables[n], **{BODY_FIELD: body_text[n]}) + context = {} + if FILE_KEY in used: + context[FILE_KEY] = values[FILE_KEY] + for f in fields: + context[f] = values[f] + for key in sorted(k for k in values if k.startswith(PART_PREFIX)): + context[key] = values[key] + for hole in holes: + if hole.kind == "part" and n in hole.part_pages: + context[hole.part] = hole.values[n].replace(BODY_MARKER, body_text[n]) + result.contexts[n] = context + _derive_jstopic(result, by_name, names) + return result + + +def _derive_jstopic(result: Derivation, by_name: dict, names: list) -> None: + """The phase-2 template that builds jstopics/X.js from a page's JSON, and + the "js_" keys it needs where a field's jstopics value is not the page's.""" + forms = Counter(result.body_form[n].name for n in names) + body_form = next(f for f in BODY_FORMS if f.name == forms.most_common(1)[0][0]) + + def jstopic_value(field, value): + return body_form.to_jstopic(value) if field == BODY_FIELD else value + + needs_override = set() + for n in names: + context = result.contexts[n] + for f in result.fields: + stored = by_name[n].fields[f] + if jstopic_value(f, context[f]) != stored: + context[JS_PREFIX + f] = stored + needs_override.add(f) + frames = Counter(by_name[n].frame for n in names if by_name[n].frame) + if not frames: + return # no jstopics file left to take the file's frame from + frame = frames.most_common(1)[0][0] + segments, pos = [], 0 + for match in _JS_FIELD_SLOT.finditer(frame): + field = match.group(1) + if field not in result.fields: + continue + quote = match.end() - 2 + segments.append(text_seg(frame[pos:quote + 1])) + expression = field + (body_form.pebble_filter if field == BODY_FIELD else "") + if field in needs_override: + expression = f"({JS_PREFIX}{field} is null ? ({expression}) : {JS_PREFIX}{field})" + segments.append(tag_seg("{{ " + expression + " | escape('js') }}")) + pos = quote + 1 + segments.append(text_seg(frame[pos:])) + result.jstopic_template = emit(segments) + + +def jstopic_fields_from_context(context: dict, fields: list, body_form: BodyForm) -> dict: + """What the phase-2 template builds for each jstopics field, before + JavaScript escaping - so a topic converted earlier can be re-derived when + its jstopics row is gone.""" + out = {} + for f in fields: + if JS_PREFIX + f in context: + out[f] = context[JS_PREFIX + f] + elif f == BODY_FIELD: + out[f] = body_form.to_jstopic(context.get(f, "")) + else: + out[f] = context.get(f, "") + return out + + +def context_json(context: dict) -> bytes: + """A context as stored: compact, UTF-8, the way populate_db.py writes its + JSON pages.""" + return json.dumps(context, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + +# --- the renderer ------------------------------------------------------------ + +_JARS = ( + ("io.pebbletemplates", "pebble", "4.1.1"), + ("org.unbescape", "unbescape", None), + ("org.slf4j", "slf4j-api", None), + # The Gson CodeOnTheGo's :common module renders documentation with (libs.google.gson). + ("com.google.code.gson", "gson", "2.12.1"), +) + + +def _version_key(version: str) -> tuple: + return tuple(int(p) if p.isdigit() else -1 for p in re.split(r"[.\-]", version)) + + +def find_classpath(extra=()) -> str: + """Pebble 4.1.1 (the app's version) and what Render.java needs beside it, + from the Gradle and Maven caches. Raises naming whatever is missing.""" + home = Path.home() + roots = [home / ".gradle/caches/modules-2/files-2.1", home / ".m2/repository"] + found, missing = [], [] + for group, artifact, version in _JARS: + hits = [] + for root in roots: + base = root / group / artifact if "gradle" in str(root) else root / group.replace(".", "/") / artifact + if not base.is_dir(): + continue + for jar in base.glob(f"{version or '*'}/**/{artifact}-*.jar"): + ver = jar.name[len(artifact) + 1:-4] + if (version is None or ver == version) and not ver.endswith(("sources", "javadoc")): + hits.append((_version_key(ver), str(jar))) + if hits: + found.append(max(hits)[1]) + else: + missing.append(f"{group}:{artifact}:{version or '*'}") + if missing: + raise RuntimeError("cannot find " + ", ".join(missing) + " in the Gradle or Maven cache; pass --classpath") + return os.pathsep.join(list(extra) + found) + + +class PebbleHarness: + """Render.java, kept running: templates are defined over its stdin and + pages rendered with the engine settings the app uses.""" + + def __init__(self, classpath: str, java: str = "java"): + self._log = tempfile.TemporaryFile() + command = [java, "-cp", classpath, str(RENDER_JAVA)] + self._proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self._log) + self._lock = threading.Lock() + + def _request(self, command: str, name: str, payload: bytes) -> tuple: + with self._lock: + try: + self._proc.stdin.write(f"{command} {name} {len(payload)}\n".encode("ascii") + payload) + self._proc.stdin.flush() + header = self._proc.stdout.readline().decode("ascii").split() + except (BrokenPipeError, OSError): + header = [] + if len(header) != 3: + self._log.seek(0) + raise RuntimeError("the Pebble renderer stopped: " + + self._log.read().decode(errors="replace").strip()[-2000:]) + body = self._proc.stdout.read(int(header[1])) + return header[0] == "OK", body, int(header[2]) + + def define(self, name: str, text: str) -> None: + ok, body, _ = self._request("T", name, text.encode("utf-8")) + if not ok: + raise RuntimeError(body.decode(errors="replace")) + + def render(self, name: str, context: bytes) -> tuple: + """(True, page bytes, nanoseconds) or (False, error message, 0).""" + ok, body, nanos = self._request("R", name, context) + return ok, (body if ok else body.decode(errors="replace")), nanos + + def close(self) -> None: + try: + self._proc.stdin.write(b"Q - 0\n") + self._proc.stdin.close() + self._proc.wait(timeout=30) + except Exception: # noqa: BLE001 - shutting down regardless + self._proc.kill() + self._log.close() + + +def body_form_of(template: str) -> BodyForm: + """The body form a stored jstopics template was derived with: the one whose + filter it applies to hmBody, else the one with no filter.""" + for form in BODY_FORMS: + if form.pebble_filter and BODY_FIELD + form.pebble_filter in template: + return form + return next(form for form in BODY_FORMS if not form.pebble_filter) + + +def context_fields(context: dict) -> list: + """The jstopics fields a page's JSON holds, in their order.""" + return [k for k in context if k != FILE_KEY and not k.startswith((PART_PREFIX, JS_PREFIX))] + + +def template_text(content) -> str: + """A Templates row's content: a UTF-8 BLOB as written here, or TEXT if + something else wrote it - the app reads either.""" + return content if isinstance(content, str) else bytes(content).decode("utf-8") + + +def first_difference(a: bytes, b: bytes) -> int: + for i, (x, y) in enumerate(zip(a, b)): + if x != y: + return i + return min(len(a), len(b)) + + +# --- the run ----------------------------------------------------------------- + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + prefix = cfg["prefix"] + log = (lambda m: print(m)) if cfg["verbose"] else (lambda m: None) + + harness = cfg.get("renderer") + own_harness = harness is None + if own_harness: + try: + classpath = cfg["classpath"] or find_classpath() + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + harness = PebbleHarness(classpath, cfg["java"]) + + conn = sqlite3.connect(db_path) + codec = None + changes = [] + try: + codec = BrotliCodec(load_dictionary(conn)) + stored_template = conn.execute("SELECT id, content FROM Templates WHERE name = ?", + (PAGE_TEMPLATE,)).fetchone() + stored_id = stored_template[0] if stored_template else None + stored_text = template_text(stored_template[1]) if stored_template else None + stored_jstopic = conn.execute("SELECT content FROM Templates WHERE name = ?", (JSTOPIC_TEMPLATE,)).fetchone() + stored_jstopic_text = template_text(stored_jstopic[0]) if stored_jstopic else None + # The prefix is matched exactly: LIKE ignores case and reads "_" as a wildcard. + js_dir = prefix + "jstopics/" + rows = [row for row in conn.execute( + "SELECT c.path, c.content, c.languageID, c.contentTypeID, c.templateId, ct.compression " + "FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " + "WHERE ct.value = ? AND substr(c.path, 1, ?) = ? ORDER BY c.path", (CONTENT_TYPE, len(prefix), prefix) + ).fetchall() if row[0].startswith(prefix)] + lengths = {row[0]: len(row[1]) for row in rows} + jstopic_rows = [row for row in conn.execute( + "SELECT c.path, ct.compression, LENGTH(c.content) FROM Content c " + "JOIN ContentTypes ct ON c.contentTypeID = ct.id WHERE substr(c.path, 1, ?) = ?", + (len(js_dir), js_dir)).fetchall() if row[0].startswith(js_dir)] + jstopic_lengths = {path: length for path, _c, length in jstopic_rows} + jstopics = {path: compression for path, compression, _l in jstopic_rows + if not is_continuation_path(jstopic_lengths, path)} + + candidates, skipped = [], [] + for path, blob, language_id, type_id, template_id, compression in rows: + rest = path[len(prefix):] + # Only X.html is a topic page: the app builds jstopics/X.js from X.html and from + # nothing else, so a page named otherwise keeps its jstopics row. + if is_continuation_path(lengths, path) or "/" in rest or not rest.endswith(".html"): + continue + stem = rest[:-len(".html")] + jstopic_path = f"{js_dir}{stem}.js" + ours = stored_id is not None and template_id == stored_id + if template_id and not ours: + skipped.append((path, f"rendered through another template (templateId {template_id})")) + continue + if jstopic_path not in jstopics and not ours: + continue # not a topic: hmftsearch.html, _hmXtopic.htm, ... + candidates.append((path, stem, reassemble(conn, path, blob), len(blob), language_id, type_id, + template_id, compression, jstopic_path)) + jstopic_blobs = {} + for c in candidates: + if c[8] in jstopics: + row = conn.execute("SELECT content FROM Content WHERE path = ?", (c[8],)).fetchone() + jstopic_blobs[c[8]] = (reassemble(conn, c[8], row[0]), jstopics[c[8]], len(row[0])) + print(f"Found {len(candidates)} Help+Manual topic page(s) under {prefix} " + f"({sum(1 for c in candidates if c[6])} already stored as JSON).") + + def decode(c): + path, stem, full, base_len, lang, ctid, tid, compression, jpath = c + try: + page = codec.decompress(full, compression) + jst = None + if jpath in jstopic_blobs: + jst = codec.decompress(*jstopic_blobs[jpath][:2]).decode("utf-8") + return path, page, jst, None + except Exception as exc: # noqa: BLE001 - one unreadable row is not the run + return path, None, None, f"could not be read: {exc}" + + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + decoded = list(pool.map(decode, candidates)) + + unreadable = [(path, why) for path, _p, _j, why in decoded if why] + info = {c[0]: c for c in candidates} + by_stem = {c[1]: c for c in candidates} + # How a topic whose jstopics row is gone maps its page body back: the way the stored + # jstopics template does, which is what the app serves. + stored_form = body_form_of(stored_jstopic_text) if stored_jstopic_text is not None else BODY_FORMS[0] + stored_json = {} + topics, original, unfit = [], {}, {} + if stored_text is not None: + harness.define("stored:" + PAGE_TEMPLATE, stored_text) + for path, page, jst, why in decoded: + if why: + continue + stem = info[path][1] + if info[path][6]: + try: + context = json.loads(page.decode("utf-8")) + except ValueError as exc: + unreadable.append((path, f"its stored JSON does not parse: {exc}")) + continue + ok, out, _ = harness.render("stored:" + PAGE_TEMPLATE, page) + if not ok: + unreadable.append((path, f"its stored JSON does not render: {out}")) + continue + stored_json[stem] = page + page = out + if jst is None: + parsed = (jstopic_fields_from_context(context, context_fields(context), stored_form), None) + else: + parsed = parse_jstopic(jst) + else: + parsed = parse_jstopic(jst) if jst is not None else None + original[stem] = page + if parsed is None: + unfit[stem] = "its jstopics file has no hmBody field" + continue + try: + text = page.decode("utf-8") + except UnicodeDecodeError as exc: + unfit[stem] = f"the page is not valid UTF-8 ({exc.reason} at byte {exc.start:,})" + continue + frame = parsed[1] if parsed[1] is not None else "" + topics.append(Topic(stem, path, text, parsed[0], frame)) + + derivation = derive(topics) + unfit.update(derivation.unfit) + template = derivation.template + + # The render check: each converted page must come back byte for byte. + nanos = [] + jsons = {} + if template is not None: + harness.define(PAGE_TEMPLATE, template) + for stem, context in derivation.contexts.items(): + data = context_json(context) + ok, out, took = harness.render(PAGE_TEMPLATE, data) + if not ok: + unfit[stem] = f"the template does not render it: {out}" + elif out != original[stem]: + unfit[stem] = (f"its rendered page differs from the stored one at byte " + f"{first_difference(out, original[stem]):,}") + else: + jsons[stem] = data + nanos.append(took) + + # Phase 2: does the jstopics template rebuild each topic's stored file? Compared field by field + # after JavaScript unescaping, and outside the fields too, so the file is the same program. + jstopic_template = derivation.jstopic_template or stored_jstopic_text + droppable, kept = [], {} + with_row = [t for t in topics if f"{js_dir}{t.name}.js" in jstopic_blobs] + if jstopic_template is not None and with_row: + harness.define(JSTOPIC_TEMPLATE, jstopic_template) + for t in with_row: + if t.name not in jsons: + kept[t.name] = "its page stays HTML, so nothing can build its jstopics file" + continue + ok, out, _ = harness.render(JSTOPIC_TEMPLATE, jsons[t.name]) + parsed = parse_jstopic(out.decode("utf-8")) if ok else None + if not ok: + kept[t.name] = f"the jstopics template does not render it: {out}" + elif parsed is None or parsed[0] != t.fields: + differ = [f for f in t.fields if parsed is None or parsed[0].get(f) != t.fields[f]] + kept[t.name] = f"the built file differs in {', '.join(differ)}" + elif parsed[1] != t.frame: + kept[t.name] = "the built file differs outside its field values" + else: + droppable.append(t.name) + elif with_row: + for t in with_row: + kept[t.name] = "no jstopics template could be derived" + # A topic found unfit before it became a Topic (no hmBody field, a page that is not UTF-8), + # or one that could not be read, keeps its row too, and with --drop-jstopics that fails the run. + for c in candidates: + if c[8] in jstopic_blobs and c[1] not in droppable and c[1] not in kept: + kept[c[1]] = (f"its page stays HTML ({unfit[c[1]]})" if c[1] in unfit + else "its page or its jstopics file could not be read") + # Topics whose row is already gone: what the app builds for them must still hold the fields + # their JSON holds, or in-page navigation to them breaks. + without_row = [t for t in topics if t not in with_row and t.name in jsons] + broken = {} + if without_row: + if jstopic_template is None: + broken = {t.name: f"its jstopics row is gone and there is no {JSTOPIC_TEMPLATE}" for t in without_row} + else: + harness.define(JSTOPIC_TEMPLATE, jstopic_template) + for t in without_row: + ok, out, _ = harness.render(JSTOPIC_TEMPLATE, jsons[t.name]) + parsed = parse_jstopic(out.decode("utf-8")) if ok else None + if parsed is None or parsed[0] != t.fields: + broken[t.name] = "the jstopics file built from its JSON does not hold the JSON's fields" + orphans = sorted(set(jstopics) - {c[8] for c in candidates}) + if cfg.get("jstopic_template_out") and derivation.jstopic_template is not None: + Path(cfg["jstopic_template_out"]).write_text(derivation.jstopic_template, encoding="utf-8", newline="") + + for hole in derivation.holes: + pages = len(hole.values) + if hole.kind == "part": + log(f" [HOLE] after {hole.context[-60:]!r}: kept as page text ({hole.part}) on " + f"{len(hole.part_pages)} of {pages} page(s)") + elif hole.kind == "choice": + conds = ", ".join(f"{p[0]} is {'not ' if p[1] else ''}empty" for p, _s in hole.branches if p) + log(f" [HOLE] after {hole.context[-60:]!r}: {len(hole.branches)} shapes chosen by {conds}") + else: + log(f" [HOLE] after {hole.context[-60:]!r}: one shape") + + # What changes. + template_changes = bool(template is not None and jsons and template != stored_text) + drop = cfg.get("drop_jstopics", False) + jstopic_changes = bool(drop and droppable and derivation.jstopic_template is not None + and derivation.jstopic_template != stored_jstopic_text) + stored_as_json, back_to_html = set(), [] + for stem, page in original.items(): + path, _stem, full, base_len, lang, ctid, tid, compression, jpath = by_stem[stem] + if stem in jsons: + stored_as_json.add(stem) + if tid and codec.decompress(full, compression) == jsons[stem]: + continue # already this JSON; the template renders it the same way + changes.append(("json", path, base_len, jsons[stem], lang, ctid, compression, len(full), + len(page))) + elif tid: + # Stored as JSON for a template it no longer fits: back to HTML. + changes.append(("html", path, base_len, page, lang, ctid, compression, len(full), len(page))) + if jpath not in jstopic_blobs: + back_to_html.append(stem) + # A row is dropped only for a page that ends up stored as JSON, written now or already. + drops = [] + if drop: + for stem in droppable: + if stem in stored_as_json: + jpath = f"{js_dir}{stem}.js" + drops.append((jpath, jstopic_blobs[jpath][2], len(jstopic_blobs[jpath][0]))) + + # A page going back to HTML whose jstopics row is gone gets the row back, built as the app + # builds it now - the stored jstopics template and the page's stored JSON - and checked + # against the fields that JSON holds. Writing it back rather than refusing the revert keeps + # the run able to replace a template that no longer fits its pages; a row that cannot be + # built that way stops the whole run instead, since keeping the page as JSON under the new + # template would serve the wrong page. + restores, refuse = [], {} + js_type = conn.execute("SELECT id, compression FROM ContentTypes WHERE value = ?", + (JSTOPIC_CONTENT_TYPE,)).fetchone() + if back_to_html and stored_jstopic_text is not None: + harness.define("stored:" + JSTOPIC_TEMPLATE, stored_jstopic_text) + for stem in back_to_html: + path, _stem, _full, _bl, lang, _ct, _tid, _comp, jpath = by_stem[stem] + if stored_jstopic_text is None or js_type is None: + refuse[stem] = (f"its page goes back to HTML, its jstopics row is gone, and there is no " + f"{JSTOPIC_TEMPLATE if js_type else JSTOPIC_CONTENT_TYPE} to write it back with") + continue + context = json.loads(stored_json[stem].decode("utf-8")) + expected = jstopic_fields_from_context(context, context_fields(context), stored_form) + ok, out, _ = harness.render("stored:" + JSTOPIC_TEMPLATE, stored_json[stem]) + parsed = parse_jstopic(out.decode("utf-8")) if ok else None + if parsed is None or parsed[0] != expected: + refuse[stem] = "its page goes back to HTML and its jstopics row cannot be built again" + continue + restores.append((jpath, out, lang, js_type[0], js_type[1])) + + def encode(change): + kind, path, base_len, data, lang, ctid, compression, old_stored, old_decoded = change + return change + (codec.compress(data, compression),) + + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + changes = list(pool.map(encode, changes)) + restores = list(pool.map(lambda r: r + (codec.compress(r[1], r[4]),), restores)) + + converted = [c for c in changes if c[0] == "json"] + reverted = [c for c in changes if c[0] == "html"] + stored_before = sum(c[7] for c in changes) + stored_after = sum(len(c[9]) for c in changes) + decoded_before = sum(c[8] for c in changes) + decoded_after = sum(len(c[3]) for c in changes) + + print(f"Derived a {human(len(template.encode('utf-8')) if template else 0)}-byte page template " + f"from {len(derivation.contexts)} topic(s): {len(derivation.holes)} hole(s), " + f"{sum(1 for h in derivation.holes if h.kind == 'choice')} with {{% if %}}, " + f"{sum(1 for h in derivation.holes if h.kind == 'part')} kept as page text.") + print(f"{len(jsons)} of {len(original)} topic page(s) render back byte for byte and are stored as JSON; " + f"{len(unfit)} stay HTML.") + if nanos: + ms = sorted(n / 1e6 for n in nanos) + print(f"Render time per page: median {statistics.median(ms):.2f} ms, worst {ms[-1]:.2f} ms.") + if with_row: + print(f"Phase 2 check: the jstopics template rebuilds {len(droppable)} of {len(with_row)} stored " + f"jstopics file(s) exactly.") + elif drop: + print("No jstopics rows left to drop.") + if without_row: + print(f"Phase 2 check: {len(without_row) - len(broken)} of {len(without_row)} topic(s) without a " + f"jstopics row get one built from their JSON with the fields it holds.") + for stem in sorted(broken): + print(f" broken: {js_dir}{stem}.js - {broken[stem]}", file=sys.stderr) + for stem in sorted(refuse): + print(f" refused: {prefix}{stem}.html - {refuse[stem]}", file=sys.stderr) + for jpath, *_rest in restores: + print(f" writes back {jpath} - its page goes back to HTML", file=sys.stderr) + if drop: + for stem in sorted(kept): + print(f" keeps {js_dir}{stem}.js - {kept[stem]}", file=sys.stderr) + for path in orphans: + print(f" left alone: {path} - no topic page {prefix}X.html of that name", file=sys.stderr) + else: + for stem in sorted(kept): + log(f" [JSTOPIC] {stem}: {kept[stem]}") + for stem in sorted(unfit): + print(f" stays HTML: {prefix}{stem}.html - {unfit[stem]}", file=sys.stderr) + for path, why in skipped: + print(f" skipped {path}: {why}", file=sys.stderr) + for path, why in unreadable: + print(f" skipped {path}: {why}", file=sys.stderr) + if changes or template_changes: + print(f"{'Would write' if cfg['dry_run'] else 'Writing'} the template" + f"{' (changed)' if template_changes else ' (unchanged)'}, {len(converted)} page(s) as JSON" + f"{f' and {len(reverted)} back as HTML' if reverted else ''}: {human(stored_before)} -> " + f"{human(stored_after)} stored byte(s), {human(decoded_before)} -> {human(decoded_after)} decoded.") + if drops or jstopic_changes: + print(f"{'Would drop' if cfg['dry_run'] else 'Dropping'} {len(drops)} jstopics row(s) " + f"({human(sum(d[2] for d in drops))} stored byte(s)), the app building each from its page with " + f"the {JSTOPIC_TEMPLATE} template{' (changed)' if jstopic_changes else ' (unchanged)'}.") + anything = bool(changes or template_changes or drops or jstopic_changes or restores) + if refuse: + print(f"error: {len(refuse)} page(s) cannot go back to HTML (listed above); nothing written.", + file=sys.stderr) + elif not anything: + print("Nothing to change.") + + wrote = False + if cfg["dry_run"]: + print("Dry run: nothing written.") + elif anything and not refuse: + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + template_id = stored_id + if template_changes: + conn.execute("INSERT INTO Templates (name, content) VALUES (?, ?) " + "ON CONFLICT(name) DO UPDATE SET content = excluded.content", + (PAGE_TEMPLATE, template.encode("utf-8"))) + template_id = conn.execute("SELECT id FROM Templates WHERE name = ?", + (PAGE_TEMPLATE,)).fetchone()[0] + for kind, path, base_len, data, lang, ctid, compression, _os, _od, stored in changes: + new_tid = template_id if kind == "json" else 0 + replace_row(conn, path, base_len, stored, lang, ctid, new_tid, + lambda m: print(m, file=sys.stderr)) + conn.execute("UPDATE Content SET templateId = ? WHERE path = ?", (new_tid, path)) + log(f" [{kind.upper()}] {path}") + if jstopic_changes: + conn.execute("INSERT INTO Templates (name, content) VALUES (?, ?) " + "ON CONFLICT(name) DO UPDATE SET content = excluded.content", + (JSTOPIC_TEMPLATE, derivation.jstopic_template.encode("utf-8"))) + for jpath, base_len, _stored in drops: + delete_media_row(conn, jpath, base_len) + log(f" [DROP] {jpath}") + for jpath, _out, lang, type_id, _compression, stored in restores: + write_chunks(conn, jpath, stored, lang, type_id, 0, lambda m: print(m, file=sys.stderr), + update_base=False) + log(f" [RESTORE] {jpath}") + conn.commit() + wrote = True + except Exception: + conn.rollback() + raise + finally: + if codec is not None: + codec.close() + conn.close() + if own_harness: + harness.close() + + if wrote: + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + if unreadable: + print(f"error: {len(unreadable)} row(s) could not be read and were left as they were (listed above).", + file=sys.stderr) + # With --drop-jstopics every jstopics row is meant to go; any left is a failure. + failed_drops = bool(cfg.get("drop_jstopics") and (kept or orphans)) + if failed_drops: + print(f"error: {len(kept) + len(orphans)} jstopics row(s) left in place (listed above).", file=sys.stderr) + if broken: + print(f"error: {len(broken)} topic(s) without a jstopics row would get a wrong one (listed above).", + file=sys.stderr) + if restores: + print(f"error: {len(restores)} page(s) went back to HTML after their jstopics row was dropped; " + f"the row{'s were' if len(restores) > 1 else ' was'} written back (listed above).", file=sys.stderr) + return 1 if unreadable or failed_drops or broken or restores or refuse else 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to edit in place, e.g. documentation.db") + p.add_argument("--dry-run", action="store_true", + help="Do all the work, the render check included, and report what would change without writing") + p.add_argument("--workers", type=int, default=8, + help="Threads used to decompress and recompress rows (default: 8)") + p.add_argument("--verbose", action="store_true", help="Describe every hole and log every page written") + p.add_argument("--prefix", default="i/", help="Where the Help+Manual export is stored (default: i/)") + p.add_argument("--classpath", default=os.environ.get("HM_RENDER_CLASSPATH"), + help="Classpath for Render.java: Pebble 4.1.1, unbescape, slf4j-api and Gson jars " + "(default: $HM_RENDER_CLASSPATH, else found in the Gradle or Maven cache)") + p.add_argument("--java", default="java", help="The java launcher, 17 or later (default: java)") + p.add_argument("--jstopic-template", type=Path, metavar="FILE", + help="Also write the phase-2 template that builds jstopics/X.js from a page's JSON to FILE") + p.add_argument("--drop-jstopics", action="store_true", + help="Phase 2: store the jstopics template and drop every jstopics row it rebuilds exactly. " + "Needs an app that builds jstopics/X.js from the template") + return p + + +def main() -> None: + args = build_parser().parse_args() + sys.exit(run({"db_path": args.db_path, "dry_run": args.dry_run, "workers": args.workers, + "verbose": args.verbose, "prefix": args.prefix, "classpath": args.classpath, + "java": args.java, "jstopic_template_out": args.jstopic_template, + "drop_jstopics": args.drop_jstopics})) + + +if __name__ == "__main__": + main() diff --git a/scripts/hm_render_check/Render.java b/scripts/hm_render_check/Render.java new file mode 100644 index 00000000..f914d4ec --- /dev/null +++ b/scripts/hm_render_check/Render.java @@ -0,0 +1,170 @@ +/* + * Render.java - renders documentation.db templates with the Pebble engine the app uses. + * + * Built the way CodeOnTheGo's DocumentationContentSource builds its engine (Pebble 4.1.1, + * default settings - HTML autoescaping on, newline trimming on, strict variables off - and + * maxRenderedSize of 4 Mi chars), and given its context the same way: Gson with + * ToNumberPolicy.LONG_OR_DOUBLE parses the row's JSON into a Map, and the + * rendered String is returned as UTF-8 bytes. Gson 2.12.1 is the version CodeOnTheGo's :common + * module renders with. + * + * Templates are defined over stdin and resolve by exact name, as the app's + * DatabaseTemplateLoader resolves Templates rows, so a caller can check a template before it is + * written anywhere. + * + * Run from source, no build step: + * + * java -cp pebble-4.1.1.jar:unbescape-1.1.6.RELEASE.jar:slf4j-api.jar:gson-2.12.1.jar Render.java + * + * Protocol, on stdin and stdout: each request is one ASCII header line, " ", + * followed by bytes. + * + * T define (or replace) template ; the bytes are its UTF-8 text + * R render template ; the bytes are the JSON context + * Q - 0 quit + * + * Each request gets one header line, "OK " or "ERR 0", followed by + * bytes: the rendered page, or the error message. is the time from parsing the + * JSON to holding the rendered bytes. + */ + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; +import com.google.gson.reflect.TypeToken; +import io.pebbletemplates.pebble.PebbleEngine; +import io.pebbletemplates.pebble.error.LoaderException; +import io.pebbletemplates.pebble.error.PebbleException; +import io.pebbletemplates.pebble.loader.Loader; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class Render { + // DocumentationContentSource.MAX_RENDERED_CHARS + static final int MAX_RENDERED_CHARS = 4 * 1024 * 1024; + + static final Map defined = new ConcurrentHashMap<>(); + + /** Resolves a name to a template defined over stdin, exactly, like DatabaseTemplateLoader. */ + static final class Templates implements Loader { + @Override + public Reader getReader(String name) { + String text = defined.get(name); + if (text == null) throw new LoaderException(null, "Template '" + name + "' not found"); + return new StringReader(text); + } + + @Override + public boolean resourceExists(String name) { + return defined.containsKey(name); + } + + @Override + public String createCacheKey(String name) { + return name; + } + + @Override + public String resolveRelativePath(String relativePath, String anchorPath) { + return relativePath; + } + + @Override + public void setCharset(String charset) { + } + + @Override + public void setPrefix(String prefix) { + } + + @Override + public void setSuffix(String suffix) { + } + } + + static PebbleEngine newEngine() { + return new PebbleEngine.Builder().loader(new Templates()).maxRenderedSize(MAX_RENDERED_CHARS).build(); + } + + public static void main(String[] args) throws Exception { + if (args.length != 0) { + System.err.println("usage: Render.java (templates and contexts come over stdin)"); + System.exit(2); + } + Gson gson = new GsonBuilder().setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create(); + Type contextType = new TypeToken>() {}.getType(); + PebbleEngine engine = newEngine(); + + InputStream in = new BufferedInputStream(System.in, 1 << 16); + OutputStream out = new BufferedOutputStream(System.out, 1 << 16); + while (true) { + String header = readLine(in); + if (header == null) break; + String[] parts = header.split(" "); + if (parts.length != 3) { + reply(out, false, ("bad request header: " + header).getBytes(StandardCharsets.UTF_8), 0); + break; + } + byte[] payload = in.readNBytes(Integer.parseInt(parts[2])); + switch (parts[0]) { + case "Q": + out.flush(); + return; + case "T": + defined.put(parts[1], new String(payload, StandardCharsets.UTF_8)); + // A replaced template must not be served from the engine's cache. + engine = newEngine(); + reply(out, true, new byte[0], 0); + break; + case "R": + long start = System.nanoTime(); + try { + String json = new String(payload, StandardCharsets.UTF_8); + if (json.isBlank() || json.trim().equals("null")) { + throw new IllegalArgumentException("Template '" + parts[1] + "' has empty or null JSON context"); + } + Map context = gson.fromJson(json, contextType); + StringWriter writer = new StringWriter(); + engine.getTemplate(parts[1]).evaluate(writer, context); + byte[] page = writer.toString().getBytes(StandardCharsets.UTF_8); + reply(out, true, page, System.nanoTime() - start); + } catch (PebbleException e) { + String message = e.getFileName() == null && e.getLineNumber() == null ? e.getPebbleMessage() : e.getMessage(); + reply(out, false, String.valueOf(message).getBytes(StandardCharsets.UTF_8), 0); + } catch (Exception | StackOverflowError e) { + reply(out, false, (e.getClass().getSimpleName() + ": " + e.getMessage()).getBytes(StandardCharsets.UTF_8), 0); + } + break; + default: + reply(out, false, ("unknown command: " + parts[0]).getBytes(StandardCharsets.UTF_8), 0); + } + } + out.flush(); + } + + static String readLine(InputStream in) throws IOException { + ByteArrayOutputStream line = new ByteArrayOutputStream(); + int c; + while ((c = in.read()) != -1 && c != '\n') line.write(c); + if (c == -1 && line.size() == 0) return null; + return line.toString(StandardCharsets.US_ASCII); + } + + static void reply(OutputStream out, boolean ok, byte[] body, long nanos) throws IOException { + out.write(((ok ? "OK " : "ERR ") + body.length + " " + nanos + "\n").getBytes(StandardCharsets.US_ASCII)); + out.write(body); + out.flush(); + } +} diff --git a/scripts/test_hm_pages_to_templates.py b/scripts/test_hm_pages_to_templates.py new file mode 100644 index 00000000..1b3867e3 --- /dev/null +++ b/scripts/test_hm_pages_to_templates.py @@ -0,0 +1,895 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "pytest", +# ] +# /// +""" +test_hm_pages_to_templates.py + +Covers hm_pages_to_templates.py: deriving the page template from small +synthetic Help+Manual pages, the JSON each page gets, that every page renders +back byte for byte through the real Pebble engine, that a second run changes +nothing, that a page which does not fit - or does not render back - stays HTML +and is named, and that a dry run writes nothing. For phase 2 +(--drop-jstopics): that the jstopics template is stored and the rows it +rebuilds are dropped, chunks included, and that a topic it does not rebuild +keeps its row, is named and fails the run. + +The synthetic topics copy the real export's habits that matter here: CRLF +pages with a BOM, jstopics bodies whose line breaks are "\\n\\r", a first and a +last topic with disabled links, a top-level topic without breadcrumbs, a meta +description Help+Manual writes differently from its jstopics value, and a +caption that is not the topic's title. + +Run it either way - as a plain script: + + uv run scripts/test_hm_pages_to_templates.py + +or under pytest: + + uv run --with brotli --with Pillow --with pytest pytest scripts/test_hm_pages_to_templates.py + +Tests that render need `java` (17 or later) and the Pebble 4.1.1, unbescape, +slf4j-api and Gson jars in the Gradle or Maven cache (or $HM_RENDER_CLASSPATH); +without them they are skipped. Every test needs the `brotli` CLI on PATH. +""" +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import hm_pages_to_templates as hm # noqa: E402 + + +class Skip(Exception): + pass + + +def _skip(reason: str): + try: + import pytest + except ImportError: + raise Skip(reason) + pytest.skip(reason) + + +_CLASSPATH = [] + + +def _classpath() -> str: + if not _CLASSPATH: + try: + if shutil.which("java") is None: + raise RuntimeError("no java on PATH") + _CLASSPATH.append(os.environ.get("HM_RENDER_CLASSPATH") or hm.find_classpath()) + except RuntimeError as exc: + _CLASSPATH.append(exc) + if isinstance(_CLASSPATH[0], Exception): + _skip(f"needs the Pebble renderer: {_CLASSPATH[0]}") + return _CLASSPATH[0] + + +def _harness() -> hm.PebbleHarness: + return hm.PebbleHarness(_classpath()) + + +# --- synthetic Help+Manual topics ---------------------------------------------- + +FILLER = "".join(f"\t.rule{n} {{ margin: {n}px; }}\r\n" for n in range(120)) + +TOPICS = [ + # name, title, caption, description (jstopics), description (page), keywords, crumbs + ("t0", "Start here", "Start here", "Welcome.", "Welcome.", "", ""), + ("t1", "Build", "Build", 'Tap "Run" to build.', "Tap "Run" to build.", "build & run", + 'Start here'), + ("t2", "Debug your app", "Debug", "Pair the device. ", "Pair the device. ", "", + 'Start here'), + ("t3", "Keys", "Keys", "Press a key.", "Press a key.", "keys", + 'Start here > Build'), + ("t4", "Projects", "Projects", "Make a project.", "Make a project.", "", ""), + ("t5", "The end", "The end", "Bye.", "Bye.", "", 'Projects'), +] + + +def _js(value: str) -> str: + return (value.replace("\\", "\\\\").replace('"', '\\"').replace("'", "\\'").replace("/", "\\/") + .replace("\n", "\\n").replace("\r", "\\r")) + + +def _body(index: int, breaks: str = "\n\r") -> str: + """A body with Help+Manual's jstopics line breaks ("\\n\\r"; the page has + CRLF), or with `breaks` for an export that writes them the same way in both.""" + return (f'

Topic {index} text, caf\u00e9 \U0001F600, and {{{{ not a tag }}}}.

{breaks}' + f'

Second paragraph of topic {index}.

') + + +def make_topic(index: int, names=None, breaks: str = "\n\r") -> tuple: + """(page bytes, jstopics bytes) for TOPICS[index].""" + name, title, caption, desc_js, desc_page, keywords, crumbs = TOPICS[index] + names = names or [t[0] for t in TOPICS] + prev = f"{names[index - 1]}.html" if index > 0 else "" + nxt = f"{names[index + 1]}.html" if index + 1 < len(names) else "" + parent = "t0.html" if index else "index.html" + keywords_page = keywords.replace("&", "&") + lines = [ + "\ufeff\r\n", '\r\n', "\t\r\n", + f"\t{title}\r\n", + '\t\r\n', + f'\t\r\n', + f'\t\r\n', + "\t\r\n", + "\t\r\n", + "\r\n", "\r\n", "\r\n", + (f'\t

Navigation: {crumbs}

\r\n' if crumbs else + '\t

Navigation: » No topics above this level «' + '

\r\n'), + f'\t

{caption}

\r\n', + (f'\t\r\n", + (f'\t\r\n", + '\t
\r\n', "\r\n", + "\t" + _body(index, breaks).replace("\n\r", "\r\n") + "\r\n", + "\t
\r\n", + f'\thmFlags.thisTopic = "{name}.html";\r\n', + f'\tvar titleBarText = "{title}";\r\n', + "\r\n", "\r\n", + ] + fields = [("hmKeywords", keywords), ("hmTitle", title), ("hmDescription", desc_js), ("hmPrevLink", prev), + ("hmNextLink", nxt), ("hmParentLink", parent), ("hmBreadCrumbs", crumbs), + ("hmTitlePath", f"Root > {title}"), + ("hmHeader", f'

{caption}

\n\r'), + ("hmBody", _body(index, breaks))] + jstopic = "hmLoadTopic({\r\n" + ",\r\n".join(f'{k}:"{_js(v)}"' for k, v in fields) + "\r\n})\r\n" + return "".join(lines).encode("utf-8"), jstopic.encode("utf-8") + + +def make_topics() -> list: + out = [] + for i, t in enumerate(TOPICS): + page, jstopic = make_topic(i) + fields, frame = hm.parse_jstopic(jstopic.decode("utf-8")) + out.append(hm.Topic(t[0], f"i/{t[0]}.html", page.decode("utf-8"), fields, frame)) + return out + + +# --- a synthetic documentation.db -------------------------------------------- + +DICTIONARY = (b'

' * 50 + b"hmLoadTopic({" * 20 + FILLER.encode() * 3) +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE('path')); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, + UNIQUE('name')); +CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL); +""" + + +def _brotli(data: bytes, decompress: bool = False) -> bytes: + """Dictionary Brotli at quality 5, independent of the tool's codec: the tool + compresses at 11, so a row it rewrote never matches one it left alone.""" + with tempfile.TemporaryDirectory() as directory: + dict_path = Path(directory) / "dictionary.bin" + dict_path.write_bytes(DICTIONARY) + args = ["brotli", "-D", str(dict_path)] + (["-d"] if decompress else ["-q", "5"]) + ["-c"] + return subprocess.run(args, input=data, stdout=subprocess.PIPE, check=True).stdout + + +def make_db(path: Path, pages: dict = None, template_ids: dict = None, breaks: str = "\n\r") -> Path: + """A database with the synthetic topics, their jstopics files, a page that + is not a topic, a page outside i/ and an unrelated template. `pages` + replaces or adds rows (None leaves a row out); `template_ids` sets rows' + templateId.""" + conn = sqlite3.connect(path) + conn.executescript(SCHEMA) + conn.execute("INSERT INTO Languages (value) VALUES ('en-US')") + conn.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (12, 'text/html', 'brotli')") + conn.execute("INSERT INTO ContentTypes (id, value, compression) VALUES (6, 'text/javascript', 'brotli')") + conn.execute("INSERT INTO CompressionDictionary (id, data) VALUES (1, ?)", (DICTIONARY,)) + conn.execute("INSERT INTO Templates (name, content) VALUES ('page.peb', ?)", (b"

{{ title }}

",)) + rows = dict(pages or {}) + for i, t in enumerate(TOPICS): + page, jstopic = make_topic(i, breaks=breaks) + rows.setdefault(f"i/{t[0]}.html", page) + rows.setdefault(f"i/jstopics/{t[0]}.js", jstopic) + rows.setdefault("i/hmftsearch.html", b"\xef\xbb\xbf\r\n

Search

\r\n") + rows.setdefault("a/other.html", b"

Not Help+Manual

") + for p, data in sorted(rows.items()): + if data is None: + continue + type_id = 6 if p.endswith(".js") else 12 + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID, templateId) " + "VALUES (?, 1, ?, ?, ?)", + (p, data if data.startswith(b"RAW:") else _brotli(data), type_id, (template_ids or {}).get(p, 0))) + conn.commit() + conn.close() + return path + + +def rows(path: Path) -> dict: + conn = sqlite3.connect(path) + try: + return {r[0]: r[1:] for r in conn.execute( + "SELECT path, content, languageID, contentTypeID, templateId FROM Content ORDER BY path")} + finally: + conn.close() + + +def sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def run_tool(db: Path, **extra) -> tuple: + cfg = {"db_path": db, "dry_run": False, "workers": 4, "verbose": True, "prefix": "i/", + "classpath": _classpath(), "java": "java", "jstopic_template_out": None} + cfg.update(extra) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = hm.run(cfg) + return code, out.getvalue(), err.getvalue() + + +def backups(db: Path) -> list: + return sorted(db.parent.glob(db.name + ".backup-*")) + + +# --- derivation ---------------------------------------------------------------- + +def test_split_lines_keeps_every_byte(): + for text in ["", "a", "a\r\nb", "a\n\rb\r", "\r\n\r\n", "x\ry\nz\r\n"]: + assert "".join(hm.split_lines(text)) == text + assert hm.split_lines("a\r\nb\rc\nd") == ["a\r\n", "b\r", "c\n", "d"] + + +def test_parse_jstopic_unescapes_fields(): + _page, jstopic = make_topic(1) + fields, frame = hm.parse_jstopic(jstopic.decode("utf-8")) + assert fields["hmDescription"] == 'Tap "Run" to build.' + assert fields["hmBreadCrumbs"] == 'Start here' + assert fields["hmBody"].count("\n\r") == 1 + assert '"' not in frame.replace('""', "") + assert hm.parse_jstopic('hmLoadTopic({hmTitle:"x"})') is None + + +def test_template_has_a_hole_for_each_per_topic_part(): + d = hm.derive(make_topics()) + t = d.template + assert not d.unfit, d.unfit + assert "\t{{ hmTitle | raw }}" in t + assert '' in t + assert '' in t + assert "{% if hmBreadCrumbs is empty %}" in t + assert "Navigation: {{ hmBreadCrumbs | raw }}

" in t + assert "{% if hmPrevLink is empty %}" in t and 'href="{{ hmPrevLink | raw }}"' in t + assert "{% if hmNextLink is empty %}" in t and 'href="{{ hmNextLink | raw }}"' in t + assert "\t{{ hmBody | raw }}\r\n" in t + assert 'hmFlags.thisTopic = "{{ file | raw }}";' in t + assert "{{ (part_topictitle is null ? hmTitle : part_topictitle) | raw }}" in t + # The skin is in the template once, and nothing that belongs to one topic is. + assert t.count(".rule119 { margin: 119px; }") == 1 + for _name, title, *_rest in TOPICS: + assert f">{title}<" not in t + assert "Second paragraph" not in t + # Delimiters in the skin's own text are escaped, not left to Pebble. + assert 'var o = {{ "{" }}{ a: 1 }}; /* {{ "{" }}% not %} {{ "{" }}# a tag #} */' in t + + +def test_json_holds_each_part_as_its_page_shows_it(): + d = hm.derive(make_topics()) + c1, c2 = d.contexts["t1"], d.contexts["t2"] + assert c1["hmDescription"] == "Tap "Run" to build." # escaped, as in the page + assert c1["js_hmDescription"] == 'Tap "Run" to build.' # the jstopics value + assert c1["hmKeywords"] == "build & run" and c1["js_hmKeywords"] == "build & run" + assert c2["hmDescription"] == "Pair the device. " # read out of the page itself + assert c2["js_hmDescription"] == "Pair the device. " + assert c2["part_topictitle"] == "Debug" and c2["hmTitle"] == "Debug your app" + assert "part_topictitle" not in c1 and "js_hmDescription" not in d.contexts["t3"] + assert c1["hmBody"] == _body(1).replace("\n\r", "\r\n") + assert c1["file"] == "t1.html" and c1["hmTitlePath"] == "Root > Build" + assert c1["hmHeader"].endswith("\n\r") + assert list(c1)[:2] == ["file", "hmKeywords"] + + +def test_emit_survives_pebble_lexing(): + texts = ["a{{b", "{%", "{#c#}", "x{", "\r\nafter", "\n\rafter", "\rafter", "\nafter", "\u2028after", + "\r\r\nafter", "}} %} #}", "{{{"] + harness = _harness() + try: + for text in texts: + segments = [hm.tag_seg("{{ v | raw }}"), hm.text_seg(text), hm.tag_seg("{{ v | raw }}"), + hm.text_seg(text)] + harness.define("t", hm.emit(segments)) + ok, out, _ = harness.render("t", json.dumps({"v": "V"}).encode()) + assert ok and out.decode("utf-8") == "V" + text + "V" + text, (text, out) + finally: + harness.close() + + +def test_every_page_renders_back_byte_for_byte(): + topics = make_topics() + d = hm.derive(topics) + harness = _harness() + try: + harness.define(hm.PAGE_TEMPLATE, d.template) + for t in topics: + ok, out, _ = harness.render(hm.PAGE_TEMPLATE, hm.context_json(d.contexts[t.name])) + assert ok, out + assert out == t.page.encode("utf-8"), t.name + finally: + harness.close() + + +def test_jstopic_template_rebuilds_each_topics_fields(): + topics = make_topics() + d = hm.derive(topics) + harness = _harness() + try: + harness.define(hm.JSTOPIC_TEMPLATE, d.jstopic_template) + for t in topics: + ok, out, _ = harness.render(hm.JSTOPIC_TEMPLATE, hm.context_json(d.contexts[t.name])) + assert ok, out + fields, frame = hm.parse_jstopic(out.decode("utf-8")) + assert fields == t.fields, t.name + assert frame == t.frame + finally: + harness.close() + + +# --- runs against a database ----------------------------------------------------- + +def test_run_stores_json_and_points_rows_at_the_template(tmp_path): + db = make_db(tmp_path / "doc.db") + before = rows(db) + code, out, err = run_tool(db) + assert code == 0, err + after = rows(db) + conn = sqlite3.connect(db) + template_id, template = conn.execute("SELECT id, content FROM Templates WHERE name = ?", + (hm.PAGE_TEMPLATE,)).fetchone() + assert conn.execute("SELECT content FROM Templates WHERE name = 'page.peb'").fetchone()[0] == b"

{{ title }}

" + assert conn.execute("SELECT 1 FROM Templates WHERE name = ?", (hm.JSTOPIC_TEMPLATE,)).fetchone() is None + conn.close() + assert template.decode("utf-8").startswith("{#") + for t in TOPICS: + path = f"i/{t[0]}.html" + content, language, type_id, tid = after[path] + assert (language, type_id, tid) == (before[path][1], before[path][2], template_id) + context = json.loads(_brotli(content, decompress=True)) + assert context["file"] == f"{t[0]}.html" + assert after[f"i/jstopics/{t[0]}.js"] == before[f"i/jstopics/{t[0]}.js"] + for path in ("i/hmftsearch.html", "a/other.html"): + assert after[path] == before[path] + assert len(backups(db)) == 1 + assert "6 of 6 topic page(s) render back byte for byte" in out + + +def test_second_run_changes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db) + assert code == 0, err + digest = sha(db) + code, out, err = run_tool(db) + assert code == 0, err + assert "Nothing to change." in out + assert sha(db) == digest + assert len(backups(db)) == 1 + + +def test_dry_run_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + digest = sha(db) + code, out, err = run_tool(db, dry_run=True) + assert code == 0, err + assert "Would write the template (changed), 6 page(s) as JSON" in out + assert "Dry run: nothing written." in out + assert sha(db) == digest + assert backups(db) == [] + + +def test_page_that_does_not_fit_stays_html(tmp_path): + odd = ("\ufeff\r\n" + "".join(f"

other skin {n}

\r\n" for n in range(150)) + + "\t" + _body(3).replace("\n\r", "\r\n") + "\r\n\r\n").encode("utf-8") + db = make_db(tmp_path / "doc.db", {"i/t3.html": odd}) + before = rows(db) + code, out, err = run_tool(db) + assert code == 0, err + after = rows(db) + assert after["i/t3.html"] == before["i/t3.html"] + assert "stays HTML: i/t3.html - shares only" in err + assert "5 of 6 topic page(s) render back byte for byte" in out and "1 stay HTML" in out + assert after["i/t4.html"][3] != 0 + + +def test_page_whose_render_differs_stays_html(tmp_path): + db = make_db(tmp_path / "doc.db") + before = rows(db) + real = _harness() + + class Corrupting: + """The real renderer, except one page comes back one byte short.""" + + def define(self, name, text): + real.define(name, text) + + def render(self, name, context): + ok, out, nanos = real.render(name, context) + if ok and name == hm.PAGE_TEMPLATE and b'"file":"t2.html"' in context: + out = out[:-1] + return ok, out, nanos + + try: + code, out, err = run_tool(db, renderer=Corrupting()) + finally: + real.close() + assert code == 0, err + after = rows(db) + assert after["i/t2.html"] == before["i/t2.html"] + assert "stays HTML: i/t2.html - its rendered page differs from the stored one" in err + assert after["i/t1.html"][3] != 0 + + +def test_unreadable_row_is_named_and_left_alone(tmp_path): + db = make_db(tmp_path / "doc.db", {"i/t4.html": b"RAW:not brotli at all"}) + code, out, err = run_tool(db) + assert code == 1 + assert "skipped i/t4.html: could not be read" in err + assert rows(db)["i/t4.html"][0] == b"RAW:not brotli at all" + + +# --- phase 2: --drop-jstopics --------------------------------------------------- + +def jstopic_rows(db: Path) -> list: + return sorted(p for p in rows(db) if p.startswith("i/jstopics/")) + + +def test_drop_jstopics_stores_the_template_and_drops_every_row(tmp_path): + db = make_db(tmp_path / "doc.db") + before = rows(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + after = rows(db) + assert jstopic_rows(db) == [] + conn = sqlite3.connect(db) + stored = conn.execute("SELECT content FROM Templates WHERE name = ?", (hm.JSTOPIC_TEMPLATE,)).fetchone()[0] + conn.close() + assert stored.decode("utf-8") == hm.derive(make_topics()).jstopic_template + assert stored.decode("utf-8").startswith("hmLoadTopic({\r\nhmKeywords:\"{{ ") + for path in ("i/hmftsearch.html", "a/other.html"): + assert after[path] == before[path] + assert all(after[f"i/{t[0]}.html"][3] != 0 for t in TOPICS) + assert "Dropping 6 jstopics row(s)" in out and len(backups(db)) == 1 + + +def test_drop_jstopics_after_a_phase_1_run_leaves_the_pages_alone(tmp_path): + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db) + assert code == 0, err + pages = {p: v for p, v in rows(db).items() if not p.startswith("i/jstopics/")} + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + assert jstopic_rows(db) == [] + assert {p: v for p, v in rows(db).items()} == pages + assert "Writing the template" not in out + + +def test_drop_jstopics_second_run_changes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + digest = sha(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + assert "No jstopics rows left to drop." in out and "Nothing to change." in out + assert sha(db) == digest and len(backups(db)) == 1 + + +def test_drop_jstopics_dry_run_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + digest = sha(db) + code, out, err = run_tool(db, drop_jstopics=True, dry_run=True) + assert code == 0, err + assert "Would drop 6 jstopics row(s)" in out + assert sha(db) == digest and backups(db) == [] + + +def test_later_run_checks_what_the_stored_jstopic_template_builds(tmp_path): + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + assert "6 of 6 topic(s) without a jstopics row get one built from their JSON" in out + conn = sqlite3.connect(db) + text = conn.execute("SELECT content FROM Templates WHERE name = ?", (hm.JSTOPIC_TEMPLATE,)).fetchone()[0] + conn.execute("UPDATE Templates SET content = ? WHERE name = ?", + (text.replace(b'hmTitle:"{{ hmTitle', b'hmTitle:"x{{ hmTitle'), hm.JSTOPIC_TEMPLATE)) + conn.commit() + conn.close() + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1 + assert "broken: i/jstopics/t0.js - the jstopics file built from its JSON" in err + + +def test_jstopic_the_template_does_not_rebuild_keeps_its_row(tmp_path): + db = make_db(tmp_path / "doc.db") + before = rows(db) + real = _harness() + + class Corrupting: + """The real renderer, except t2's jstopics file comes back with another title.""" + + def define(self, name, text): + real.define(name, text) + + def render(self, name, context): + ok, out, nanos = real.render(name, context) + if ok and name == hm.JSTOPIC_TEMPLATE and b'"file":"t2.html"' in context: + out = out.replace(b'hmTitle:"Debug your app"', b'hmTitle:"Debug your ap"') + return ok, out, nanos + + try: + code, out, err = run_tool(db, drop_jstopics=True, renderer=Corrupting()) + finally: + real.close() + assert code == 1 + assert jstopic_rows(db) == ["i/jstopics/t2.js"] + assert rows(db)["i/jstopics/t2.js"] == before["i/jstopics/t2.js"] + assert "keeps i/jstopics/t2.js - the built file differs in hmTitle" in err + assert "1 jstopics row(s) left in place" in err + + +def test_jstopic_with_other_text_around_its_fields_keeps_its_row(tmp_path): + page, jstopic = make_topic(3) + odd = jstopic.replace(b"hmLoadTopic({", b"hmLoadTopic( {") + db = make_db(tmp_path / "doc.db", {"i/jstopics/t3.js": odd}) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1 + assert jstopic_rows(db) == ["i/jstopics/t3.js"] + assert "keeps i/jstopics/t3.js - the built file differs outside its field values" in err + + +def test_topic_whose_page_stays_html_keeps_its_jstopics_row(tmp_path): + odd = ("\ufeff\r\n" + "".join(f"

other skin {n}

\r\n" for n in range(150)) + + "\t" + _body(3).replace("\n\r", "\r\n") + "\r\n\r\n").encode("utf-8") + db = make_db(tmp_path / "doc.db", {"i/t3.html": odd}) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1 + assert jstopic_rows(db) == ["i/jstopics/t3.js"] + assert "keeps i/jstopics/t3.js - its page stays HTML" in err + + +def test_drop_jstopics_deletes_a_chunked_row_whole(tmp_path): + """A jstopics row stored in chunks goes with all of them. CHUNK_SIZE is + made tiny for the test so a small row can be stored the chunked way.""" + import optimize_db_media + saved = optimize_db_media.CHUNK_SIZE + optimize_db_media.CHUNK_SIZE = 40 + try: + db = make_db(tmp_path / "doc.db") + conn = sqlite3.connect(db) + blob = conn.execute("SELECT content FROM Content WHERE path = 'i/jstopics/t5.js'").fetchone()[0] + assert len(blob) > 80 and len(blob) % 40 + conn.execute("UPDATE Content SET content = ? WHERE path = 'i/jstopics/t5.js'", (blob[:40],)) + for n, start in enumerate(range(40, len(blob), 40), start=1): + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 6)", + (f"i/jstopics/t5.js-{n}", blob[start:start + 40])) + conn.commit() + conn.close() + assert len(jstopic_rows(db)) > 7 + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + assert jstopic_rows(db) == [] + assert "left alone" not in err + finally: + optimize_db_media.CHUNK_SIZE = saved + + +# --- review of PR #43 ------------------------------------------------------------- + +def test_topic_page_not_named_html_keeps_its_jstopics_row(tmp_path): + """R43-01: the app builds jstopics/X.js only from X.html, so X.htm is not a topic page.""" + page, _jstopic = make_topic(5) + db = make_db(tmp_path / "doc.db", {"i/t5.html": None, "i/t5.htm": page}) + before = rows(db) + code, out, err = run_tool(db, drop_jstopics=True) + after = rows(db) + assert code == 1 + assert after["i/t5.htm"] == before["i/t5.htm"] + assert jstopic_rows(db) == ["i/jstopics/t5.js"] and after["i/jstopics/t5.js"] == before["i/jstopics/t5.js"] + assert "left alone: i/jstopics/t5.js" in err + assert "Found 5 Help+Manual topic page(s)" in out + + +def _json_row(db: Path, path: str) -> dict: + conn = sqlite3.connect(db) + try: + return json.loads(_brotli(conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone()[0], + decompress=True)) + finally: + conn.close() + + +def _set_json_row(db: Path, path: str, context: dict) -> None: + conn = sqlite3.connect(db) + conn.execute("UPDATE Content SET content = ? WHERE path = ?", (_brotli(hm.context_json(context)), path)) + conn.commit() + conn.close() + + +def test_page_back_to_html_gets_its_dropped_jstopics_row_back(tmp_path): + """R43-02: a page that no longer fits the template goes back to HTML; its + jstopics row, dropped earlier, is built again from the stored template.""" + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + context = _json_row(db, "i/t2.html") + context["hmTitle"] = "Debug\r\n" * 200 + "your app" # pushes its page far off the others' shell + _set_json_row(db, "i/t2.html", context) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1 + assert "writes back i/jstopics/t2.js" in err + after = rows(db) + assert after["i/t2.html"][3] == 0 + assert _brotli(after["i/t2.html"][0], decompress=True).count(b"Debug\r\n") > 200 + content, _lang, type_id, tid = after["i/jstopics/t2.js"] + assert (type_id, tid) == (6, 0) + fields, _frame = hm.parse_jstopic(_brotli(content, decompress=True).decode("utf-8")) + assert fields == hm.jstopic_fields_from_context(context, hm.context_fields(context), hm.BODY_FORMS[0]) + assert jstopic_rows(db) == ["i/jstopics/t2.js"] + + +def test_page_back_to_html_with_no_way_to_rebuild_its_row_writes_nothing(tmp_path): + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + context = _json_row(db, "i/t2.html") + context["hmTitle"] = "Debug\r\n" * 200 + "your app" + _set_json_row(db, "i/t2.html", context) + conn = sqlite3.connect(db) + conn.execute("DELETE FROM Templates WHERE name = ?", (hm.JSTOPIC_TEMPLATE,)) + conn.commit() + conn.close() + digest = sha(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1 + assert "refused: i/t2.html - its page goes back to HTML, its jstopics row is gone" in err + assert sha(db) == digest + + +def test_export_whose_bodies_keep_their_crlf_runs_twice_cleanly(tmp_path): + """R43-05: bodies written with the same breaks in page and jstopics take the + form with no filter, and a later run reads that form from the stored template.""" + db = make_db(tmp_path / "doc.db", breaks="\r\n") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + conn = sqlite3.connect(db) + stored = conn.execute("SELECT content FROM Templates WHERE name = ?", (hm.JSTOPIC_TEMPLATE,)).fetchone()[0] + conn.close() + assert b"replace(" not in stored + assert "js_hmBody" not in _json_row(db, "i/t1.html") + digest = sha(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + assert "6 of 6 topic(s) without a jstopics row get one built" in out and "Nothing to change." in out + assert sha(db) == digest + + +def test_page_back_to_html_whose_row_the_stored_template_builds_wrong_writes_nothing(tmp_path): + """R43-12: a row the stored template renders, but with other fields - here + it reads a js_ key the JSON lacks - is not written back.""" + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + context = _json_row(db, "i/t2.html") + context["hmTitle"] = "Debug\r\n" * 200 + "your app" + _set_json_row(db, "i/t2.html", context) + conn = sqlite3.connect(db) + text = conn.execute("SELECT content FROM Templates WHERE name = ?", (hm.JSTOPIC_TEMPLATE,)).fetchone()[0] + assert text.count(b'hmTitle:"{{ hmTitle ') == 1 + conn.execute("UPDATE Templates SET content = ? WHERE name = ?", + (text.replace(b'hmTitle:"{{ hmTitle ', b'hmTitle:"{{ js_hmTitle '), hm.JSTOPIC_TEMPLATE)) + conn.commit() + conn.close() + digest = sha(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1 + assert "refused: i/t2.html - its page goes back to HTML and its jstopics row cannot be built again" in err + assert sha(db) == digest + + +def test_page_back_to_html_with_no_content_type_for_its_row_fails_the_run(tmp_path): + """R43-12: the refusal alone makes the run exit 1, with the jstopics + template stored and nothing else wrong.""" + db = make_db(tmp_path / "doc.db") + code, _out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + context = _json_row(db, "i/t2.html") + context["hmTitle"] = "Debug\r\n" * 200 + "your app" + _set_json_row(db, "i/t2.html", context) + conn = sqlite3.connect(db) + conn.execute("DELETE FROM ContentTypes WHERE value = ?", (hm.JSTOPIC_CONTENT_TYPE,)) + conn.commit() + conn.close() + digest = sha(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1, err + assert ("refused: i/t2.html - its page goes back to HTML, its jstopics row is gone, and there is no " + "text/javascript to write it back with") in err + assert "broken:" not in err and "left in place" not in err + assert sha(db) == digest + + +def test_jstopics_rows_whose_prefix_matches_only_without_case_are_not_dropped_or_counted(tmp_path): + """R43-10, R43-12: with --drop-jstopics too, I/jstopics/ is not i/jstopics/.""" + page1, jstopic1 = make_topic(1) + db = make_db(tmp_path / "doc.db", {"I/t1.html": page1, "I/jstopics/t1.js": jstopic1}) + before = rows(db) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 0, err + after = rows(db) + assert after["I/t1.html"] == before["I/t1.html"] and after["I/jstopics/t1.js"] == before["I/jstopics/t1.js"] + assert jstopic_rows(db) == [] and "left alone" not in err + + +def test_topic_found_unfit_before_it_is_read_keeps_its_row_and_fails_the_drop(tmp_path): + """R43-11: a jstopics file with no hmBody, or a page that is not UTF-8, + kept the topic's row with exit 0, though --drop-jstopics means every row + should go. A page that cannot be read keeps its row too.""" + page4, _jstopic4 = make_topic(4) + for name, pages, stem, why in ( + ("no-body", {"i/jstopics/t3.js": b'hmLoadTopic({ hmTitle:"Third" });'}, "t3", + "its page stays HTML (its jstopics file has no hmBody field)"), + ("not-utf8", {"i/t4.html": page4.replace(b"Make a project.", b"Make a \xff project.")}, "t4", + "its page stays HTML (the page is not valid UTF-8"), + ("unreadable", {"i/t5.html": b"RAW:not brotli at all"}, "t5", + "its page or its jstopics file could not be read")): + (tmp_path / name).mkdir() + db = make_db(tmp_path / name / "doc.db", pages) + code, out, err = run_tool(db, drop_jstopics=True) + assert code == 1, (name, err) + assert jstopic_rows(db) == [f"i/jstopics/{stem}.js"], name + assert f"keeps i/jstopics/{stem}.js - {why}" in err, err + assert "1 jstopics row(s) left in place" in err, err + + +def test_pebble_empty_agrees_with_pebbles_is_empty(): + values = ["", " ", "\t\r\n", "\u00a0", "a", " a ", "\x0b"] + harness = _harness() + try: + harness.define("t", "{% if x is empty %}E{% else %}N{% endif %}") + for value in values: + ok, out, _ = harness.render("t", json.dumps({"x": value}).encode()) + assert ok and (out == b"E") == hm.pebble_empty(value), repr(value) + finally: + harness.close() + + +def test_rows_the_step_does_not_own_are_left_alone(tmp_path): + """A page under another template, a page in a subfolder, and pages whose + prefix matches only without case: none is a topic page.""" + page1, jstopic1 = make_topic(1) + extra = {"i/t9.html": page1, "i/jstopics/t9.js": jstopic1, + "i/sub/t1.html": page1, "i/jstopics/sub/t1.js": jstopic1, + "I/t1.html": page1, "I/jstopics/t1.js": jstopic1} + db = make_db(tmp_path / "doc.db", extra, template_ids={"i/t9.html": 1}) + before = rows(db) + code, out, err = run_tool(db) + assert code == 0, err + after = rows(db) + for path in extra: + assert after[path] == before[path], path + assert "skipped i/t9.html: rendered through another template (templateId 1)" in err + assert "Found 6 Help+Manual topic page(s)" in out + + +def test_chunked_topic_page_is_read_whole_and_rewritten(tmp_path): + import optimize_db_media + saved = optimize_db_media.CHUNK_SIZE + optimize_db_media.CHUNK_SIZE = 64 + try: + db = make_db(tmp_path / "doc.db") + conn = sqlite3.connect(db) + blob = conn.execute("SELECT content FROM Content WHERE path = 'i/t4.html'").fetchone()[0] + assert len(blob) % 64 + conn.execute("UPDATE Content SET content = ? WHERE path = 'i/t4.html'", (blob[:64],)) + for n, start in enumerate(range(64, len(blob), 64), start=1): + conn.execute("INSERT INTO Content (path, languageID, content, contentTypeID) VALUES (?, 1, ?, 12)", + (f"i/t4.html-{n}", blob[start:start + 64])) + conn.commit() + conn.close() + code, out, err = run_tool(db) + assert code == 0, err + after = rows(db) + assert not [p for p in after if p.startswith("i/t4.html-")] + assert after["i/t4.html"][3] != 0 and "6 of 6 topic page(s) render back" in out + finally: + optimize_db_media.CHUNK_SIZE = saved + + +def test_page_with_a_nul_stays_html(tmp_path): + page, _jstopic = make_topic(3) + db = make_db(tmp_path / "doc.db", {"i/t3.html": page.replace(b".rule7 {", b".rule7\x00 {")}) + before = rows(db) + code, out, err = run_tool(db) + assert code == 0, err + assert rows(db)["i/t3.html"] == before["i/t3.html"] + assert "stays HTML: i/t3.html - the page contains a NUL character" in err + + +def test_page_sharing_three_quarters_of_its_lines_stays_html(tmp_path): + page, _jstopic = make_topic(3) + text = page.decode("utf-8") + for n in range(25): + text = text.replace(f".rule{n} {{ margin: {n}px; }}", f".other{n} {{ padding: {n}px; }}") + db = make_db(tmp_path / "doc.db", {"i/t3.html": text.encode("utf-8")}) + before = rows(db) + code, out, err = run_tool(db) + assert code == 0, err + assert rows(db)["i/t3.html"] == before["i/t3.html"] + shared = re.search(r"stays HTML: i/t3.html - shares only (\d+)% of its lines", err) + assert shared and 60 < int(shared.group(1)) < 90, err + + +def test_classpath_uses_the_gson_the_app_renders_with(): + classpath = _classpath() + if os.environ.get("HM_RENDER_CLASSPATH"): + _skip("the classpath was given, not found") + assert "gson-2.12.1.jar" in classpath and "pebble-4.1.1.jar" in classpath + + +def _main() -> int: + """Runs the tests without pytest, so this file works the way the tools + beside it do (`uv run scripts/test_hm_pages_to_templates.py`).""" + failures, skipped = [], [] + for name, function in sorted(globals().items()): + if not name.startswith("test_") or not callable(function): + continue + with tempfile.TemporaryDirectory() as directory: + needs_tmp = "tmp_path" in function.__code__.co_varnames[:function.__code__.co_argcount] + try: + function(Path(directory)) if needs_tmp else function() + except Skip as exc: + skipped.append(name) + print(f"skip {name}: {exc}") + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: # noqa: BLE001 - a failure is the result here + if type(exc).__name__ == "Skipped": # pytest.skip, when pytest is installed + skipped.append(name) + print(f"skip {name}: {exc}") + continue + frame = traceback.extract_tb(exc.__traceback__)[-1] + failures.append(f"{name} (line {frame.lineno}: {frame.line}): {type(exc).__name__}: {exc}") + print(f"FAIL {name}", file=sys.stderr) + else: + print(f"ok {name}") + print() + print(f"{len(failures)} failure(s), {len(skipped)} skipped" if failures or skipped else "all tests passed") + for failure in failures: + print(f" {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(_main())