diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index ae38d727..e7aa311c 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -65,14 +65,16 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v5 - # scripts/test_migrate_tooltip_schema.py needs only the standard library and pytest. - - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema' + # scripts/test_migrate_tooltip_schema.py needs only the standard library, pytest and the brotli CLI (above). + - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + Android and Java loaders' run: | uv run --with-requirements requirements.txt --with pytest -- \ python -m pytest \ ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON \ scripts/sync_kotlin_stdlib_docs \ scripts/test_migrate_tooltip_schema.py \ + ProcessDocs/AndroidDocs/AndroidDocsToJson \ + scripts/sync_java_docs \ -q - name: docdb-studio diff --git a/CLAUDE.md b/CLAUDE.md index d3735e14..060c04d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,7 +61,7 @@ edit that database** — it contains no part of the production Android app itsel > Note also that `image/webp` (id 26) and `video/quicktime` (id 28) now exist in `ContentTypes`, so > the workflows' `INSERT OR IGNORE ... image/webp` step is a no-op against current copies. -> ### Schema 3.0.0: keyed tooltip tables, in the shipped copy only +> ### Schema 3.0.0: keyed tooltip tables and compressed templates, in the shipped copy only > > `scripts/migrate_tooltip_schema.py` (ADFA-3305) rewrites the tooltip tables and declares **3.0.0**: > `Tooltips(id, categoryId, tag)` WITHOUT ROWID, keyed on `(categoryId, tag)`; @@ -70,11 +70,31 @@ edit that database** — it contains no part of the production Android app itsel > string once in `TooltipButtonDescriptions` and `TooltipButtonUris`. Code On The Go from > CodeOnTheGo #2163 on reads only these tables, and refuses tooltips from a database declaring less. > +> The same step stores **every `Templates` row Brotli-compressed** against `CompressionDictionary`, +> with no per-row marker: in 3.0.0 all templates are compressed, and the app always decodes them. +> Every other tool that reads or writes Templates expects them plain: `populate_db.py`, +> `sync_javadoc_json_to_db.py`, `ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py` +> and `verify_android_json_db.py`, and in open PRs `minify_text.py` (#44), the `a/` +> pages-to-templates step (`android_pages_to_templates.py`, #49) and `hm_pages_to_templates.py` +> (#43). On the master the templates stay plain. Some of these run on the copy that ships as well +> (the minify and the `a/` step do), so there every one of them runs before the migration. The order +> on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, +> `migrate_tooltip_schema.py`, PR #48's `remint_dictionary.py --seeds 4` (which recompresses +> templates too in 3.0.0), then `rebuild_database.py --pack Content,TooltipTexts`; +> `hm_pages_to_templates.py` goes after the `i/` page steps it lists, before the migration like +> every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the +> re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0. +> +> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` write plain templates, +> so each refuses a database declaring 3.0.0 or later before writing anything, backup included. +> `load_android_json_db.py` keeps the major it finds, so a late run would otherwise leave a file +> declaring 3.1.0 with plain templates in it, which the app cannot read. +> > **Every tool in this repo still reads and writes the 2.x shape**: `check-tools` (whose daily > `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`), > `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`), `TooltipManager.py` and > `docdb-studio`. So the step runs on the copy that ships, after all of them, and never on the -> master database they edit; nothing converts 3.0 back. +> master database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH. > > In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the > id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)` @@ -144,8 +164,9 @@ removes it. Measurements are in its docstring. **This repo writes `Templates`, but not `Bookshelf` or `BookCategories`.** `populate_db.py` (`page.peb`, `nav.peb`), `sync_javadoc_json_to_db.py` (`javadoc.peb`) and `load_android_json_db.py` (the `android-*.peb` templates) write rows there and point the pages they write at them via -`templateId`; `verify_android_json_db.py` reads them; and `remint_dictionary.py` and -`verify_remint_dictionary.py` recompress and compare them from 3.0.0 on (above). +`templateId`; `verify_android_json_db.py` reads them; `migrate_tooltip_schema.py` compresses +every one for 3.0.0; and `remint_dictionary.py` and `verify_remint_dictionary.py` recompress and +compare them from 3.0.0 on (above). `insert_optimized_media.py` reads and writes `templateId` too. `Bookshelf` and `BookCategories` come from App Dev For All's plugin system, which writes into the documentation database - the bookshelf feature from @@ -206,10 +227,11 @@ shape: every tool here reads and writes that, and only the shipped copy holds th ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), `create_empty_database.py`, `list_database_documents.py`, `migrate_tooltip_schema.py` (ADFA-3305: - converts the tooltip tables to the WITHOUT ROWID, lookup-table shape Code On The Go reads from - `DocumentationDatabaseVersion` 3.0.0 on; run it on the copy that ships, after every step that writes - tooltips, since they all write the old shape, never on the master they edit, and pack - `Content,TooltipTexts` afterwards, not `Tooltips`), `minify_text.py` (ADFA-6627: minifies the + converts to documentation schema 3.0.0, which Code On The Go reads from 3.0.0 on: the WITHOUT ROWID, + lookup-table tooltip shape, and every template Brotli-compressed against the dictionary; run it on the + copy that ships, after every step that writes tooltips or templates, since they all write the old + shape and plain templates, never on the master they edit; re-mint afterwards, then pack + `Content,TooltipTexts`, not `Tooltips`), `minify_text.py` (ADFA-6627: minifies the HTML, CSS, JS and SVG rows and the HTML templates without changing what a page shows; run `ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py` and `verify_remint_dictionary.py` straight after it, or the minified rows compress worse than before), diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py index 8df0c047..11aec50a 100755 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py @@ -45,8 +45,8 @@ sys.path.insert(0, str(_KOTLIN_PIPELINE)) from populate_db import ( # noqa: E402 (the path has to be set up first) - DictionaryCompressor, fragment_chain, get_content_type, get_id, insert_chunked_content, - load_dictionary, upsert_template, + DictionaryCompressor, compressed_templates_refusal, fragment_chain, get_content_type, get_id, + insert_chunked_content, load_dictionary, upsert_template, ) from migrate_content_to_dictionary_brotli import write_item # noqa: E402 @@ -328,11 +328,15 @@ def main(argv: list[str] | None = None) -> int: source_conn = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True) try: missing = check_schema(source_conn) + # record_version keeps the major, so a 3.x source would come out 3.(x+1) with plain templates in it. + refusal = None if missing else compressed_templates_refusal(source_conn) finally: source_conn.close() if missing: parser.error(f"{args.source} is missing table(s) this loader writes: " + ", ".join(missing)) + if refusal: + parser.error(f"{args.source}: {refusal}; nothing was written") print(f"==> copying {args.source} -> {args.out}") shutil.copy2(args.source, args.out) diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py index b8905e0b..0af304b0 100644 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py @@ -260,3 +260,37 @@ def test_the_comment_carries_the_counts(self, db): comment = db.execute("SELECT comment FROM DocumentationDatabaseVersion").fetchone()[0] assert "12,106 rows replaced" in comment and "800 added" in comment + + +class TestCompressedTemplates: + """From documentation 3.0.0 every template is stored compressed and the app decodes every one. record_version + keeps the source's major, so loading a 3.x source would leave 3.(x+1) with plain Android templates in it.""" + + def run(self, db, tmp_path, *versions): + db.executescript(VERSION_TABLES) + db.execute("INSERT INTO ContentTypes (value, compression) VALUES ('text/css', 'brotli')") + db.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (?, 0, 0, 'test', 'test', ?)", versions) + db.commit() + (tmp_path / "json").mkdir() + return loader.main([str(tmp_path / "json"), "--source", str(tmp_path / "test.db"), + "--out", str(tmp_path / "out.db"), "--workers", "2"]) + + @pytest.mark.parametrize("major", [3, 4]) + def test_a_source_declaring_3_or_later_is_refused_before_anything_is_written(self, db, tmp_path, capsys, major): + with pytest.raises(SystemExit) as exited: + self.run(db, tmp_path, (2, "2026-01-01 00:00:00"), (major, "2026-10-19 00:00:00")) + assert exited.value.code == 2 + assert f"declares documentation version {major}" in capsys.readouterr().err + assert not (tmp_path / "out.db").exists() + + def test_a_source_declaring_2_still_loads(self, db, tmp_path): + # The newest row by changeTime is 2, though a 3 was inserted after it. + assert self.run(db, tmp_path, (2, "2026-10-19 00:00:00"), (3, "2026-01-01 00:00:00")) == 0 + out = sqlite3.connect(tmp_path / "out.db") + try: + assert out.execute("SELECT major, minor FROM DocumentationDatabaseVersion " + "ORDER BY rowid DESC LIMIT 1").fetchone() == (2, 1) + assert out.execute("SELECT COUNT(*) FROM Templates").fetchone() == (3,) + finally: + out.close() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py index fbbe5233..e7e58921 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py @@ -569,7 +569,37 @@ def insert_file(conn, data: bytes, name: str, db_path: str, language_id: int, co return True +# From this documentation major on, migrate_tooltip_schema.py (ADFA-3305) has stored every template compressed +# against the dictionary, and the app decodes every one: a plain template written into such a database would not +# render, and the file would still declare 3.x. +COMPRESSED_TEMPLATES_MAJOR = 3 + + +def declared_major(conn) -> int | None: + """The documentation major the database declares - its newest DocumentationDatabaseVersion row, ordered as + Code On The Go orders them (changeTime, then rowid) - or None if it declares none.""" + if not _table_exists(conn, "DocumentationDatabaseVersion"): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion " + "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() + return None if row is None else int(row[0]) + + +def compressed_templates_refusal(conn) -> str | None: + """Why no plain template may be written into this database, or None.""" + major = declared_major(conn) + if major is None or major < COMPRESSED_TEMPLATES_MAJOR: + return None + return (f"the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 every " + "template is stored compressed, and this step writes plain ones. Run it before migrate_tooltip_schema.py") + + def upsert_template(conn, name: str, content: str) -> int: + """Writes one plain template. Raises RuntimeError, writing nothing, if the database's templates are stored + compressed.""" + refusal = compressed_templates_refusal(conn) + if refusal: + raise RuntimeError(refusal) conn.execute( "INSERT INTO Templates (name, content) VALUES (?, ?) " "ON CONFLICT(name) DO UPDATE SET content = excluded.content", @@ -718,6 +748,15 @@ def main(): if not args.db_path.is_file(): print(f"error: {args.db_path} does not exist", file=sys.stderr) sys.exit(1) + # Checked before the conversion and the backup, so a refusal costs nothing and writes nothing. + conn = sqlite3.connect(f"file:{args.db_path}?mode=ro", uri=True) + try: + refusal = compressed_templates_refusal(conn) + finally: + conn.close() + if refusal: + print(f"error: {refusal}; nothing was changed", file=sys.stderr) + sys.exit(1) config = load_config(args.config) variables = load_variables(docs_root) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py new file mode 100644 index 00000000..8ee4ec21 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Tests that populate_db.py writes no plain template into a database whose templates are compressed (ADFA-3305). + +From documentation 3.0.0 migrate_tooltip_schema.py stores every template compressed, and the app decodes every one, +so a plain page.peb or nav.peb written afterwards would not render while the file still declared 3.x. + +Run directly: python3 test_populate_db_templates.py +""" +import hashlib +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +from populate_db import compressed_templates_refusal, declared_major, upsert_template + +HERE = Path(__file__).resolve().parent + +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE(path)); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE(name)); +CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL); +CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL, + who TEXT NOT NULL, comment TEXT NOT NULL, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP); +INSERT INTO Languages (value) VALUES ('en-US'); +INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli'), ('text/css', 'brotli'), + ('text/javascript', 'brotli'); +INSERT INTO CompressionDictionary (id, data) VALUES (1, CAST('
{{ probe | raw }}
" + + def __init__(self, dictionary: bytes): + self._brotli = shutil.which("brotli") + if self._brotli is None: + raise Refused("the brotli CLI is not on PATH; 3.0.0 stores every template compressed with it") + self._dir = tempfile.TemporaryDirectory(prefix="migrate-schema-dictionary-") + self._dictionary = Path(self._dir.name) / "dictionary.bin" + self._dictionary.write_bytes(dictionary) + self.dictionary = dictionary + stream, error = self._run("-w", str(WINDOW_BITS), data=self.PROBE) + if stream is not None: + text, error = self._run("-d", data=stream) + if text == self.PROBE: + return + error = error or "it decoded to other bytes" + self.close() + raise Refused(f"the brotli CLI ({self._brotli}) could not compress and decode a test text against the " + f"dictionary: {error}") + + def _run(self, *args: str, data: bytes) -> tuple: + """(output, None) if the CLI succeeds, else (None, what it printed on stderr).""" + result = subprocess.run([self._brotli, "-D", str(self._dictionary), *args, "-c"], input=data, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if result.returncode == 0: + return result.stdout, None + return None, result.stderr.decode("utf-8", "replace").strip() or f"exit status {result.returncode}" + + def compress(self, data: bytes) -> bytes: + out, error = self._run("-w", str(WINDOW_BITS), data=data) + if out is None: + raise RuntimeError(f"the brotli CLI could not compress a template: {error}") + return out + + def decompress(self, data: bytes): + """The plaintext, or None if `data` is not a Brotli stream against this dictionary. A large-window stream + decodes here, though not in the app: see large_window.""" + return self._run("-d", data=data)[0] + + def close(self) -> None: + self._dir.cleanup() + + +def load_codec(conn) -> DictionaryCodec: + """A codec for the database's CompressionDictionary; raises Refused if it has none.""" + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'CompressionDictionary'" + ).fetchone(): + raise Refused("no CompressionDictionary table; 3.0.0 stores every template compressed against it") + row = conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone() + if row is None or not row[0]: + raise Refused("CompressionDictionary holds no dictionary; 3.0.0 stores every template compressed against it") + return DictionaryCodec(bytes(row[0])) + + +def stored_templates(conn) -> dict: + """Every Templates row as {id: (name, stored bytes)}; empty if there is no Templates table.""" + if not conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'Templates'").fetchone(): + return {} + out = {} + for template_id, name, content in conn.execute("SELECT id, name, content FROM Templates ORDER BY id"): + out[template_id] = (name, None if content is None + else content.encode("utf-8") if isinstance(content, str) else bytes(content)) + return out + + +def _parallel(function, items: list) -> list: + with ThreadPoolExecutor(max_workers=min(32, os.cpu_count() or 4)) as pool: + return list(pool.map(function, items)) + + +def plain_templates(conn, codec: DictionaryCodec) -> tuple: + """({id: (name, plaintext)}, problems) for a database whose templates are still plain text, as every + tool that writes Templates writes them. A template that already decodes against the dictionary is taken + for one compressed already - every one, or a mix that means some other step ran - and compressing it + again would hide its text, so it is a problem; so is one that is neither UTF-8 text nor such a stream, + or NULL. A very short text can also be a valid stream (see the module docstring); one is refused here as + compressed already rather than guessed at.""" + stored = stored_templates(conn) + decoded = _parallel(lambda item: None if item[1][1] is None else codec.decompress(item[1][1]), + list(stored.items())) + plain, compressed, unreadable = {}, [], [] + for (template_id, (name, data)), as_brotli in zip(stored.items(), decoded): + if data is None: + unreadable.append(name) + elif as_brotli is not None: + compressed.append(name) + else: + try: + data.decode("utf-8") + except UnicodeDecodeError: + unreadable.append(name) + continue + plain[template_id] = (name, data) + found = [] + if compressed: + found.append(f"{human(len(compressed))} of {human(len(stored))} template(s) already decode against the " + f"dictionary, so they look compressed already ({', '.join(compressed[:5])}" + f"{', ...' if len(compressed) > 5 else ''}); this step compresses plain templates only") + if unreadable: + found.append(f"{human(len(unreadable))} template(s) are neither UTF-8 text nor compressed against the " + f"dictionary ({', '.join(unreadable[:5])}{', ...' if len(unreadable) > 5 else ''})") + return plain, found + + +def compress_templates(conn, codec: DictionaryCodec, plain: dict) -> int: + """Stores every template compressed, inside the caller's transaction; returns the stored bytes. Each one + is proved to decode back to its exact text, and to be a stream the app can decode, before it is written.""" + def one(item): + template_id, (name, data) = item + blob = codec.compress(data) + if large_window(blob): + raise RuntimeError(f"template {name} was compressed to a large-window stream, which the app cannot decode") + if codec.decompress(blob) != data: + raise RuntimeError(f"template {name} does not decode back to its text after compression") + return template_id, blob + total = 0 + for template_id, blob in _parallel(one, list(plain.items())): + conn.execute("UPDATE Templates SET content = ? WHERE id = ?", (blob, template_id)) + total += len(blob) + return total + + +def templates_read_back(conn, codec: DictionaryCodec, plain: dict) -> list: + """Differences between the plain templates and what Templates now holds, decoded the way the app decodes + them; empty if none.""" + differences = [] + stored = stored_templates(conn) + if {i: n for i, (n, _) in stored.items()} != {i: n for i, (n, _) in plain.items()}: + differences.append("the Templates rows' ids or names changed") + decoded = _parallel(lambda item: codec.decompress(item[1][1]) if item[1][1] is not None else None, + list(stored.items())) + for (template_id, (name, data)), text in zip(stored.items(), decoded): + if template_id in plain and text != plain[template_id][1]: + differences.append(f"template {name} does not decode to its text") + elif large_window(data): + differences.append(f"template {name} is a large-window stream, which the app cannot decode") + return differences + + def backup_database(db_path: Path) -> Path: """VACUUM INTO a timestamped sibling, the next free suffix if that exists. The same as optimize_db_media.backup_database, which this step does not import - because that module needs Pillow and brotli and this one touches neither.""" + because that module needs Pillow and the brotli package, and this one needs + neither.""" stamp = time.strftime("%Y%m%d-%H%M%S") backup_path = db_path.with_name(f"{db_path.name}.backup-{stamp}") attempt = 1 @@ -377,7 +588,8 @@ def read_back(conn, before: dict) -> list: def convert(conn) -> list: - """Rewrites the tooltip tables inside the caller's transaction; returns what it dropped besides them.""" + """Rewrites the tooltip tables inside the caller's transaction; returns what it dropped besides them. The + version row is declare_version's, once the templates are compressed too.""" dropped = [name for (name,) in conn.execute( "SELECT name FROM sqlite_master WHERE type = 'index' AND name = ? COLLATE NOCASE " "AND tbl_name = 'TooltipButtons' COLLATE NOCASE", (REPLACED_INDEX,))] @@ -392,10 +604,13 @@ def convert(conn) -> list: # Dropping Tooltips_old also drops its sqlite_sequence row: nothing hands out its ids any more. conn.execute("DROP TABLE TooltipButtons_old") conn.execute("DROP TABLE Tooltips_old") + return dropped + + +def declare_version(conn) -> None: conn.execute("DELETE FROM DocumentationDatabaseVersion") conn.execute("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment) VALUES (?, 0, 0, ?, ?)", (TO_MAJOR, WHO, COMMENT)) - return dropped def run(cfg: dict) -> int: @@ -405,6 +620,7 @@ def run(cfg: dict) -> int: return 1 conn = sqlite3.connect(db_path, isolation_level=None) + codec = None try: try: if schema_state(conn) == "new": @@ -414,9 +630,28 @@ def run(cfg: dict) -> int: f"declares major {major}, so the app refuses this file; a step run after the " f"conversion rewrote the version row. Declare {TO_MAJOR}.0.0 by hand once you know " "it changed nothing else") - print(f"{db_path}: the tooltip tables are already in the new shape; nothing to do.") + codec = load_codec(conn) + stored = stored_templates(conn) + decoded = _parallel(lambda item: None if item[1] is None else codec.decompress(item[1]), + list(stored.values())) + plain = [name for (name, data), text in zip(stored.values(), decoded) + if text is None or large_window(data)] + if plain: + raise Refused(f"the tooltip tables are already in the new shape, but {human(len(plain))} of " + f"{human(len(stored))} template(s) do not decode against the dictionary as the app " + f"decodes them ({', '.join(plain[:5])}{', ...' if len(plain) > 5 else ''}), and " + f"{TO_MAJOR}.0.0 stores every template compressed; a step run after the conversion " + "wrote them") + print(f"{db_path}: the tooltip tables are already in the new shape and every template is " + "compressed; nothing to do.") return 0 found = problems(conn) + try: + codec = load_codec(conn) + plain, template_problems = plain_templates(conn, codec) + found += template_problems + except Refused as exc: + found.append(str(exc)) except Refused as exc: print(f"error: {exc}; nothing was changed", file=sys.stderr) return 1 @@ -429,8 +664,10 @@ def run(cfg: dict) -> int: counts = {name: conn.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0] for name in ("Tooltips", "TooltipButtons")} distinct = conn.execute("SELECT COUNT(DISTINCT description), COUNT(DISTINCT uri) FROM TooltipButtons").fetchone() + text_bytes = sum(len(data) for _name, data in plain.values()) print(f"Converting {human(counts['Tooltips'])} tooltip(s) and {human(counts['TooltipButtons'])} button(s); " - f"{human(distinct[0])} distinct description(s), {human(distinct[1])} distinct URI(s).") + f"{human(distinct[0])} distinct description(s), {human(distinct[1])} distinct URI(s); " + f"compressing {human(len(plain))} template(s), {human(text_bytes)} bytes of text.") if not cfg["dry_run"]: print(f"Backing up {db_path} ...") @@ -439,8 +676,21 @@ def run(cfg: dict) -> int: conn.execute("BEGIN IMMEDIATE") try: before = snapshot(conn) + # The templates checked above are the ones compressed: none may have changed since, nor the dictionary + # the codec read before this transaction, which they are compressed against. + if {i: d for i, (_n, d) in stored_templates(conn).items()} != {i: d for i, (_n, d) in plain.items()}: + raise Refused("the templates changed while this step ran") + row = conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone() + if row is None or row[0] != codec.dictionary: + raise Refused("the dictionary changed while this step ran") dropped = convert(conn) - differences = read_back(conn, before) + stored_bytes = compress_templates(conn, codec, plain) + declare_version(conn) + differences = read_back(conn, before) + templates_read_back(conn, codec, plain) + except Refused as exc: + conn.execute("ROLLBACK") + print(f"error: {exc}; rolled back, nothing was changed", file=sys.stderr) + return 1 except BaseException: conn.execute("ROLLBACK") raise @@ -448,10 +698,12 @@ def run(cfg: dict) -> int: conn.execute("ROLLBACK") for difference in differences[:20]: print(f"error: {difference}", file=sys.stderr) - print(f"error: {human(len(differences))} difference(s) reading the tooltips back; rolled back, nothing " - "was changed", file=sys.stderr) + print(f"error: {human(len(differences))} difference(s) reading the tooltips or templates back; rolled " + "back, nothing was changed", file=sys.stderr) return 1 print(f"Every tooltip and its buttons read back identically ({human(len(before['shown']))} tooltip(s)).") + print(f"Every template decodes back to its text ({human(len(plain))} template(s), {human(text_bytes)} -> " + f"{human(stored_bytes)} bytes).") for name in dropped: print(f" dropped index {name}: TooltipButtons' new primary key is the same index") if cfg["dry_run"]: @@ -459,10 +711,12 @@ def run(cfg: dict) -> int: print("Dry run: nothing written.") return 0 conn.execute("COMMIT") - except sqlite3.Error as exc: + except (sqlite3.Error, RuntimeError) as exc: print(f"error: {exc}; nothing was changed", file=sys.stderr) return 1 finally: + if codec is not None: + codec.close() conn.close() print("Vacuuming database...") @@ -473,7 +727,9 @@ def run(cfg: dict) -> int: vac.close() print(f"\nDone. DocumentationDatabaseVersion {FROM_MAJOR}.x -> {TO_MAJOR}.0.0; " f"{human(size_before)} -> {human(db_path.stat().st_size)} bytes.") - print("Next: rebuild_database.py with --pack Content,TooltipTexts; it cannot pack the WITHOUT ROWID Tooltips.") + print("Next: re-mint the dictionary with OfflineDocumentationTools PR #48's remint_dictionary.py --seeds 4, " + f"which recompresses the templates too in {TO_MAJOR}.0.0 (one without it refuses this file), then " + "rebuild_database.py with --pack Content,TooltipTexts; it cannot pack the WITHOUT ROWID Tooltips.") return 0 @@ -481,7 +737,8 @@ def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("db_path", type=Path, help="SQLite database to convert in place, e.g. documentation.db") p.add_argument("--dry-run", action="store_true", - help="Convert and check inside a transaction, then roll it back: nothing written or backed up") + help="Convert, compress and check inside a transaction, then roll it back: nothing written or " + "backed up") return p diff --git a/scripts/sync_java_docs/sync_javadoc_json_to_db.py b/scripts/sync_java_docs/sync_javadoc_json_to_db.py index 932608f3..17251fd3 100755 --- a/scripts/sync_java_docs/sync_javadoc_json_to_db.py +++ b/scripts/sync_java_docs/sync_javadoc_json_to_db.py @@ -29,7 +29,9 @@ - Member-level `signature` and `url` are dropped, because no template reads them -- the detail macros compose a member's signature from its parts and link it by anchor. See PRUNED_* below. -A timestamped backup is taken before anything is written. +A timestamped backup is taken before anything is written. A database declaring documentation version 3.0.0 or +later is refused before that: from 3.0.0 migrate_tooltip_schema.py stores every template compressed, and this +writes a plain one. """ from __future__ import annotations @@ -93,6 +95,21 @@ REFERENCE_KEYS = frozenset({"specifiedBy", "members", "overrides"}) +# From this documentation major on, every template is stored compressed (migrate_tooltip_schema.py, ADFA-3305). +COMPRESSED_TEMPLATES_MAJOR = 3 + + +def declared_major(conn) -> int | None: + """The documentation major the database declares - its newest DocumentationDatabaseVersion row, ordered as + Code On The Go orders them (changeTime, then rowid) - or None if it declares none.""" + if conn.execute("SELECT 1 FROM sqlite_master WHERE type = 'table' " + "AND name = 'DocumentationDatabaseVersion'").fetchone() is None: + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion " + "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() + return None if row is None else int(row[0]) + + def backup_database(db_path: str) -> str: stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") backup = f"{db_path}.bak.{stamp}" @@ -229,6 +246,17 @@ def main() -> int: file=sys.stderr) return 2 + check = sqlite3.connect(f"file:{args.db}?mode=ro", uri=True) + try: + major = declared_major(check) + finally: + check.close() + if major is not None and major >= COMPRESSED_TEMPLATES_MAJOR: + print(f"Error: the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 " + f"every template is stored compressed, and {TEMPLATE_NAME} would be written plain. Run this before " + "migrate_tooltip_schema.py; nothing was changed.", file=sys.stderr) + return 1 + if args.dry_run: print("Dry run: no backup will be made and nothing will be written.") else: diff --git a/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py b/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py new file mode 100644 index 00000000..4de86431 --- /dev/null +++ b/scripts/sync_java_docs/test_sync_javadoc_json_to_db.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Tests that sync_javadoc_json_to_db.py writes no plain template into a database whose templates are compressed. + +From documentation 3.0.0 migrate_tooltip_schema.py (ADFA-3305) stores every template compressed, and the app decodes +every one, so a plain javadoc.peb written afterwards would not render while the file still declared 3.x. + +Run directly (python3 scripts/sync_java_docs/test_sync_javadoc_json_to_db.py) or under pytest. Needs the brotli CLI. +""" +import hashlib +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "sync_javadoc_json_to_db.py" + +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE(path)); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE(name)); +CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT); +CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL, + who TEXT NOT NULL, comment TEXT NOT NULL, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP); +INSERT INTO Languages (value) VALUES ('en-US'); +INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli'); +""" + + +class CompressedTemplatesRefusalTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.dir = Path(directory.name) + self.db_path = self.dir / "documentation.db" + conn = sqlite3.connect(self.db_path) + conn.executescript(SCHEMA) + conn.close() + (self.dir / "api").mkdir() + (self.dir / "templates").mkdir() + (self.dir / "templates" / "javadoc.peb").write_text("{{ page }}", encoding="utf-8") + + def declare(self, *rows): + """Appends (major, changeTime) rows to DocumentationDatabaseVersion, in order.""" + conn = sqlite3.connect(self.db_path) + conn.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (?, 0, 0, 'test', 'test', ?)", rows) + conn.commit() + conn.close() + + def sync(self) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, str(SCRIPT), str(self.dir / "api"), "--db", str(self.db_path), + "--templates", str(self.dir / "templates")], + capture_output=True, text=True, cwd=self.dir, check=False) + + def sha(self) -> str: + return hashlib.sha256(self.db_path.read_bytes()).hexdigest() + + def assert_refused(self, major: int): + sha = self.sha() + result = self.sync() + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn(f"declares documentation version {major}", result.stderr) + self.assertIn("Run this before migrate_tooltip_schema.py; nothing was changed.", result.stderr) + self.assertEqual(self.sha(), sha) + self.assertEqual(sorted(p.name for p in self.dir.glob("documentation.db*")), ["documentation.db"]) + + def test_refuses_major_3_before_the_backup(self): + self.declare((2, "2026-01-01 00:00:00"), (3, "2026-10-19 00:00:00")) + self.assert_refused(3) + + def test_refuses_a_later_major(self): + self.declare((4, "2026-10-19 00:00:00")) + self.assert_refused(4) + + def test_reads_the_newest_changetime_first(self): + self.declare((3, "2026-10-19 00:00:00"), (2, "2026-01-01 00:00:00")) + self.assert_refused(3) + + def test_writes_the_template_at_major_2(self): + # The newest rowid breaks the changeTime tie. + self.declare((3, "2026-10-19 00:00:00"), (2, "2026-10-19 00:00:00")) + result = self.sync() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + conn = sqlite3.connect(self.db_path) + try: + self.assertEqual(conn.execute("SELECT name, content FROM Templates").fetchall(), + [("javadoc.peb", b"{{ page }}")]) + finally: + conn.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_migrate_tooltip_schema.py b/scripts/test_migrate_tooltip_schema.py index b3ae83d4..82b82344 100644 --- a/scripts/test_migrate_tooltip_schema.py +++ b/scripts/test_migrate_tooltip_schema.py @@ -10,9 +10,11 @@ Covers migrate_tooltip_schema.py: that every tooltip and its buttons read back through the new tables, with the queries Code On The Go runs, exactly as they -did through the old ones; that those queries search by key; that a second run, -a dry run and every refusal leave the file byte for byte as it was; and that a -conversion its own read-back check rejects is rolled back. +did through the old ones; that those queries search by key; that every template +is stored compressed against the database's dictionary and decodes back to its +text; that a second run, a dry run and every refusal leave the file byte for +byte as it was; and that a conversion its own read-back check rejects is rolled +back. Run it either way - as a plain script: @@ -22,7 +24,8 @@ uv run --with pytest pytest scripts/test_migrate_tooltip_schema.py -Only the standard library is needed for the first. +The first needs only the standard library and the brotli CLI on PATH, which +the step compresses templates with. """ from __future__ import annotations @@ -30,7 +33,9 @@ import hashlib import io import re +import shutil import sqlite3 +import subprocess import sys import tempfile import traceback @@ -96,6 +101,16 @@ UNIQUE(path) ); CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT ''); +CREATE TABLE CompressionDictionary ( + id INTEGER PRIMARY KEY CHECK (id = 1), + data BLOB NOT NULL +); +CREATE TABLE Templates ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + content BLOB NOT NULL, + UNIQUE(name) +); CREATE TRIGGER AddBook AFTER INSERT ON Content WHEN NEW.path LIKE '%.pdf' BEGIN INSERT INTO Bookshelf (contentID, title) VALUES (NEW.id, CURRENT_TIMESTAMP || NEW.id); @@ -103,6 +118,12 @@ """ CATEGORIES = ["ide", "xml", "java", "kotlin"] +DICTIONARY = b"Detail {n} — été
" @@ -145,6 +170,36 @@ def _run(db_path: Path, **overrides) -> tuple: return code, out.getvalue(), err.getvalue() +def _plain(template) -> bytes: + return template.encode("utf-8") if isinstance(template, str) else template + + +def _brotli(dictionary: bytes, data: bytes, *args: str): + """The brotli CLI against `dictionary`, as the pipeline's codec uses it, independently of the tool's own codec: + the output, or None if it fails (for -d: `data` is no stream against that dictionary). Its decoder also takes + large-window streams, which the app's does not; _templates refuses those.""" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "dictionary.bin" + path.write_bytes(dictionary) + result = subprocess.run(["brotli", "-D", str(path), *args, "-c"], input=data, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False) + return result.stdout if result.returncode == 0 else None + + +def _large_window(stream: bytes) -> bool: + """A large-window header (RFC 7932 leaves its first seven bits, 1 000 001, invalid), which brotli4j refuses.""" + return bool(stream) and stream[0] & 0x7F == 0x11 + + +def _templates(db_path: Path) -> dict: + """{name: (stored bytes, plaintext or None)}, decoded against the database's own dictionary as the app decodes + them: None for a large-window stream, which the CLI would decode.""" + rows = _query(db_path, "SELECT name, content FROM Templates ORDER BY id") + dictionary = _query(db_path, "SELECT data FROM CompressionDictionary")[0][0] + return {name: (_plain(content), None if _large_window(_plain(content)) + else _brotli(dictionary, _plain(content), "-d")) for name, content in rows} + + def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -261,7 +316,7 @@ def test_a_second_run_changes_nothing(tmp_path): backup.unlink() sha = _sha(db) code, out, _err = _run(db) - assert code == 0 and "already in the new shape; nothing to do" in out + assert code == 0 and "already in the new shape and every template is compressed; nothing to do" in out assert _sha(db) == sha and _backups(db) == [] @@ -288,7 +343,7 @@ def _assert_converted_refused(tmp_path, phrase: str, sql: str): def test_refuses_a_converted_database_that_declares_version_2(tmp_path): - # What populate_db.py's declare_database_version leaves when it runs after the conversion. + # A later version row declaring 2, as a step run after the conversion or a hand edit could leave. _assert_converted_refused(tmp_path, "already in the new shape, but DocumentationDatabaseVersion declares major 2", "INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " "VALUES (2, 0, 0, 'populate_db', 'later', '2999-01-01')") @@ -304,7 +359,7 @@ def test_leaves_a_converted_database_declaring_a_later_version_alone(tmp_path): "VALUES (4, 0, 0, 'later', 'later', '2999-01-01')") sha = _sha(db) code, out, _err = _run(db) - assert code == 0 and "already in the new shape; nothing to do" in out + assert code == 0 and "already in the new shape and every template is compressed; nothing to do" in out assert _sha(db) == sha and _backups(db) == [] @@ -668,6 +723,357 @@ def test_rolls_back_a_change_to_a_tooltip_the_app_cannot_reach(tmp_path): assert "rolled back, nothing was changed" in err and "through the app's query" not in err, err +def test_compresses_every_template_and_each_decodes_back(tmp_path): + db = _database(tmp_path) + code, out, err = _run(db) + assert code == 0, err + assert "Every template decodes back to its text (3 template(s)" in out, out + templates = _templates(db) + assert {name: text for name, (_stored, text) in templates.items()} == {n: _plain(t) for n, t in TEMPLATES} + assert all(stored != _plain(dict(TEMPLATES)[name]) for name, (stored, _text) in templates.items()) + assert _query(db, "SELECT typeof(content) FROM Templates GROUP BY 1") == [("blob",)] + + +def test_a_dry_run_compresses_and_checks_but_leaves_templates_plain(tmp_path): + db = _database(tmp_path) + sha = _sha(db) + code, out, err = _run(db, dry_run=True) + assert code == 0, err + assert "compressing 3 template(s)" in out and "Every template decodes back" in out, out + assert _sha(db) == sha and _backups(db) == [] + assert all(text is None for _stored, text in _templates(db).values()) + + +def test_refuses_a_database_without_a_dictionary_row(tmp_path): + _assert_refused(tmp_path, "CompressionDictionary holds no dictionary", "DELETE FROM CompressionDictionary") + + +def test_refuses_a_database_without_a_dictionary_table(tmp_path): + _assert_refused(tmp_path, "no CompressionDictionary table", "DROP TABLE CompressionDictionary") + + +def test_refuses_templates_compressed_already(tmp_path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + for name, template in TEMPLATES: + conn.execute("UPDATE Templates SET content = ? WHERE name = ?", (_brotli(DICTIONARY, _plain(template)), name)) + conn.commit() + conn.close() + sha = _sha(db) + code, _out, err = _run(db) + assert code == 1 and "3 of 3 template(s) already decode against the dictionary" in err, err + assert _sha(db) == sha and _backups(db) == [] + + +def test_refuses_a_mix_of_plain_and_compressed_templates(tmp_path): + db = _database(tmp_path) + conn = sqlite3.connect(db) + conn.execute("UPDATE Templates SET content = ? WHERE name = 'nav.peb'", (_brotli(DICTIONARY, b""),)) + conn.commit() + conn.close() + sha = _sha(db) + code, _out, err = _run(db) + assert code == 1 and "1 of 3 template(s) already decode against the dictionary, so they look compressed already (nav.peb)" in err, err + assert _sha(db) == sha and _backups(db) == [] + + +def test_refuses_a_one_byte_template_that_is_also_a_brotli_stream(tmp_path): + # Each of these plain texts is a complete Brotli stream that decodes to empty, so it is taken for compressed. + for text in "3579;=?": + assert _brotli(DICTIONARY, text.encode(), "-d") == b"", text + assert _brotli(DICTIONARY, b"", "-w", "24") == b"?" + # A text starting with < or { breaks a reserved or padding bit in its first byte, whatever follows. + assert not any(_brotli(DICTIONARY, bytes([first, second]), "-d") is not None for first in b"<{" for second in + b" <>{}%/a?") + _assert_refused(tmp_path, "1 of 3 template(s) already decode against the dictionary, so they look compressed " + "already (nav.peb)", + "UPDATE Templates SET content = CAST('?' AS BLOB) WHERE name = 'nav.peb'") + + +def test_refuses_a_template_that_is_neither_text_nor_compressed(tmp_path): + _assert_refused(tmp_path, "1 template(s) are neither UTF-8 text nor compressed against the dictionary (nav.peb)", + "UPDATE Templates SET content = x'ff00ff' WHERE name = 'nav.peb'") + + +def test_refuses_a_null_template(tmp_path): + _assert_refused(tmp_path, "1 template(s) are neither UTF-8 text nor compressed against the dictionary (nav.peb)", + "UPDATE Templates SET content = NULL WHERE name = 'nav.peb'", + schema=OLD_SCHEMA.replace("content BLOB NOT NULL,\n UNIQUE(name)", "content BLOB,\n UNIQUE(name)")) + + +def test_rolls_back_a_template_stored_with_the_wrong_text(tmp_path): + # Each blob decodes, but nav.peb's holds layout.pebble's text: only the read-back after writing sees it. + db = _database(tmp_path) + compress = tool.compress_templates + + def swapped(conn, codec, plain): + total = compress(conn, codec, plain) + conn.execute("UPDATE Templates SET content = (SELECT content FROM Templates WHERE name = 'layout.pebble') " + "WHERE name = 'nav.peb'") + return total + with _patched_attr(tool, "compress_templates", swapped): + code, _out, err = _run(db, dry_run=True) + assert code == 1 and "template nav.peb does not decode to its text" in err, err + assert "rolled back, nothing was changed" in err, err + + +def test_refuses_when_the_brotli_cli_is_missing(tmp_path): + db = _database(tmp_path) + sha = _sha(db) + with _patched_attr(shutil, "which", lambda _name: None): + code, _out, err = _run(db) + assert code == 1 and "the brotli CLI is not on PATH" in err, err + assert _sha(db) == sha and _backups(db) == [] + + +def _failing_brotli(tmp_path: Path, script: str) -> str: + """A `brotli` on PATH that runs `script` (Python) in place of the CLI, with the real one as REAL.""" + stub = tmp_path / "bin" / "brotli" + stub.parent.mkdir() + stub.write_text(f"#!{sys.executable}\nimport subprocess, sys\nREAL = {shutil.which('brotli')!r}\n{script}") + stub.chmod(0o755) + return str(stub) + + +# A brotli that does not know -D, as an old or a different build would. +REJECTS_DICTIONARIES = 'sys.stderr.write("brotli: unknown option -D\\n")\nsys.exit(1)\n' + + +def test_refuses_when_the_brotli_cli_fails_before_the_backup(tmp_path): + db = _database(tmp_path) + sha = _sha(db) + stub = _failing_brotli(tmp_path, REJECTS_DICTIONARIES) + with _patched_attr(shutil, "which", lambda _name: stub): + code, _out, err = _run(db) + assert code == 1 and "could not compress and decode a test text against the dictionary: brotli: unknown " \ + "option -D" in err, err + assert not any(blame in err for blame in ("do not decode", "neither UTF-8", "could not compress a template")), err + assert _sha(db) == sha and _backups(db) == [] + + +def test_refuses_a_converted_database_when_the_brotli_cli_fails(tmp_path): + # Every template decodes; the CLI is what fails, and the message must say so. + db = _converted(tmp_path) + sha = _sha(db) + stub = _failing_brotli(tmp_path, REJECTS_DICTIONARIES) + with _patched_attr(shutil, "which", lambda _name: stub): + code, _out, err = _run(db) + assert code == 1 and "brotli: unknown option -D" in err and "do not decode" not in err, err + assert _sha(db) == sha and _backups(db) == [] + + +def test_refuses_a_brotli_cli_that_decodes_to_other_bytes(tmp_path): + # It exits 0 but decodes everything to the same bytes, so every template would look compressed already. + db = _database(tmp_path) + sha = _sha(db) + stub = _failing_brotli(tmp_path, "data = sys.stdin.buffer.read()\n" + "if '-d' in sys.argv:\n" + " sys.stdout.buffer.write(b'other
')\n" + " sys.exit(0)\n" + "sys.stdout.buffer.write(subprocess.run([REAL, *sys.argv[1:]], input=data, " + "stdout=subprocess.PIPE, check=True).stdout)\n") + with _patched_attr(shutil, "which", lambda _name: stub): + code, _out, err = _run(db) + assert code == 1 and "could not compress and decode a test text against the dictionary: it decoded to other " \ + "bytes" in err, err + assert "already decode" not in err and _sha(db) == sha and _backups(db) == [] + + +def test_reports_the_cli_error_when_a_template_fails_to_compress(tmp_path): + # The CLI passes its probe, then fails on one template, as an out-of-memory CLI would. + db = _database(tmp_path) + stub = _failing_brotli(tmp_path, "data = sys.stdin.buffer.read()\n" + "if b'