diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml index ae38d727..e7aa311c 100644 --- a/.github/workflows/python-tests.yaml +++ b/.github/workflows/python-tests.yaml @@ -65,14 +65,16 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v5 - # scripts/test_migrate_tooltip_schema.py needs only the standard library and pytest. - - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema' + # scripts/test_migrate_tooltip_schema.py needs only the standard library, pytest and the brotli CLI (above). + - name: 'ProcessKotlinWebsiteJSON + sync_kotlin_stdlib_docs + migrate_tooltip_schema + Android and Java loaders' run: | uv run --with-requirements requirements.txt --with pytest -- \ python -m pytest \ ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON \ scripts/sync_kotlin_stdlib_docs \ scripts/test_migrate_tooltip_schema.py \ + ProcessDocs/AndroidDocs/AndroidDocsToJson \ + scripts/sync_java_docs \ -q - name: docdb-studio diff --git a/CLAUDE.md b/CLAUDE.md index d3735e14..060c04d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,7 +61,7 @@ edit that database** — it contains no part of the production Android app itsel > Note also that `image/webp` (id 26) and `video/quicktime` (id 28) now exist in `ContentTypes`, so > the workflows' `INSERT OR IGNORE ... image/webp` step is a no-op against current copies. -> ### Schema 3.0.0: keyed tooltip tables, in the shipped copy only +> ### Schema 3.0.0: keyed tooltip tables and compressed templates, in the shipped copy only > > `scripts/migrate_tooltip_schema.py` (ADFA-3305) rewrites the tooltip tables and declares **3.0.0**: > `Tooltips(id, categoryId, tag)` WITHOUT ROWID, keyed on `(categoryId, tag)`; @@ -70,11 +70,31 @@ edit that database** — it contains no part of the production Android app itsel > string once in `TooltipButtonDescriptions` and `TooltipButtonUris`. Code On The Go from > CodeOnTheGo #2163 on reads only these tables, and refuses tooltips from a database declaring less. > +> The same step stores **every `Templates` row Brotli-compressed** against `CompressionDictionary`, +> with no per-row marker: in 3.0.0 all templates are compressed, and the app always decodes them. +> Every other tool that reads or writes Templates expects them plain: `populate_db.py`, +> `sync_javadoc_json_to_db.py`, `ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py` +> and `verify_android_json_db.py`, and in open PRs `minify_text.py` (#44), the `a/` +> pages-to-templates step (`android_pages_to_templates.py`, #49) and `hm_pages_to_templates.py` +> (#43). On the master the templates stay plain. Some of these run on the copy that ships as well +> (the minify and the `a/` step do), so there every one of them runs before the migration. The order +> on that copy is: minify, row drops, Kotlin trim, `a/` pages-to-templates, +> `migrate_tooltip_schema.py`, PR #48's `remint_dictionary.py --seeds 4` (which recompresses +> templates too in 3.0.0), then `rebuild_database.py --pack Content,TooltipTexts`; +> `hm_pages_to_templates.py` goes after the `i/` page steps it lists, before the migration like +> every other Templates writer. PR #48 has to merge no later than this step's PRs (#45, #50): the +> re-mint and verifier before it handle Content only, so they refuse a database declaring 3.0.0. +> +> `populate_db.py`, `sync_javadoc_json_to_db.py` and `load_android_json_db.py` write plain templates, +> so each refuses a database declaring 3.0.0 or later before writing anything, backup included. +> `load_android_json_db.py` keeps the major it finds, so a late run would otherwise leave a file +> declaring 3.1.0 with plain templates in it, which the app cannot read. +> > **Every tool in this repo still reads and writes the 2.x shape**: `check-tools` (whose daily > `docdb-regression-test.yaml` run selects `TooltipButtons.uri` and `Tooltips.detail`), > `sync_kdoc_json_to_db.py` (step 5 of `build-kotlin-docs.yaml`), `TooltipManager.py` and > `docdb-studio`. So the step runs on the copy that ships, after all of them, and never on the -> master database they edit; nothing converts 3.0 back. +> master database they edit; nothing converts 3.0 back. It needs the `brotli` CLI on PATH. > > In the 3.0 shape `Tooltips.id` must be unique and name a `TooltipTexts` row: a writer takes the > id its `TooltipTexts` insert hands out, never one of its own. Nothing enforces it (a `UNIQUE (id)` @@ -144,8 +164,9 @@ removes it. Measurements are in its docstring. **This repo writes `Templates`, but not `Bookshelf` or `BookCategories`.** `populate_db.py` (`page.peb`, `nav.peb`), `sync_javadoc_json_to_db.py` (`javadoc.peb`) and `load_android_json_db.py` (the `android-*.peb` templates) write rows there and point the pages they write at them via -`templateId`; `verify_android_json_db.py` reads them; and `remint_dictionary.py` and -`verify_remint_dictionary.py` recompress and compare them from 3.0.0 on (above). +`templateId`; `verify_android_json_db.py` reads them; `migrate_tooltip_schema.py` compresses +every one for 3.0.0; and `remint_dictionary.py` and `verify_remint_dictionary.py` recompress and +compare them from 3.0.0 on (above). `insert_optimized_media.py` reads and writes `templateId` too. `Bookshelf` and `BookCategories` come from App Dev For All's plugin system, which writes into the documentation database - the bookshelf feature from @@ -206,10 +227,11 @@ shape: every tool here reads and writes that, and only the shipped copy holds th ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), `create_empty_database.py`, `list_database_documents.py`, `migrate_tooltip_schema.py` (ADFA-3305: - converts the tooltip tables to the WITHOUT ROWID, lookup-table shape Code On The Go reads from - `DocumentationDatabaseVersion` 3.0.0 on; run it on the copy that ships, after every step that writes - tooltips, since they all write the old shape, never on the master they edit, and pack - `Content,TooltipTexts` afterwards, not `Tooltips`), `minify_text.py` (ADFA-6627: minifies the + converts to documentation schema 3.0.0, which Code On The Go reads from 3.0.0 on: the WITHOUT ROWID, + lookup-table tooltip shape, and every template Brotli-compressed against the dictionary; run it on the + copy that ships, after every step that writes tooltips or templates, since they all write the old + shape and plain templates, never on the master they edit; re-mint afterwards, then pack + `Content,TooltipTexts`, not `Tooltips`), `minify_text.py` (ADFA-6627: minifies the HTML, CSS, JS and SVG rows and the HTML templates without changing what a page shows; run `ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/remint_dictionary.py` and `verify_remint_dictionary.py` straight after it, or the minified rows compress worse than before), diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py index 8df0c047..11aec50a 100755 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/load_android_json_db.py @@ -45,8 +45,8 @@ sys.path.insert(0, str(_KOTLIN_PIPELINE)) from populate_db import ( # noqa: E402 (the path has to be set up first) - DictionaryCompressor, fragment_chain, get_content_type, get_id, insert_chunked_content, - load_dictionary, upsert_template, + DictionaryCompressor, compressed_templates_refusal, fragment_chain, get_content_type, get_id, + insert_chunked_content, load_dictionary, upsert_template, ) from migrate_content_to_dictionary_brotli import write_item # noqa: E402 @@ -328,11 +328,15 @@ def main(argv: list[str] | None = None) -> int: source_conn = sqlite3.connect(f"file:{args.source}?mode=ro", uri=True) try: missing = check_schema(source_conn) + # record_version keeps the major, so a 3.x source would come out 3.(x+1) with plain templates in it. + refusal = None if missing else compressed_templates_refusal(source_conn) finally: source_conn.close() if missing: parser.error(f"{args.source} is missing table(s) this loader writes: " + ", ".join(missing)) + if refusal: + parser.error(f"{args.source}: {refusal}; nothing was written") print(f"==> copying {args.source} -> {args.out}") shutil.copy2(args.source, args.out) diff --git a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py index b8905e0b..0af304b0 100644 --- a/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py +++ b/ProcessDocs/AndroidDocs/AndroidDocsToJson/test_load_android_json_db.py @@ -260,3 +260,37 @@ def test_the_comment_carries_the_counts(self, db): comment = db.execute("SELECT comment FROM DocumentationDatabaseVersion").fetchone()[0] assert "12,106 rows replaced" in comment and "800 added" in comment + + +class TestCompressedTemplates: + """From documentation 3.0.0 every template is stored compressed and the app decodes every one. record_version + keeps the source's major, so loading a 3.x source would leave 3.(x+1) with plain Android templates in it.""" + + def run(self, db, tmp_path, *versions): + db.executescript(VERSION_TABLES) + db.execute("INSERT INTO ContentTypes (value, compression) VALUES ('text/css', 'brotli')") + db.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (?, 0, 0, 'test', 'test', ?)", versions) + db.commit() + (tmp_path / "json").mkdir() + return loader.main([str(tmp_path / "json"), "--source", str(tmp_path / "test.db"), + "--out", str(tmp_path / "out.db"), "--workers", "2"]) + + @pytest.mark.parametrize("major", [3, 4]) + def test_a_source_declaring_3_or_later_is_refused_before_anything_is_written(self, db, tmp_path, capsys, major): + with pytest.raises(SystemExit) as exited: + self.run(db, tmp_path, (2, "2026-01-01 00:00:00"), (major, "2026-10-19 00:00:00")) + assert exited.value.code == 2 + assert f"declares documentation version {major}" in capsys.readouterr().err + assert not (tmp_path / "out.db").exists() + + def test_a_source_declaring_2_still_loads(self, db, tmp_path): + # The newest row by changeTime is 2, though a 3 was inserted after it. + assert self.run(db, tmp_path, (2, "2026-10-19 00:00:00"), (3, "2026-01-01 00:00:00")) == 0 + out = sqlite3.connect(tmp_path / "out.db") + try: + assert out.execute("SELECT major, minor FROM DocumentationDatabaseVersion " + "ORDER BY rowid DESC LIMIT 1").fetchone() == (2, 1) + assert out.execute("SELECT COUNT(*) FROM Templates").fetchone() == (3,) + finally: + out.close() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py index fbbe5233..e7e58921 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py @@ -569,7 +569,37 @@ def insert_file(conn, data: bytes, name: str, db_path: str, language_id: int, co return True +# From this documentation major on, migrate_tooltip_schema.py (ADFA-3305) has stored every template compressed +# against the dictionary, and the app decodes every one: a plain template written into such a database would not +# render, and the file would still declare 3.x. +COMPRESSED_TEMPLATES_MAJOR = 3 + + +def declared_major(conn) -> int | None: + """The documentation major the database declares - its newest DocumentationDatabaseVersion row, ordered as + Code On The Go orders them (changeTime, then rowid) - or None if it declares none.""" + if not _table_exists(conn, "DocumentationDatabaseVersion"): + return None + row = conn.execute("SELECT major FROM DocumentationDatabaseVersion " + "ORDER BY changeTime DESC, rowid DESC LIMIT 1").fetchone() + return None if row is None else int(row[0]) + + +def compressed_templates_refusal(conn) -> str | None: + """Why no plain template may be written into this database, or None.""" + major = declared_major(conn) + if major is None or major < COMPRESSED_TEMPLATES_MAJOR: + return None + return (f"the database declares documentation version {major}; from {COMPRESSED_TEMPLATES_MAJOR}.0.0 every " + "template is stored compressed, and this step writes plain ones. Run it before migrate_tooltip_schema.py") + + def upsert_template(conn, name: str, content: str) -> int: + """Writes one plain template. Raises RuntimeError, writing nothing, if the database's templates are stored + compressed.""" + refusal = compressed_templates_refusal(conn) + if refusal: + raise RuntimeError(refusal) conn.execute( "INSERT INTO Templates (name, content) VALUES (?, ?) " "ON CONFLICT(name) DO UPDATE SET content = excluded.content", @@ -718,6 +748,15 @@ def main(): if not args.db_path.is_file(): print(f"error: {args.db_path} does not exist", file=sys.stderr) sys.exit(1) + # Checked before the conversion and the backup, so a refusal costs nothing and writes nothing. + conn = sqlite3.connect(f"file:{args.db_path}?mode=ro", uri=True) + try: + refusal = compressed_templates_refusal(conn) + finally: + conn.close() + if refusal: + print(f"error: {refusal}; nothing was changed", file=sys.stderr) + sys.exit(1) config = load_config(args.config) variables = load_variables(docs_root) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py new file mode 100644 index 00000000..8ee4ec21 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/test_populate_db_templates.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Tests that populate_db.py writes no plain template into a database whose templates are compressed (ADFA-3305). + +From documentation 3.0.0 migrate_tooltip_schema.py stores every template compressed, and the app decodes every one, +so a plain page.peb or nav.peb written afterwards would not render while the file still declared 3.x. + +Run directly: python3 test_populate_db_templates.py +""" +import hashlib +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +from populate_db import compressed_templates_refusal, declared_major, upsert_template + +HERE = Path(__file__).resolve().parent + +SCHEMA = """ +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE Content (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, UNIQUE(path)); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE(name)); +CREATE TABLE CompressionDictionary (id INTEGER PRIMARY KEY CHECK (id = 1), data BLOB NOT NULL); +CREATE TABLE DocumentationDatabaseVersion (major INT NOT NULL, minor INT NOT NULL, patch INT NOT NULL, + who TEXT NOT NULL, comment TEXT NOT NULL, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP); +INSERT INTO Languages (value) VALUES ('en-US'); +INSERT INTO ContentTypes (value, compression) VALUES ('text/html', 'brotli'), ('text/css', 'brotli'), + ('text/javascript', 'brotli'); +INSERT INTO CompressionDictionary (id, data) VALUES (1, CAST('' AS BLOB)); +INSERT INTO Templates (name, content) VALUES ('page.peb', CAST('stored before' AS BLOB)); +""" + + +class CompressedTemplatesRefusalTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.dir = Path(directory.name) + self.db_path = self.dir / "documentation.db" + conn = sqlite3.connect(self.db_path) + conn.executescript(SCHEMA) + conn.close() + + def declare(self, *rows): + """Appends (major, changeTime) rows to DocumentationDatabaseVersion, in order.""" + conn = sqlite3.connect(self.db_path) + conn.executemany("INSERT INTO DocumentationDatabaseVersion (major, minor, patch, who, comment, changeTime) " + "VALUES (?, 0, 0, 'test', 'test', ?)", rows) + conn.commit() + conn.close() + + def connect(self): + conn = sqlite3.connect(self.db_path) + self.addCleanup(conn.close) + return conn + + def templates(self) -> list: + return self.connect().execute("SELECT name, content FROM Templates ORDER BY id").fetchall() + + def test_upsert_template_refuses_major_3(self): + self.declare((2, "2026-01-01 00:00:00"), (3, "2026-10-19 00:00:00")) + before = self.templates() + conn = self.connect() + with self.assertRaises(RuntimeError) as raised: + upsert_template(conn, "page.peb", "new") + self.assertIn("declares documentation version 3", str(raised.exception)) + self.assertIn("Run it before migrate_tooltip_schema.py", str(raised.exception)) + conn.commit() + self.assertEqual(self.templates(), before) + + def test_upsert_template_refuses_a_later_major(self): + self.declare((4, "2026-10-19 00:00:00")) + with self.assertRaises(RuntimeError): + upsert_template(self.connect(), "nav.peb", "