4"""Generate the rabook_compile byte-identity parity fixture headers.
6The on-device compiler (rabook_compile_from_epub) must emit a RABOOK1 flat
7blob byte-identical to the desktop reference tools/epub_compile/src/epub_compile.py.
8The EPUB parity fixture is pinned to a well-formed text-only + SVG EPUB (raster
9images are covered separately by the --downscale fixture below). This script
10bakes both sides of that comparison into a C header the host test embeds:
12 * s_parity_epub[] -- the fixture .epub bytes (written to a RAM FAT volume
13 and opened with epub, the test's compiler input).
14 * s_parity_golden[] -- the golden RABOOK1 flat blob: epub_compile.py run on
15 the same .epub, with its RBKC chunked container
16 stripped (header + chunk table) and inflated.
18Run it via `just tools::rabook_golden` after any change to the format, the
19emitter, or the fixture; the committed header is then the frozen acceptance
20target the parity test diffs against. The desktop tool needs Pillow installed.
23 rabook_parity_gen.py SRC_DIR OUT_HEADER [EXAMPLE_HEADER]
24 rabook_parity_gen.py --realbook SRC_DIR OUT_HEADER
25 rabook_parity_gen.py --downscale OUT_HEADER
26 rabook_parity_gen.py --color OUT_HEADER
28SRC_DIR holds the fixture's EPUB component files (META-INF/, OEBPS/); the
29`mimetype` entry is synthesized as the fixed EPUB constant. Extend the fixture
30(add CSS / a sub-1600px image) by editing SRC_DIR, then rerun.
32The `--color` form bakes rabook_color_parity_fixture.h: a synthetic RGB PNG plus
33the golden 4-bpp blob the desktop tool emits for it on the DEFAULT no-downscale
34path (host stb_luma8 + gray4_kernel.gray4_encode). test_ra8_rabook_color_parity.c
35decodes the same PNG with the firmware's stb_image and encodes with the same
36kernel, so byte-identity proves the default path is one luma and one quantiser
37host-vs-device -- decode included -- closing the gap #337 describes.
39The `--realbook` form bakes a single header (s_realbook_epub +
40s_realbook_golden_noimg) from a real-book fixture (verbatim Standard Ebooks
41chapters) using the desktop tool's --no-images path -- the text/CSS-only golden
42the on-device skip-images compile must match byte-for-byte (#151).
44The `--downscale` form bakes rabook_downscale_parity_fixture.h: a synthetic gray
45source plus the golden 4-bpp blob the desktop tool emits for it via the exact
46integer bilinear kernel (tools/epub_compile/src/gray4_kernel.py, a mirror of
47ra8_rabook_gray4_downscale/_encode). The firmware kernel run over the same source
48must match that golden byte-for-byte, closing the host-vs-device downscale gap
49(#213) -- the opt-in downscale path is now one deterministic kernel, not a
50LANCZOS-vs-bilinear exception.
53from __future__
import annotations
63from pathlib
import Path
73_ZIP_EPOCH = (1980, 1, 1, 0, 0, 0)
74_MIMETYPE = b
"application/epub+zip"
77def _build_epub(src_dir: Path) -> bytes:
78 """Pack SRC_DIR into a byte-deterministic .epub (mimetype first, all STORED).
80 Every member is stored uncompressed with a fixed timestamp so the embedded
81 bytes never drift across zlib versions or run times; both the desktop tool
82 and epub read the members by name, so STORED is read identically.
84 members = sorted(p
for p
in src_dir.rglob(
"*")
if p.is_file())
86 with zipfile.ZipFile(buf,
"w")
as zf:
87 mt = zipfile.ZipInfo(
"mimetype", _ZIP_EPOCH)
88 mt.compress_type = zipfile.ZIP_STORED
89 zf.writestr(mt, _MIMETYPE)
91 rel = path.relative_to(src_dir).as_posix()
92 info = zipfile.ZipInfo(rel, _ZIP_EPOCH)
93 info.compress_type = zipfile.ZIP_STORED
94 zf.writestr(info, path.read_bytes())
98def _compile_desktop(epub_bytes: bytes, *, no_images: bool =
False) -> bytes:
99 """Run epub_compile.py on the .epub bytes; return the inflated flat blob.
101 When `no_images` is true the desktop tool is invoked with `--no-images`, so
102 the emitted blob drops the image table + cover index (text/CSS-only). That
103 is the golden the on-device skip-images path (scratch `skip_images = true`)
104 must match byte-for-byte.
106 repo = Path(__file__).resolve().parents[2]
107 tool = repo /
"tools" /
"epub_compile" /
"src" /
"epub_compile.py"
108 with tempfile.TemporaryDirectory()
as td:
109 src = Path(td) /
"fixture.epub"
110 out = Path(td) /
"golden.rabook"
111 src.write_bytes(epub_bytes)
112 argv = [sys.executable, str(tool), str(src), str(out)]
114 argv.append(
"--no-images")
120 container = out.read_bytes()
121 if container[: len(_RBKC_MAGIC)] != _RBKC_MAGIC:
122 msg =
"desktop output is not an RBKC container"
123 raise ValueError(msg)
124 chunk_bytes, want, count, reserved = struct.unpack_from(
"<IQII", container, 4)
125 if reserved != 0
or chunk_bytes == 0
or count != (want + chunk_bytes - 1) // chunk_bytes:
126 msg =
"malformed RBKC header"
127 raise ValueError(msg)
128 offsets = struct.unpack_from(f
"<{count + 1}Q", container, _RBKC_HEADER_LEN)
129 payload = _RBKC_HEADER_LEN + 8 * (count + 1)
131 zlib.decompress(container[payload + offsets[i] : payload + offsets[i + 1]])
132 for i
in range(count)
134 if len(blob) != want:
135 msg = f
"inflated size {len(blob)} != header {want}"
136 raise ValueError(msg)
140def _emit_array(name: str, data: bytes) -> str:
142 for i
in range(0, len(data), _BYTES_PER_ROW):
143 chunk = data[i : i + _BYTES_PER_ROW]
144 rows.append(
" " +
"".join(f
"0x{b:02X}, " for b
in chunk).rstrip())
145 body =
"\n".join(rows)
146 return f
"static const uint8_t {name}[] = {{\n{body}\n}};\n"
149def _render(epub_bytes: bytes, golden: bytes, golden_noimg: bytes) -> str:
152 " * @file rabook_parity_fixture.h\n"
153 " * @brief Byte-identity parity fixture for rabook_compile (#151).\n"
154 " * @details Pins one synthetic EPUB against desktop-generated RABOOK1\n"
155 " * bytes with and without images for firmware parity checks.\n"
157 " * @generated by scripts/gen/rabook_parity_gen.py "
158 "(just tools::rabook_golden);\n"
159 " * do not hand-edit. The generator owns this file's length:\n"
160 " * it bakes the fixture .epub plus the golden RABOOK1 flat\n"
161 " * blobs that tools/epub_compile/src/epub_compile.py emits for it,\n"
162 " * one with images (the default) and one with --no-images (the\n"
163 " * skip-images path: text/CSS-only, no image table or cover).\n"
165 " * @copyright Copyright (c) 2026 Brighton Sikarskie\n"
166 " * SPDX-License-Identifier: MIT\n"
171 "#include <stdint.h>\n"
173 "/** @brief Fixture .epub bytes (text-only, well-formed XHTML). */\n"
174 f
"{_emit_array('s_parity_epub', epub_bytes)}"
176 "/** @brief Golden RABOOK1 flat blob (desktop epub_compile.py, "
177 "RBKC-stripped). */\n"
178 f
"{_emit_array('s_parity_golden', golden)}"
180 "/** @brief Golden RABOOK1 flat blob for --no-images "
181 "(skip-images path). */\n"
182 f
"{_emit_array('s_parity_golden_noimg', golden_noimg)}"
184 "/** @brief Sizes of the embedded fixture and golden blobs (bytes). */\n"
185 "enum : uint32_t {\n"
186 f
" k_parity_epub_len = {len(epub_bytes)}U, /**< Length of s_parity_epub in bytes. */\n"
187 f
" k_parity_golden_len = {len(golden)}U, "
188 "/**< Length of s_parity_golden in bytes. */\n"
189 f
" k_parity_golden_noimg_len = {len(golden_noimg)}U, "
190 "/**< Length of s_parity_golden_noimg in bytes. */\n"
195def _render_realbook(epub_bytes: bytes, golden_noimg: bytes) -> str:
198 " * @file rabook_realbook_fixture.h\n"
199 " * @brief Real-book byte-identity fixture for rabook_compile (#151).\n"
200 " * @details Pins representative Walden XHTML and its desktop-generated\n"
201 " * text-only RABOOK1 bytes for firmware parity checks.\n"
203 " * @generated by scripts/gen/rabook_parity_gen.py "
204 "(just tools::rabook_golden);\n"
205 " * do not hand-edit. Built from real Standard Ebooks Walden\n"
206 " * chapters (tests/fixtures/rabook_realbook/) -- including the\n"
207 " * ones carrying the significant `</abbr> <abbr>` inter-element\n"
208 " * whitespace -- this bakes the fixture .epub plus the golden\n"
209 " * RABOOK1 flat blob that tools/epub_compile/src/epub_compile.py\n"
210 " * emits with --no-images (text/CSS-only). The on-device\n"
211 " * skip-images compile must equal it byte-for-byte, proving the\n"
212 " * chapter DOM (and its preserved inline whitespace) round-trips\n"
213 " * against the desktop reference on real content.\n"
215 " * @copyright Copyright (c) 2026 Brighton Sikarskie\n"
216 " * SPDX-License-Identifier: MIT\n"
221 "#include <stdint.h>\n"
223 "/** @brief Real-book fixture .epub bytes (verbatim Walden chapters). */\n"
224 f
"{_emit_array('s_realbook_epub', epub_bytes)}"
226 "/** @brief Golden RABOOK1 flat blob for --no-images "
227 "(skip-images path). */\n"
228 f
"{_emit_array('s_realbook_golden_noimg', golden_noimg)}"
230 "/** @brief Sizes of the embedded fixture and golden blob (bytes). */\n"
231 "enum : uint32_t {\n"
232 f
" k_realbook_epub_len = {len(epub_bytes)}U, "
233 "/**< Length of s_realbook_epub in bytes. */\n"
234 f
" k_realbook_golden_noimg_len = {len(golden_noimg)}U, "
235 "/**< Length of s_realbook_golden_noimg in bytes. */\n"
240def _render_example(epub_bytes: bytes, golden: bytes) -> str:
243 " * @file parity_fixture.h\n"
244 " * @brief Baked parity .epub + golden blob for the M33 compile (#149).\n"
245 " * @details Pins the shared synthetic EPUB and desktop RABOOK1 bytes so\n"
246 " * the secondary-core compiler can prove byte identity.\n"
248 " * @generated by scripts/gen/rabook_parity_gen.py "
249 "(just tools::rabook_golden);\n"
250 " * do not hand-edit. The compile_on_m33 CPU1 image compiles\n"
251 " * s_m33_parity_epub on the Cortex-M33 and the M85 compares the\n"
252 " * emitted blob to s_m33_parity_golden (byte-identity on the\n"
253 " * secondary core). Same fixture + golden as the M85 parity gate.\n"
255 " * @copyright Copyright (c) 2026 Brighton Sikarskie\n"
256 " * SPDX-License-Identifier: MIT\n"
261 "#include <stdint.h>\n"
263 "/** @brief Fixture .epub bytes compiled on the M33 (text/CSS/SVG). */\n"
264 f
"{_emit_array('s_m33_parity_epub', epub_bytes)}"
266 "/** @brief Golden RABOOK1 blob the M33 output must equal byte-for-byte. */\n"
267 f
"{_emit_array('s_m33_parity_golden', golden)}"
269 "/** @brief Sizes of the embedded fixture and golden blobs (bytes). */\n"
270 "enum : uint32_t {\n"
271 f
" k_m33_parity_epub_len = {len(epub_bytes)}U, "
272 "/**< Length of s_m33_parity_epub in bytes. */\n"
273 f
" k_m33_parity_golden_len = {len(golden)}U, "
274 "/**< Length of s_m33_parity_golden in bytes. */\n"
292def _downscale_source() -> bytes:
293 """Deterministic synthetic 8-bpp gray source for the downscale-parity fixture."""
295 ((x * _DS_RAMP_X) + (y * _DS_RAMP_Y)) & _DS_BYTE_MASK
296 for y
in range(_DS_SRC_H)
297 for x
in range(_DS_SRC_W)
301def _crosscheck_desktop_tool(src: bytes, golden: bytes) ->
None:
302 """Fail generation unless epub_compile.py emits `golden` for `src` downscaled.
304 Encodes `src` as a grayscale PNG, runs the production BlobBuilder image arm
305 with the fixture max-edge, and asserts the packed bytes equal the kernel golden
306 -- so the baked fixture can never drift from the shipping desktop tool.
308 from PIL
import Image
309 from rabook_blob
import BlobBuilder
312 Image.frombytes(
"L", (_DS_SRC_W, _DS_SRC_H), src).save(png,
"PNG")
313 builder = BlobBuilder()
314 idx = builder.add_raster_image(
"cover.png", png.getvalue(), max_image_edge=_DS_MAX_EDGE)
315 raw = builder.images[idx][4]
317 msg =
"epub_compile.py downscale output diverged from the gray4_kernel golden"
318 raise RuntimeError(msg)
321def _render_downscale(src: bytes, out_w: int, out_h: int, golden: bytes, sha_hex: str) -> str:
324 " * @file rabook_downscale_parity_fixture.h\n"
325 " * @brief Downscale-kernel byte-identity fixture for ra8_rabook_gray4 (#213).\n"
326 " * @details Pins a synthetic grayscale source against the desktop integer\n"
327 " * downscale kernel's packed output for firmware parity checks.\n"
329 " * @generated by scripts/gen/rabook_parity_gen.py --downscale\n"
330 " * (just tools::rabook_golden); do not hand-edit. Bakes a\n"
331 " * synthetic 8-bpp gray source (s_ds_src) plus the golden 4-bpp\n"
332 " * blob (s_ds_golden) the desktop compiler emits for it via the\n"
333 " * exact integer bilinear kernel (tools/epub_compile/src/gray4_kernel\n"
334 " * .py, a mirror of ra8_rabook_gray4_downscale/_encode).\n"
335 " * test_ra8_rabook_downscale_parity.c runs the firmware kernel over\n"
336 " * s_ds_src and asserts byte-identity to s_ds_golden, so the opt-in\n"
337 " * downscale path is one deterministic kernel host-vs-device.\n"
339 " * @copyright Copyright (c) 2026 Brighton Sikarskie\n"
340 " * SPDX-License-Identifier: MIT\n"
345 "#include <stdint.h>\n"
347 "/** @brief Synthetic 8-bpp gray source (row-major, k_ds_src_w*k_ds_src_h). */\n"
348 f
"{_emit_array('s_ds_src', src)}"
350 "/** @brief Golden 4-bpp packed downscale output (desktop gray4_kernel). */\n"
351 f
"{_emit_array('s_ds_golden', golden)}"
353 "/** @brief SHA-256 (hex) of s_ds_golden -- the host downscale digest. */\n"
354 f
'static const char s_ds_golden_sha256[] = "{sha_hex}";\n'
356 "/** @brief Fixture dimensions and golden length (bytes). */\n"
357 "enum : uint32_t {\n"
358 f
" k_ds_src_w = {_DS_SRC_W}U, /**< Source width in pixels. */\n"
359 f
" k_ds_src_h = {_DS_SRC_H}U, /**< Source height in pixels. */\n"
360 f
" k_ds_max_edge = {_DS_MAX_EDGE}U, /**< Maximum output edge in pixels. */\n"
361 f
" k_ds_out_w = {out_w}U, /**< Downscaled width in pixels. */\n"
362 f
" k_ds_out_h = {out_h}U, /**< Downscaled height in pixels. */\n"
363 f
" k_ds_golden_len = {len(golden)}U, /**< Length of s_ds_golden in bytes. */\n"
368def _main_downscale(argv: list[str]) -> int:
370 sys.stderr.write(
"usage: rabook_parity_gen.py --downscale OUT_HEADER\n")
373 repo = Path(__file__).resolve().parents[2]
374 sys.path.insert(0, str(repo /
"tools" /
"epub_compile" /
"src"))
376 from gray4_kernel
import gray4_transcode
378 src = _downscale_source()
379 out_w, out_h, golden = gray4_transcode(src, _DS_SRC_W, _DS_SRC_H, _DS_MAX_EDGE)
380 _crosscheck_desktop_tool(src, golden)
381 sha_hex = hashlib.sha256(golden).hexdigest()
382 out.write_text(_render_downscale(src, out_w, out_h, golden, sha_hex), encoding=
"ascii")
384 f
"{out}: {_DS_SRC_W}x{_DS_SRC_H} -> {out_w}x{out_h} gray4 downscale, "
385 f
"{len(golden)} B golden (sha256 {sha_hex})\n"
403_COLOR_BYTE_MASK = 0xFF
406def _color_source_rgb() -> bytes:
407 """Deterministic 16x16 RGB source (row-major, 3 bytes/pixel) for the fixture."""
409 for i
in range(_COLOR_W * _COLOR_H):
410 out.append((i * _COLOR_R_STEP) & _COLOR_BYTE_MASK)
411 out.append((i * _COLOR_G_STEP) & _COLOR_BYTE_MASK)
412 out.append((i * _COLOR_B_STEP) & _COLOR_BYTE_MASK)
416def _render_color(png: bytes, golden: bytes, sha_hex: str, crc: int) -> str:
419 " * @file rabook_color_parity_fixture.h\n"
420 " * @brief Colour-raster host-vs-device byte-identity fixture (#337).\n"
421 " * @details Pins a synthetic RGB PNG against the desktop luma and gray4\n"
422 " * pipeline's packed output for firmware parity checks.\n"
424 " * @generated by scripts/gen/rabook_parity_gen.py --color\n"
425 " * (just tools::rabook_golden); do not hand-edit. Bakes a\n"
426 " * synthetic RGB PNG source (s_color_src_png) plus the golden\n"
427 " * 4-bpp blob (s_color_golden) the desktop compiler emits for it\n"
428 " * through the shared luma + quantiser (stb_luma8 ->\n"
429 " * gray4_kernel.gray4_encode). test_ra8_rabook_color_parity.c\n"
430 " * decodes the SAME PNG with stb_image and encodes with the SAME\n"
431 " * firmware kernel, asserting byte-identity -- so the no-downscale\n"
432 " * default path is one luma and one quantiser host-vs-device,\n"
433 " * decode included (issue #337).\n"
435 " * @copyright Copyright (c) 2026 Brighton Sikarskie\n"
436 " * SPDX-License-Identifier: MIT\n"
441 "#include <stdint.h>\n"
443 "/** @brief Synthetic RGB source, PNG-encoded (stb_image / Pillow decodable). */\n"
444 f
"{_emit_array('s_color_src_png', png)}"
446 "/** @brief Golden 4-bpp packed output (host stb_luma8 + gray4_kernel). */\n"
447 f
"{_emit_array('s_color_golden', golden)}"
449 "/** @brief SHA-256 (hex) of s_color_golden -- the host digest. */\n"
450 f
'static const char s_color_golden_sha256[] = "{sha_hex}";\n'
452 "/** @brief Source dimensions, PNG length, golden length, and golden CRC-32. */\n"
453 "enum : uint32_t {\n"
454 f
" k_color_img_w = {_COLOR_W}U, /**< Source image width in pixels. */\n"
455 f
" k_color_img_h = {_COLOR_H}U, /**< Source image height in pixels. */\n"
456 f
" k_color_src_png_len = {len(png)}U, /**< Length of s_color_src_png in bytes. */\n"
457 f
" k_color_golden_len = {len(golden)}U, /**< Length of s_color_golden in bytes. */\n"
458 f
" k_color_golden_crc32 = 0x{crc:08X}U, /**< CRC-32/ISO-HDLC of s_color_golden. */\n"
463def _main_color(argv: list[str]) -> int:
465 sys.stderr.write(
"usage: rabook_parity_gen.py --color OUT_HEADER\n")
468 repo = Path(__file__).resolve().parents[2]
469 sys.path.insert(0, str(repo /
"tools" /
"epub_compile" /
"src"))
471 from PIL
import Image
472 from rabook_blob
import BlobBuilder
474 rgb = _color_source_rgb()
475 png_buf = io.BytesIO()
476 Image.frombytes(
"RGB", (_COLOR_W, _COLOR_H), rgb).save(png_buf,
"PNG")
477 png = png_buf.getvalue()
479 builder = BlobBuilder()
480 idx = builder.add_raster_image(
"cover.png", png)
481 golden = builder.images[idx][4]
482 sha_hex = hashlib.sha256(golden).hexdigest()
483 crc = zlib.crc32(golden) & 0xFFFFFFFF
484 out.write_text(_render_color(png, golden, sha_hex, crc), encoding=
"ascii")
486 f
"{out}: {_COLOR_W}x{_COLOR_H} RGB -> {len(golden)} B gray4 golden "
487 f
"(crc32 {crc:08X}, sha256 {sha_hex})\n"
492def _main_realbook(argv: list[str]) -> int:
494 sys.stderr.write(
"usage: rabook_parity_gen.py --realbook SRC_DIR OUT_HEADER\n")
496 src_dir = Path(argv[2])
498 epub_bytes = _build_epub(src_dir)
499 golden_noimg = _compile_desktop(epub_bytes, no_images=
True)
500 out.write_text(_render_realbook(epub_bytes, golden_noimg), encoding=
"ascii")
502 f
"{out}: {len(epub_bytes)} B epub -> {len(golden_noimg)} B --no-images golden blob\n"
507def main(argv: list[str]) -> int:
508 """Generate one of the .rabook byte-identity parity fixtures.
510 Four modes -- default, ``--realbook``, ``--downscale`` and ``--color`` -- each
511 baking a different input through the compiler and emitting the expected output
512 as a C header. The firmware then asserts it reproduces those bytes exactly,
513 which is what proves the host compiler and the on-device one agree.
515 Returns 0 on success, non-zero on a usage error.
517 if len(argv) >= 2
and argv[1] ==
"--downscale":
518 return _main_downscale(argv)
519 if len(argv) >= 2
and argv[1] ==
"--color":
520 return _main_color(argv)
521 if len(argv) >= 2
and argv[1] ==
"--realbook":
522 return _main_realbook(argv)
523 if len(argv)
not in (3, 4):
525 "usage: rabook_parity_gen.py SRC_DIR OUT_HEADER [EXAMPLE_HEADER]\n"
526 " rabook_parity_gen.py --realbook SRC_DIR OUT_HEADER\n"
527 " rabook_parity_gen.py --downscale OUT_HEADER\n"
528 " rabook_parity_gen.py --color OUT_HEADER\n"
531 src_dir = Path(argv[1])
533 epub_bytes = _build_epub(src_dir)
534 golden = _compile_desktop(epub_bytes)
535 golden_noimg = _compile_desktop(epub_bytes, no_images=
True)
536 out.write_text(_render(epub_bytes, golden, golden_noimg), encoding=
"ascii")
538 f
"{out}: {len(epub_bytes)} B epub -> {len(golden)} B golden blob "
539 f
"(+{len(golden_noimg)} B --no-images golden)\n"
542 example = Path(argv[3])
543 example.write_text(_render_example(epub_bytes, golden), encoding=
"ascii")
544 sys.stdout.write(f
"{example}: example-side header written\n")
548if __name__ ==
"__main__":
549 sys.exit(
main(sys.argv))
void main(void)
The application entry point Reset_Handler hands control to.