Add CI check that rebuilds Ventoy's EFI binaries reproducibly

.github/workflows/edk2-efi-repro.yml

@@ -0,0 +1,34 @@
+name: EDK2 EFI reproducibility
+
+# Rebuilds Ventoy's own EFI binaries (Ventoy/VtoyUtil apps, iso9660/udf EfiFs drivers;
+# x64/ia32/aa64) and fails unless each matches the signed copy in the release byte for
+# byte, apart from the Authenticode signature. Runs against a published release, so it
+# is not triggered by ordinary pushes (new source would not match the old release).
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Release version to check, e.g. 1.1.18 (default: VENTOY_VERSION in INSTALL/grub/grub.cfg)"
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install host tools
+        run: sudo apt-get update -q && sudo apt-get install -y -q mtools
+      - uses: actions/cache@v4
+        with:
+          path: EDK2/ci/cache
+          key: edk2-efi-repro-${{ hashFiles('EDK2/ci/run-all.sh') }}
+      - name: Rebuild and compare
+        # On a release the tagged tree's INSTALL/grub/grub.cfg VENTOY_VERSION is the release version.
+        run: sh EDK2/ci/run-all.sh ${{ inputs.version }}

EDK2/build_efifs_repro.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+# Reproducible build of Ventoy's iso9660/udf EfiFs drivers.
+# Base: pbatard/efifs commit 6e00f7e (GRUB submodule 4dd4ceec0 + efifs 0000-GRUB-fixes.patch),
+# mounted read-only at /efifs, plus Ventoy's three modified files from EDK2/efiffs/mod.
+# Built as EfiFsPkg inside the EDK2 tree at the original path embedded in the shipped drivers.
+# Optional: ARCHS="X64 IA32 AARCH64" (AARCH64 needs aarch64-linux-gnu-gcc on PATH).
+set -e
+ORIG=/home/share/Ventoy/EDKII
+EDK=$ORIG/edk2-edk2-stable201911
+OUT=${OUT:-/ventoy/EDK2/repro_out}
+ARCHS=${ARCHS:-"X64 IA32"}
+
+rm -rf "$ORIG"; mkdir -p "$ORIG" "$OUT"; cd "$ORIG"
+unzip -q /ventoy/EDK2/edk2-edk2-stable201911.zip
+cp -a /ventoy/EDK2/edk2_mod/edk2-edk2-stable201911/. "$EDK"/
+cp -a /efifs "$EDK/EfiFsPkg"
+# PREPATCHED=1: /efifs already has Ventoy's changes applied as a patch (EDK2/ventoy_efifs.patch)
+[ "${PREPATCHED:-0}" = 1 ] || cp -a /ventoy/EDK2/efiffs/mod/. "$EDK/EfiFsPkg/"
+cd "$EDK"
+make -j"$(nproc)" -C BaseTools/ >/tmp/basetools.log 2>&1
+rm -rf ./Conf/.cache ./Conf/.AutoGenIdFile.txt
+unset WORKSPACE; . ./edksetup.sh >/dev/null
+
+for A in $ARCHS; do
+    case $A in X64) p=x64;; IA32) p=ia32;; AARCH64) p=aa64;; esac
+    (cd EfiFsPkg && sh ./set_grub_cpu.sh "$A" >/dev/null)
+    for M in Iso9660 Udf; do
+        GCC48_AARCH64_PREFIX=aarch64-linux-gnu- \
+        build -p EfiFsPkg/EfiFsPkg.dsc -a "$A" -b RELEASE -t GCC48 \
+              -m "EfiFsPkg/EfiFsPkg/$M.inf" >"/tmp/build_${M}_$A.log" 2>&1 \
+            || { echo "BUILD_FAIL $M $A"; tail -8 "/tmp/build_${M}_$A.log"; continue; }
+        lc=$(echo "$M" | tr 'A-Z' 'a-z')
+        cp -f "$(ls Build/EfiFs/RELEASE_GCC48/$A/EfiFsPkg/EfiFsPkg/$M/OUTPUT/*.efi | head -1)" \
+              "$OUT/${lc}_${p}.efi"
+        echo "BUILD_OK ${lc}_${p}.efi"
+    done
+done

EDK2/build_repro_all.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Reproducible build of Ventoy's own EDK2 UEFI apps (Ventoy, VtoyUtil) for X64 and IA32.
+# The EDK2 tree must sit at the ORIGINAL path embedded in the shipped binaries
+# (/home/share/Ventoy/EDKII/...) so the __FILE__ strings match byte for byte.
+# Run inside the repo's CentOS 7 build image with the Ventoy tree mounted at /ventoy.
+# Optional: ARCHS="X64 IA32 AARCH64" (AARCH64 needs aarch64-linux-gnu-gcc on PATH).
+set -e
+ORIG=/home/share/Ventoy/EDKII
+OUT=${OUT:-/ventoy/EDK2/repro_out}
+ARCHS=${ARCHS:-"X64 IA32"}
+
+rm -rf "$ORIG"; mkdir -p "$ORIG" "$OUT"; cd "$ORIG"
+unzip -q /ventoy/EDK2/edk2-edk2-stable201911.zip
+cp -a /ventoy/EDK2/edk2_mod/edk2-edk2-stable201911/. ./edk2-edk2-stable201911/
+cd edk2-edk2-stable201911
+make -j"$(nproc)" -C BaseTools/ >/tmp/basetools.log 2>&1
+rm -rf ./Conf/.cache ./Conf/.AutoGenIdFile.txt
+unset WORKSPACE; . ./edksetup.sh >/dev/null
+
+for A in $ARCHS; do
+    case $A in X64) p=x64;; IA32) p=ia32;; AARCH64) p=aa64;; esac
+    for M in Ventoy VtoyUtil; do
+        GCC48_AARCH64_PREFIX=aarch64-linux-gnu- \
+        build -p MdeModulePkg/MdeModulePkg.dsc -a "$A" -b RELEASE -t GCC48 \
+              -m "MdeModulePkg/Application/$M/$M.inf" >"/tmp/build_${M}_$A.log" 2>&1 \
+            || { echo "BUILD_FAIL $M $A"; tail -5 "/tmp/build_${M}_$A.log"; continue; }
+        lc=$(echo "$M" | tr 'A-Z' 'a-z')
+        cp -f "Build/MdeModule/RELEASE_GCC48/$A/MdeModulePkg/Application/$M/$M/OUTPUT/$M.efi" \
+              "$OUT/${lc}_${p}.efi"
+        echo "BUILD_OK ${lc}_${p}.efi"
+    done
+done

EDK2/ci/.gitignore

@@ -0,0 +1,3 @@
+cache/
+out/
+ref/

EDK2/ci/compare_signed_pe.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Check that an unsigned rebuilt PE/EFI image matches a committed (possibly signed) one.
+
+Usage: compare_signed_pe.py REBUILT COMMITTED
+
+The committed file may carry an appended Authenticode signature. It is cut off at
+the Certificate Table offset, then the two images are compared byte for byte. The
+only differences allowed are the two header fields that signing changes:
+the optional-header CheckSum and the Certificate Table data-directory entry.
+Exit status: 0 = match, 1 = mismatch, 2 = bad input.
+"""
+import struct
+import sys
+
+
+def layout(data):
+    pe, = struct.unpack_from("<I", data, 0x3C)
+    if data[pe:pe + 4] != b"PE\0\0":
+        raise ValueError("not a PE image")
+    opt = pe + 24
+    magic, = struct.unpack_from("<H", data, opt)
+    ddir = opt + (112 if magic == 0x20B else 96)
+    cert = ddir + 4 * 8
+    cert_off, cert_size = struct.unpack_from("<II", data, cert)
+    nsec, = struct.unpack_from("<H", data, pe + 6)
+    opt_size, = struct.unpack_from("<H", data, pe + 20)
+    sections = []
+    base = opt + opt_size
+    for i in range(nsec):
+        name, _vs, _va, rsize, rptr = struct.unpack_from("<8sIIII", data, base + 40 * i)
+        sections.append((name.rstrip(b"\0").decode("latin1"), rptr, rsize))
+    allowed = {**{o: "CheckSum" for o in range(opt + 64, opt + 68)},
+               **{o: "CertificateTable dir" for o in range(cert, cert + 8)}}
+    return sections, allowed, cert_off, cert_size
+
+
+def compare(rebuilt_path, committed_path):
+    rebuilt = open(rebuilt_path, "rb").read()
+    committed = open(committed_path, "rb").read()
+    _, _, r_off, r_size = layout(rebuilt)
+    if r_off or r_size:
+        raise ValueError("rebuilt image is already signed")
+    sections, allowed, c_off, c_size = layout(committed)
+    unsigned = committed[:c_off] if c_off else committed
+    report = {"signature_bytes": c_size, "sections": {}, "header_diffs": [], "ok": True}
+    if len(unsigned) != len(rebuilt):
+        report["ok"] = False
+        report["size"] = (len(rebuilt), len(unsigned))
+        return report
+    in_section = set()
+    for name, ptr, size in sections:
+        a, b = rebuilt[ptr:ptr + size], unsigned[ptr:ptr + size]
+        n = sum(x != y for x, y in zip(a, b))
+        report["sections"][name] = n
+        in_section.update(range(ptr, ptr + size))
+        if n:
+            report["ok"] = False
+    for off, (x, y) in enumerate(zip(rebuilt, unsigned)):
+        if x != y and off not in in_section:
+            field = allowed.get(off)
+            report["header_diffs"].append((off, field))
+            if field is None:
+                report["ok"] = False
+    return report
+
+
+def main():
+    if len(sys.argv) != 3:
+        print(__doc__)
+        return 2
+    try:
+        r = compare(sys.argv[1], sys.argv[2])
+    except (OSError, ValueError, struct.error) as e:
+        print(f"ERROR {e}")
+        return 2
+    secs = " ".join(f"{k}={v}" for k, v in r["sections"].items())
+    fields = sorted({f or f"UNEXPECTED@{o}" for o, f in r["header_diffs"]})
+    print(f"{'MATCH' if r['ok'] else 'MISMATCH'} {sys.argv[2]}")
+    print(f"  signature={r['signature_bytes']}B  differing section bytes: {secs or '-'}")
+    print(f"  header bytes differing: {len(r['header_diffs'])} ({', '.join(fields) or 'none'})")
+    if "size" in r:
+        print(f"  size mismatch: rebuilt={r['size'][0]} committed-unsigned={r['size'][1]}")
+    return 0 if r["ok"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

EDK2/ci/run-all.sh

@@ -0,0 +1,106 @@
+#!/bin/sh
+# Rebuild every EFI binary Ventoy compiles itself (Ventoy/VtoyUtil apps and the
+# iso9660/udf EfiFs drivers, for x64/ia32/aa64) and check that each one matches the
+# copy shipped in a Ventoy release byte for byte, apart from Ventoy's Authenticode
+# signature (see compare_signed_pe.py).
+#
+# Usage: sh EDK2/ci/run-all.sh [VERSION]     (default: VENTOY_VERSION in INSTALL/grub/grub.cfg)
+# Needs on the host: docker, curl, git, patch, python3, xz, mtools (mcopy), sha256sum.
+# Downloads are cached in EDK2/ci/cache and checked against pinned SHA-256s.
+set -eu
+cd "$(dirname "$0")/../.."
+ROOT=$PWD
+CI=$ROOT/EDK2/ci
+CACHE=$CI/cache
+OUT=$CI/out
+REF=$CI/ref
+VER=${1:-$(sed -n 's/.*VENTOY_VERSION="\([^"]*\)".*/\1/p' INSTALL/grub/grub.cfg | head -1)}
+mkdir -p "$CACHE" "$OUT" "$REF"
+
+CENTOS7=centos@sha256:be65f488b7764ad3638f236b7b515b3678369a5124c47b8d32916d6487418ea4
+EDK2_ZIP_SHA256=c6f691aa91afbaab811a369fe729f61d8e5b58bb5ad79a45446c9ee849c1a60b
+LINARO=gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu
+# From ventoy/vtoytoolchain release 1.0; Linaro no longer serves the original download.
+LINARO_SHA256=27f1dc2c491ed61ae8f0d4b0c11de59cd2f7dd9c94761ee7153006fcac1bf9ab
+EFIFS_URL=https://github.com/pbatard/efifs.git
+EFIFS_OLD=6e00f7e   # base of Ventoy's EDK2/efiffs/org; used for the ia32/aa64 drivers
+EFIFS_X64=61494b4   # x64 drivers were built in 2020-04 from efifs master (37e58c9..61494b4)
+
+log() { echo "== $*"; }
+
+fetch() {  # url out sha256
+    if [ -f "$2" ] && echo "$3  $2" | sha256sum -c --quiet - 2>/dev/null; then return; fi
+    curl -fsSL --retry 3 -o "$2" "$1"
+    echo "$3  $2" | sha256sum -c --quiet -
+}
+
+efifs() {  # commit -> $CACHE/efifs-<commit>, with efifs's GRUB patch and Ventoy's patch
+    d=$CACHE/efifs-$1
+    if [ ! -d "$d" ]; then
+        git clone -q "$EFIFS_URL" "$d"
+        git -C "$d" checkout -q "$1"
+        git -C "$d" submodule update -q --init grub gnu-efi
+    fi
+    git -C "$d/grub" checkout -q -- .
+    git -C "$d" checkout -q -- src
+    (cd "$d/grub" && git apply ../0000-GRUB-fixes.patch)
+    (cd "$d" && patch -s -p1 < "$CACHE/ventoy_efifs.patch")
+}
+
+build() {  # extra docker args..., then the script and its env
+    docker run --rm -v "$ROOT":/ventoy -v "$CACHE":/cache:ro -w /ventoy "$@"
+}
+
+log "build image (repo Dockerfile on $CENTOS7)"
+docker pull -q "$CENTOS7" >/dev/null
+docker tag "$CENTOS7" centos:7
+docker build -q -t ventoy-build:ci "$ROOT" >/dev/null
+
+log "inputs"
+fetch https://codeload.github.com/tianocore/edk2/zip/edk2-stable201911 \
+      EDK2/edk2-edk2-stable201911.zip "$EDK2_ZIP_SHA256"
+fetch "https://github.com/ventoy/vtoytoolchain/releases/download/1.0/$LINARO.tar.xz" \
+      "$CACHE/$LINARO.tar.xz" "$LINARO_SHA256"
+[ -d "$CACHE/$LINARO" ] || tar xf "$CACHE/$LINARO.tar.xz" -C "$CACHE"
+(cd EDK2/efiffs && diff -ruN org mod > "$CACHE/ventoy_efifs.patch") || [ $? -eq 1 ]
+efifs "$EFIFS_OLD"
+efifs "$EFIFS_X64"
+
+AA64PATH="PATH=/usr/sbin:/usr/bin:/sbin:/bin:/cache/$LINARO/bin"
+log "build Ventoy/VtoyUtil apps"
+build -e "$AA64PATH" -e ARCHS="X64 IA32 AARCH64" -e OUT=/ventoy/EDK2/ci/out \
+      ventoy-build:ci sh EDK2/build_repro_all.sh
+log "build EfiFs drivers (ia32/aa64 from efifs $EFIFS_OLD)"
+build -v "$CACHE/efifs-$EFIFS_OLD":/efifs:ro -e "$AA64PATH" -e ARCHS="IA32 AARCH64" \
+      -e PREPATCHED=1 -e OUT=/ventoy/EDK2/ci/out ventoy-build:ci sh EDK2/build_efifs_repro.sh
+log "build EfiFs drivers (x64 from efifs $EFIFS_X64)"
+build -v "$CACHE/efifs-$EFIFS_X64":/efifs:ro -e ARCHS=X64 \
+      -e PREPATCHED=1 -e OUT=/ventoy/EDK2/ci/out ventoy-build:ci sh EDK2/build_efifs_repro.sh
+
+log "reference: ventoy-$VER release"
+base=https://github.com/ventoy/Ventoy/releases/download/v$VER
+tgz=ventoy-$VER-linux.tar.gz
+curl -fsSL --retry 3 -o "$CACHE/sha256-$VER.txt" "$base/sha256.txt"
+sum=$(awk -v f="$tgz" '$2==f {print $1}' "$CACHE/sha256-$VER.txt")
+[ -n "$sum" ] || { echo "no checksum for $tgz in sha256.txt"; exit 1; }
+fetch "$base/$tgz" "$CACHE/$tgz" "$sum"
+tar xzf "$CACHE/$tgz" -C "$CACHE" "./ventoy-$VER/ventoy/ventoy.disk.img.xz"
+xz -dc "$CACHE/ventoy-$VER/ventoy/ventoy.disk.img.xz" > "$CACHE/ventoy.disk.img"
+rm -f "$REF"/*.efi
+mcopy -n -i "$CACHE/ventoy.disk.img" '::/ventoy/*.efi' "$REF/"
+
+log "compare"
+fail=0
+for a in x64 ia32 aa64; do
+    for m in ventoy vtoyutil iso9660 udf; do
+        f=${m}_$a.efi
+        if python3 -I "$CI/compare_signed_pe.py" "$OUT/$f" "$REF/$f"; then :; else fail=1; fi
+        if python3 -I "$CI/compare_signed_pe.py" "$OUT/$f" "INSTALL/ventoy/$f" >/dev/null; then
+            echo "  committed INSTALL/ventoy/$f: matches"
+        else
+            echo "  committed INSTALL/ventoy/$f: does NOT match (stale blob?)"
+        fi
+    done
+done
+[ "$fail" = 0 ] && log "ALL MATCH ventoy-$VER" || log "MISMATCH against ventoy-$VER"
+exit "$fail"