Add PyTorch CPU packages

recipes/recipes_native/generic-testing/generic_testing/check_lib.py

@@ -2,6 +2,7 @@
 import os
 from pathlib import Path
 import json
+import re
 import subprocess
 
 
@@ -17,6 +18,47 @@ def run_llvm_readobj(path, *options):
         sys.exit(1)
 
 
+def is_memoryless_function_module(path):
+    with open(path, "rb") as binary:
+        if binary.read(8) != b"\0asm\x01\0\0\0":
+            return False
+        sections = run_llvm_readobj(path, "--sections")
+        types = set(re.findall(r"^\s*Type: (\w+) ", sections, re.MULTILINE))
+        if (not re.search(r"^\s*Name: dylink\.0$", sections, re.MULTILINE)
+                or not types <= {"CUSTOM", "TYPE", "IMPORT", "FUNCTION", "EXPORT", "CODE"}):
+            return False
+        if "IMPORT" not in types:
+            return True
+        match = re.search(r"Type: IMPORT \(0x2\)\s+Size: (\d+)\s+Offset: (\d+)", sections)
+        if match is None:
+            return False
+        size, offset = map(int, match.groups())
+        binary.seek(offset)
+        imports = iter(binary.read(size))
+
+    # LLVM supplies section boundaries; decode only the import descriptions.
+    def read_uleb():
+        value = 0
+        for shift in range(0, 35, 7):
+            byte = next(imports)
+            value |= (byte & 0x7f) << shift
+            if byte < 0x80:
+                return value
+        raise ValueError("Invalid Wasm integer")
+
+    try:
+        for _ in range(read_uleb()):
+            for _ in range(2):  # Skip the module and field names.
+                for _ in range(read_uleb()):
+                    next(imports)
+            if next(imports) != 0:  # Only function imports are width-neutral.
+                return False
+            read_uleb()  # Function type index.
+        return next(imports, None) is None
+    except (StopIteration, ValueError):
+        return False
+
+
 def check_arch_of_lib(path, arch):
     output = run_llvm_readobj(path, "-h")
     # arch is expected to be e.g. "emscripten-wasm32" or "emscripten-wasm64"
@@ -26,6 +68,11 @@ def check_arch_of_lib(path, arch):
         print(f"Error: {path} is not a WebAssembly library")
         sys.exit(1)
 
+    # LLVM defaults to wasm32 for function-only wrappers with no memory width.
+    if (slim_arch == "wasm64" and "Arch: wasm32" in output
+            and is_memoryless_function_module(path)):
+        return
+
     if f"Arch: {slim_arch}" not in output:
         print(f"Error: {path} is not built for architecture {slim_arch}")
         sys.exit(1)
@@ -57,8 +104,7 @@ def check_no_pthread_symbols(path, allow_pthread):
 
     pthread_matches = [
         i for i, line in enumerate(lines)
-        if "pthread" in line.lower()
-        and not line.strip().startswith("File:")
+        if re.search(r"^\s*Name:\s+_*pthread_", line)
     ]
 
     if pthread_matches:

recipes/recipes_native/generic-testing/recipe.yaml

@@ -1,5 +1,5 @@
 context:
-  version: '0.3.2'
+  version: '0.3.3'
   name: generic-testing
 
 package:
@@ -12,23 +12,28 @@ source:
 
 
 build:
-  number: 1
+  number: 0
   noarch: python
   script:  build.sh
   python:
       entry_points:
         - generic-testing = generic_testing.__main__:main
 requirements:
   host:
-    - python
+    - python =${{ python_min }}
   run:
+    - python >=${{ python_min }}
     - llvm-tools
 
 tests:
   - package_contents:
       files:
       - site-packages/generic_testing/__init__.py
       - site-packages/generic_testing/__main__.py
+  - script: python test_check_lib.py
+    files:
+      recipe:
+      - test_check_lib.py
 
 
 about:

recipes/recipes_native/generic-testing/test_check_lib.py

@@ -0,0 +1,93 @@
+import contextlib
+import io
+from pathlib import Path
+import tempfile
+import unittest
+from unittest import mock
+
+from generic_testing.check_lib import (
+    check_arch_of_lib,
+    check_no_pthread_symbols,
+    is_memoryless_function_module,
+    run_llvm_readobj,
+)
+
+
+# Emscripten 6.0.8 shared wrapper: void *entry(void) { return wrapped(); }
+FACADE = bytes.fromhex(
+    "0061736d01000000000f0864796c696e6b2e300104000000000108026000017f"
+    "600000020f0103656e76077772617070656400000303020100071d02115f5f"
+    "7761736d5f63616c6c5f63746f7273000105656e74727900020a090202000b"
+    "040010000b"
+)
+HEADER = b"\0asm\x01\0\0\0"
+
+
+class MemorylessTests(unittest.TestCase):
+    def setUp(self):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.path = Path(directory.name) / "example.so"
+
+    def test_forwarders(self):
+        for result_type in (b"\x7f", b"\x7e"):
+            self.path.write_bytes(FACADE.replace(
+                b"\x60\x00\x01\x7f", b"\x60\x00\x01" + result_type))
+            for arch in ("emscripten-wasm32", "emscripten-wasm64"):
+                with self.subTest(result_type=result_type, arch=arch):
+                    check_arch_of_lib(self.path, arch)
+
+    def test_memory_width_is_still_checked(self):
+        for flags, arch, other in ((0, "wasm32", "wasm64"),
+                                   (4, "wasm64", "wasm32")):
+            imported = b"\x01\x03env\x06memory\x02" + bytes((flags, 1))
+            for memory in (bytes((5, 3, 1, flags, 1)),
+                           bytes((2, len(imported))) + imported):
+                self.path.write_bytes(FACADE[:25] + memory)
+                check_arch_of_lib(self.path, "emscripten-" + arch)
+                with contextlib.redirect_stdout(io.StringIO()):
+                    with self.assertRaises(SystemExit):
+                        check_arch_of_lib(self.path, "emscripten-" + other)
+
+    def test_non_function_imports_are_not_exempt(self):
+        for description in ("020001", "020401", "037e00", "01700001"):
+            payload = b"\x01\x03env\x01x" + bytes.fromhex(description)
+            self.path.write_bytes(
+                FACADE[:25] + bytes((2, len(payload))) + payload)
+            run_llvm_readobj(self.path, "-h")
+            self.assertFalse(is_memoryless_function_module(self.path))
+
+    def test_archives_objects_and_truncated_modules_are_not_exempt(self):
+        for data in (b"!<arch>\n", HEADER, HEADER + b"\x02\x05\x01",
+                     FACADE[:25] + b"\x02\x01\x01", FACADE[:-1]):
+            self.path.write_bytes(data)
+            with contextlib.redirect_stdout(io.StringIO()):
+                try:
+                    self.assertFalse(is_memoryless_function_module(self.path))
+                except SystemExit as error:
+                    self.assertEqual(error.code, 1)
+
+
+class PthreadSymbolTests(unittest.TestCase):
+    def test_unrelated_symbol_names(self):
+        output = "\n".join([
+            "File: /tmp/pthread/example.so",
+            "Name: FLAGS_pthreadpool_size",
+            "Name: _ZN4llvm15JumpThreadingPass3runEv",
+            "Name: pthreadpool_parallelize_1d",
+        ])
+        with mock.patch("generic_testing.check_lib.run_llvm_readobj", return_value=output):
+            check_no_pthread_symbols(Path("example.so"), False)
+
+    def test_actual_pthread_symbols_are_rejected(self):
+        for name in ("pthread_create", "_pthread_join", "__pthread_mutex_lock",
+                     "pthread_create.llvm.123"):
+            with self.subTest(name=name), contextlib.redirect_stdout(io.StringIO()):
+                with mock.patch("generic_testing.check_lib.run_llvm_readobj",
+                                return_value=f"  Name: {name}\n"):
+                    with self.assertRaises(SystemExit):
+                        check_no_pthread_symbols(Path("example.so"), False)
+
+
+if __name__ == "__main__":
+    unittest.main()

recipes/recipes_native/pytester/pytester/empack_config.yaml

@@ -1,10 +1,4 @@
 packages:
-  setuptools:
-    exclude_patterns:
-      - pattern: '**/_distutils_hack/**'
-      - pattern: '**/distutils-precedence.pth'
-      - pattern: '**/setuptools/**'
-      - pattern: '**/pkg_resources/_vendor/**'
   zlib:
     exclude_patterns:
       - pattern: '**/so'

recipes/recipes_native/pytester/recipe.yaml

@@ -1,6 +1,6 @@
 context:
   version: '1.6.1'
-  build_number: 3
+  build_number: 4
 
 source:
   - path: bin

recipes/recipes_wasm/pytorch/build.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+unset DEBUG REL_WITH_DEB_INFO
+export CMAKE_GENERATOR="Unix Makefiles"
+export MAX_JOBS="${CPU_COUNT}"
+export CMAKE_BUILD_PARALLEL_LEVEL="${CPU_COUNT}"
+toolchain_args="${CMAKE_ARGS:-}"
+python_include="${PREFIX}/include/python${PY_VER}"
+numpy_include="${SP_DIR}/numpy/_core/include"
+python_soabi="cpython-${PY_VER//./}-${target_platform#emscripten-}-emscripten"
+export CFLAGS="${CFLAGS:-} -fPIC -fwasm-exceptions -msimd128 -mno-tail-call -sSUPPORT_LONGJMP=wasm"
+export CXXFLAGS="${CXXFLAGS:-} -fPIC -fwasm-exceptions -msimd128 -mno-tail-call -sSUPPORT_LONGJMP=wasm"
+export LDFLAGS="${LDFLAGS:-} -O2 -fwasm-exceptions -msimd128 -mno-tail-call -sSUPPORT_LONGJMP=wasm -Wl,-rpath,\$ORIGIN -Wl,-rpath,\$ORIGIN/lib -Wl,-rpath,/lib"
+export CMAKE_ARGS="${CMAKE_ARGS:-} \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_PLATFORM_NO_VERSIONED_SONAME=ON \
+ -DPython_EXECUTABLE=${PYTHON} \
+ -DPython_INCLUDE_DIR=${python_include} \
+ -DPython_NumPy_INCLUDE_DIR=${numpy_include} \
+ -DPython_SOABI=${python_soabi} \
+ -DPython3_EXECUTABLE=${PYTHON} \
+ -DPython3_INCLUDE_DIR=${python_include} \
+ -DPython3_NumPy_INCLUDE_DIR=${numpy_include} \
+ -DPython3_SOABI=${python_soabi} \
+ -DPYTHON_EXECUTABLE=${PYTHON} -DPYTHON_INCLUDE_DIR=${python_include} \
+ -DCAFFE2_CUSTOM_PROTOC_EXECUTABLE=${BUILD_PREFIX}/bin/protoc \
+ -Dprotobuf_BUILD_PROTOC_BINARIES=OFF -Dprotobuf_BUILD_LIBPROTOC=OFF \
+ -DUSE_EIGEN_SPARSE=OFF -DUSE_SYSTEM_EIGEN_INSTALL=OFF \
+ -DBLAS=OpenBLAS -DBLAS_FORTRAN_STRLEN_END=ON \
+ -DOpenBLAS_INCLUDE_DIR=${PREFIX}/include \
+ -DOpenBLAS_LIB=${PREFIX}/lib/libopenblas.so \
+ -DOPEN_LAPACK_WORKS=ON -DLAPACK_CGESDD_WORKS=ON \
+ -DBUILD_SHARED_LIBS=ON -DBUILD_PYTHON=ON -DBUILD_FUNCTORCH=ON \
+ -DBUILD_TEST=OFF -DBUILD_BINARY=OFF -DUSE_NUMPY=ON \
+ -DUSE_CUDA=OFF -DUSE_ROCM=OFF -DUSE_XPU=OFF -DUSE_MPS=OFF \
+ -DUSE_DISTRIBUTED=OFF -DUSE_TENSORPIPE=OFF -DUSE_GLOO=OFF -DUSE_MPI=OFF \
+ -DUSE_OPENMP=OFF -DUSE_MKLDNN=OFF -DUSE_NNPACK=OFF \
+ -DUSE_PYTORCH_QNNPACK=OFF -DUSE_XNNPACK=OFF -DUSE_FBGEMM=OFF \
+ -DUSE_KINETO=OFF -DUSE_ITT=OFF"
+
+# FindBLAS's no-argument probes do not match the Fortran ABI. The declarations
+# are checked by the patch and real GEMM/LAPACK calls in the browser tests.
+"${PYTHON}" -m pip install . --no-deps --no-build-isolation -v
+
+bridge_dir="${SP_DIR}/torch/_inductor/wasm"
+emcmake cmake ${toolchain_args} -S torch/_inductor/wasm -B build-inductor \
+    -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_PREFIX_PATH="${PREFIX}" -DCMAKE_INSTALL_PREFIX="${bridge_dir}"
+emmake make -C build-inductor -j"${CPU_COUNT}"
+emmake make -C build-inductor install
+
+sysroot="${bridge_dir}/sysroot"
+emscripten_root="${BUILD_PREFIX}/opt/emsdk/upstream/emscripten"
+mkdir -p "${sysroot}/include" "${sysroot}/lib/clang/23"
+cp -a "${emscripten_root}/cache/sysroot/include/." "${sysroot}/include/"
+cp -a "${python_include}" "${sysroot}/include/"
+cp -a "${PREFIX}/lib/clang/23/include" "${sysroot}/lib/clang/23/"
+
+mkdir -p recipe-licenses
+cp "${emscripten_root}/system/lib/libcxx/LICENSE.TXT" recipe-licenses/LLVM.txt
+cp "${PREFIX}/include/llvm/Support/LICENSE.TXT" recipe-licenses/LLVM-Support.txt
+cp "${emscripten_root}/system/lib/libc/musl/COPYRIGHT" recipe-licenses/musl.txt
+cp "${emscripten_root}/LICENSE" recipe-licenses/Emscripten.txt
+
+# Shared C++ libraries/headers are a Python-independent libtorch output.
+mkdir -p "${PREFIX}/lib" "${PREFIX}/include" "${PREFIX}/share"
+for library in libc10.so libshm.so libtorch.so libtorch_cpu.so; do
+    mv "${SP_DIR}/torch/lib/${library}" "${PREFIX}/lib/"
+done
+cp -a "${SP_DIR}/torch/include/." "${PREFIX}/include/"
+cp -a "${SP_DIR}/torch/share/." "${PREFIX}/share/"
+rm -rf "${SP_DIR}/torch/include" "${SP_DIR}/torch/share"

recipes/recipes_wasm/pytorch/cmake_test/CMakeLists.txt

@@ -0,0 +1,12 @@
+cmake_minimum_required(VERSION 4.3.3)
+project(check_libtorch NONE)
+find_package(Torch REQUIRED CONFIG)
+find_package(ATen REQUIRED CONFIG)
+foreach(header IN ITEMS ATen/ATen.h torch/csrc/api/include/torch/torch.h)
+  if(NOT EXISTS "${TORCH_INSTALL_PREFIX}/include/${header}")
+    message(FATAL_ERROR "Missing libtorch header: ${header}")
+  endif()
+endforeach()
+if(NOT EXISTS "${ATEN_INCLUDE_DIR}/ATen/ATen.h")
+  message(FATAL_ERROR "ATen include directory is not relocatable")
+endif()