diff --git a/BinaryBuilderAuditor.jl/Project.toml b/BinaryBuilderAuditor.jl/Project.toml index c893f82..b13fa19 100644 --- a/BinaryBuilderAuditor.jl/Project.toml +++ b/BinaryBuilderAuditor.jl/Project.toml @@ -23,6 +23,9 @@ ObjectFile = "d8793406-e978-5875-9003-1fc021f44a92" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +[sources] +ObjectFile = {rev = "ct/static-combined", url = "https://github.com/topolarity/ObjectFile.jl.git"} + [extras] BinaryBuilderToolchains = "33566f4c-336c-3150-6d30-4374274e6143" JLLPrefixes = "afc68a34-7891-4c5a-9da1-1c62935e7b0d" diff --git a/BinaryBuilderAuditor.jl/src/AuditInfo.jl b/BinaryBuilderAuditor.jl/src/AuditInfo.jl index 17b0e2f..65c5a4f 100644 --- a/BinaryBuilderAuditor.jl/src/AuditInfo.jl +++ b/BinaryBuilderAuditor.jl/src/AuditInfo.jl @@ -3,7 +3,7 @@ export AuditInfo, AuditDependencyInfo struct AuditDependencyInfo # library products of the JLL, for the platform being audited - libs::Vector{JLLLibraryProduct} + libs::Vector{AbstractJLLProduct} end struct AuditLibraryInfo @@ -24,6 +24,7 @@ end function AuditInfo(deps::Dict{Symbol,AuditDependencyInfo}) sonames = Dict{String,AuditLibraryInfo}() for (jll_name, dep) in deps, lib in dep.libs + isa(lib, JLLStaticLibraryProduct) && continue # static archives have no SONAME sonames[basename(lib.soname)] = AuditLibraryInfo(jll_name, lib.varname, lib.path) end return AuditInfo(deps, sonames) diff --git a/BinaryBuilderAuditor.jl/src/AuditResult.jl b/BinaryBuilderAuditor.jl/src/AuditResult.jl index 0cc79b1..8c1e89a 100644 --- a/BinaryBuilderAuditor.jl/src/AuditResult.jl +++ b/BinaryBuilderAuditor.jl/src/AuditResult.jl @@ -19,10 +19,10 @@ struct AuditResult pass_results::Dict{String,Vector{PassResult}} # These contain the learned interdependency structure of the libraries - jll_lib_products::Vector{JLLLibraryProduct} + jll_lib_products::Vector{AbstractJLLProduct} end -AuditResult(scan::ScanResult) = AuditResult(scan, Dict{String,Vector{PassResult}}(), JLLLibraryProduct[]) +AuditResult(scan::ScanResult) = AuditResult(scan, Dict{String,Vector{PassResult}}(), AbstractJLLProduct[]) Base.success(ar::AuditResult) = success(ar.pass_results) function Base.success(pass_results::Dict{String,Vector{PassResult}}) diff --git a/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl b/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl index 2b74337..5ed0b66 100644 --- a/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl +++ b/BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl @@ -13,11 +13,11 @@ include("LdScriptParser.jl") include("passes/RelativeSymlink.jl") include("passes/LibrarySONAME.jl") include("passes/DynamicLinkage.jl") +include("passes/StaticLibraries.jl") include("passes/Licenses.jl") - function audit!(prefix::String, - library_products::Vector{LibraryProduct}, + products::Vector{<:AbstractProduct}, info::AuditInfo; prefix_alias::String = prefix, platform::AbstractPlatform = HostPlatform(), @@ -31,7 +31,7 @@ function audit!(prefix::String, scan = scan_files( prefix, platform, - library_products, + products, env, ) result = AuditResult(scan) @@ -49,6 +49,9 @@ function audit!(prefix::String, # Solve dynamic linkage, deriving each library product's record resolve_dynamic_links!(result, info) + # Describe the static archive of each library product, if any, plus any standalone static libraries + resolve_static_libraries!(result, info) + # Ensure that all libraries and executables have the correct RPATH setup if !readonly rpaths_consistent!(result, info) diff --git a/BinaryBuilderAuditor.jl/src/Scanning.jl b/BinaryBuilderAuditor.jl/src/Scanning.jl index 1d13c63..3481de3 100644 --- a/BinaryBuilderAuditor.jl/src/Scanning.jl +++ b/BinaryBuilderAuditor.jl/src/Scanning.jl @@ -29,9 +29,16 @@ struct ScanResult # and point to the correct `SONAME`. soname_forwards::Dict{String,String} + # List of all static archives contained within the prefix (by `rel_path`) + # Note that on Windows this includes import libraries, which are archives too. + static_libraries::Set{String} + # This maps `rel_path` to a library product `LibraryProduct` library_products::Dict{String,LibraryProduct} + # This maps the `rel_path` of a static archive to its `StaticLibraryProduct`. + static_library_products::Dict{String,StaticLibraryProduct} + # For easy lookup of things by symlink alias symlinks::Dict{String,String} @@ -56,18 +63,41 @@ function get_object_handle(path::String, platform::AbstractPlatform) end end +""" + is_static_archive(path::String) + +Returns `true` if the file at `path` is a static archive (an `ar` archive), +judged by its contents rather than its name. GNU thin archives are not counted, +as they only refer to their members by path, and so cannot be shipped. +""" +function is_static_archive(path::String) + open(path) do io + try + return !isthin(readmeta(io, ArchiveHandle)) + catch e + # Not an archive at all (`MagicMismatch`, `EOFError`), or a malformed one + if isa(e, MagicMismatch) || isa(e, EOFError) || isa(e, ArgumentError) + return false + end + rethrow(e) + end + end +end + function scan_files(prefix::String, platform::AbstractPlatform, - library_products::Vector{LibraryProduct} = LibraryProduct[], + products::Vector{<:AbstractProduct} = AbstractProduct[], env::Dict{String,String} = Dict{String,String}( "prefix" => prefix, "bb_full_target" => triplet(platform), ); prefix_alias::String = prefix) prefix = safe_realpath(prefix) + library_products = LibraryProduct[p for p in products if isa(p, LibraryProduct)] # Do a scan over the prefix, find all symlinks, binary objects, etc.... all_files = Dict{String,StatStruct}() binary_objects = Dict{String,ObjectHandle}() + static_libraries = Set{String}() symlinks = Dict{String,String}() for (root, dirs, files) in walkdir(prefix) for f in files @@ -77,8 +107,9 @@ function scan_files(prefix::String, platform::AbstractPlatform, # `lstat()` now, as we're going to make use of it multiple times all_files[f_key] = lstat(f_path) - # `readmeta()` on all binary objects, but ignore symlinks, we'll - # always resolve to the actual file when dealing with it later on. + # `readmeta()` on all binary objects and identify all static archives, + # but ignore symlinks, we'll always resolve to the actual file when + # dealing with it later on. if !islink(all_files[f_key]) oh = get_object_handle(f_path, platform) if oh !== nothing @@ -86,6 +117,8 @@ function scan_files(prefix::String, platform::AbstractPlatform, throw(ArgumentError("File $(f_key) contains multiple matching object handles")) end binary_objects[f_key] = oh + elseif is_static_archive(f_path) + push!(static_libraries, f_key) end end @@ -150,6 +183,32 @@ function scan_files(prefix::String, platform::AbstractPlatform, library_product_map[relpath_search(symlinks, lib_located_path)] = lib end + # Locate all static library products, each of which must be one of the archives found above + static_product_map = Dict{String,StaticLibraryProduct}() + function locate_static(slp::StaticLibraryProduct) + located_path = locate(slp, prefix; env, platform) + if located_path === nothing + @error("Unable to locate static library", slp, prefix, platform) + error() + end + rel_path = relpath_search(symlinks, located_path) + if rel_path ∉ static_libraries + @error("Static library is not an archive", slp, rel_path, prefix, platform) + error() + end + static_product_map[rel_path] = slp + end + for lib in library_products + if lib.static !== nothing + locate_static(lib.static) + end + end + for slp in products + if isa(slp, StaticLibraryProduct) + locate_static(slp) + end + end + # Get the (memoized) toolchain that holds our third-party tools at, at_prefix = deployed_auditor_toolchain( CrossPlatform(BBHostPlatform() => host_if_crossplatform(platform)), @@ -166,7 +225,9 @@ function scan_files(prefix::String, platform::AbstractPlatform, soname_locator, missing_sonames, soname_forwards, + static_libraries, library_product_map, + static_product_map, symlinks, ) end diff --git a/BinaryBuilderAuditor.jl/src/passes/StaticLibraries.jl b/BinaryBuilderAuditor.jl/src/passes/StaticLibraries.jl new file mode 100644 index 0000000..68e457c --- /dev/null +++ b/BinaryBuilderAuditor.jl/src/passes/StaticLibraries.jl @@ -0,0 +1,84 @@ +using BinaryBuilderProducts, JLLGenerator + +function resolve_static_libraries!(result::AuditResult, info::AuditInfo) + scan, pass_results, products = result.scan, result.pass_results, result.jll_lib_products + if isempty(scan.static_library_products) + return result + end + own_libraries = Set{Symbol}(lib.varname for lib in values(scan.library_products)) + union!(own_libraries, (slp.varname for slp in values(scan.static_library_products))) + + # Unlike their dynamic counterparts, static libraries do not track dependency information + # so the only resolution we can do here is to (1) translate the LibraryDependency's the + # user provided explicitly, or (2) inherit the info from the dynamic audit. + # + # TODO: A proper audit would also verify that undefined symbols can be resolved as expected + # by these dependencies. + dep_string(d::JLLLibraryDep) = d.mod === nothing ? string(d.varname) : string(d.mod, ".", d.varname) + + function resolve(rel_path::String, dep::LibraryDependency) + jll_dep = JLLLibraryDep(dep.pkg, dep.varname) + if dep.pkg === nothing + if dep.varname ∉ own_libraries + push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path, + "Dependency '$(dep.varname)' does not name any library product of this build") + end + else + dep_info = get(info.deps, dep.pkg, nothing) + if dep_info === nothing + push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path, + "Dependency '$(dep_string(jll_dep))' names '$(dep.pkg)', which is not a dependency of this build") + elseif !any(lib -> lib.varname == dep.varname, dep_info.libs) + available = join(sort(unique(string(lib.varname) for lib in dep_info.libs)), ", ") + push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path, + "Dependency '$(dep_string(jll_dep))' does not name a library product of '$(dep.pkg)' (it provides: $(isempty(available) ? "nothing" : available))") + end + end + return jll_dep + end + resolve(rel_path::String, deps::Vector{LibraryDependency}) = unique!(JLLLibraryDep[resolve(rel_path, dep) for dep in deps]) + + dynamic_libs = Dict{Symbol,JLLLibraryProduct}(p.varname => p for p in products if isa(p, JLLLibraryProduct)) + for (rel_path, slp) in scan.static_library_products + shlib = get(dynamic_libs, slp.varname, nothing) + if shlib === nothing + # no corresponding dynamic library - require explicit deps + if slp.deps === nothing || slp.system_deps === nothing + push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path, + "Standalone StaticLibraryProduct '$(slp.varname)' has no dynamic variant to inherit from; declare both `deps` and `system_deps` explicitly") + continue + end + deps = resolve(rel_path, slp.deps) + system_deps = unique(slp.system_deps) + else + if slp.deps === nothing + deps = unique(shlib.deps) # inherit from dynamic library + else + deps = resolve(rel_path, slp.deps) + # warn if the declared deps do not match the dynamic library + note_omissions(rel_path, "dependencies", String[ + d.mod === nothing ? string(d.varname) : "$(d.mod).$(d.varname)" + for d in setdiff(shlib.deps, deps) + ]) + end + if slp.system_deps === nothing + system_deps = unique(shlib.system_deps) # inherit from dynamic library + else + system_deps = unique(slp.system_deps) + # warn if the declared system_deps do not match the dynamic library + note_omissions(rel_path, "system dependencies", setdiff(shlib.system_deps, system_deps)) + end + end + push!(products, JLLStaticLibraryProduct(slp.varname, rel_path; deps, system_deps)) + end + + sort!(products; by = p -> (p.varname, isa(p, JLLStaticLibraryProduct))) + + return result +end + +function note_omissions(rel_path::String, what::String, omitted::Vector{String}) + isempty(omitted) && return nothing + @warn("Declared $(what) of '$(rel_path)' omit $(length(omitted)) inherited from the dynamic variant: $(join(omitted, ", "))") + return nothing +end diff --git a/BinaryBuilderAuditor.jl/test/ScanningTests.jl b/BinaryBuilderAuditor.jl/test/ScanningTests.jl index 9e9cf38..6368adf 100644 --- a/BinaryBuilderAuditor.jl/test/ScanningTests.jl +++ b/BinaryBuilderAuditor.jl/test/ScanningTests.jl @@ -41,3 +41,24 @@ using Test, BinaryBuilderAuditor, ObjectFile, JLLPrefixes, Base.BinaryPlatforms, liblzma_soname = scan.soname_forwards[basename(liblzma_relpath)] @test haskey(scan.binary_objects, scan.soname_locator[liblzma_soname]) end + +@testset "is_static_archive" begin + using BinaryBuilderAuditor: is_static_archive + mktempdir() do dir + for (name, contents, expected) in ( + # An archive without any members is still an archive + ("empty_archive.a", b"!\n", true), + # Thin archives only refer to their members, so cannot be shipped + ("thin.a", b"!\n", false), + ("text.a", b"! but not really", false), + ("short.a", b"!\n", fill(UInt8(' '), 60)), false), + ) + path = joinpath(dir, name) + write(path, contents) + @test is_static_archive(path) == expected + end + end +end diff --git a/BinaryBuilderAuditor.jl/test/passes/StaticLibrariesTests.jl b/BinaryBuilderAuditor.jl/test/passes/StaticLibrariesTests.jl new file mode 100644 index 0000000..c54c4a0 --- /dev/null +++ b/BinaryBuilderAuditor.jl/test/passes/StaticLibrariesTests.jl @@ -0,0 +1,201 @@ +using Test, BinaryBuilderAuditor, BinaryBuilderProducts, JLLGenerator, BinaryBuilderToolchains, Base.BinaryPlatforms +using BinaryBuilderAuditor: resolve_static_libraries! +using JLLGenerator: generate_toml_dict + +# The emitter describes each declared archive from its recipe and its dynamic +# variant; nothing is read from the archive itself. +@testset "static library records" begin + target_platform = Platform("x86_64", "linux") + platform = CrossPlatform(BBHostPlatform() => target_platform) + toolchain = CToolchain(platform; use_ccache=false) + libplus_c_path = joinpath(dirname(@__DIR__), "source", "libplus.c") + libmult_c_path = joinpath(dirname(@__DIR__), "source", "libmult.c") + libplus_soname = versioned_shlib("libplus", 1, target_platform) + libmult_soname = versioned_shlib("libmult", 2, target_platform) + + # Build each library both ways, from the same objects + function build_prefix(prefix) + libdir = joinpath(prefix, "lib") + mkpath(libdir) + with_toolchains([toolchain]) do _, env + ar = get(env, "AR", "ar") + for (name, src, soname, extra) in (("libplus", libplus_c_path, libplus_soname, ``), + ("libmult", libmult_c_path, libmult_soname, `-L $(libdir) -l:$(libplus_soname)`)) + run(setenv(`$(env["CC"]) -o $(joinpath(libdir, soname)) -shared $(src) $(extra) $(soname_flag(target_platform, soname))`, env)) + run(setenv(`$(env["CC"]) -c -o $(joinpath(libdir, "$(name).o")) $(src)`, env)) + run(setenv(`$(ar) rcs $(joinpath(libdir, "$(name).a")) $(joinpath(libdir, "$(name).o"))`, env)) + rm(joinpath(libdir, "$(name).o")) + end + symlink(joinpath(libdir, libplus_soname), joinpath(libdir, "libplus$(dlext(target_platform))")) + end + end + + # Run the dynamic and static passes, returning the filled-in result + function run_passes(prefix, library_products; static_library_products = StaticLibraryProduct[], + info = AuditInfo()) + scan = scan_files(prefix, target_platform, AbstractProduct[library_products..., static_library_products...]) + result = AuditResult(scan) + ensure_sonames!(result) + resolve_dynamic_links!(result, info) + resolve_static_libraries!(result, info) + return result + end + function emit(prefix, library_products; kwargs...) + result = run_passes(prefix, library_products; kwargs...) + @test success(result.pass_results) + return result.jll_lib_products + end + results_for(result, rel_path) = [r for r in get(result.pass_results, "resolve_static_libraries!", PassResult[]) if r.identifier == rel_path] + has_status(result, rel_path, status) = any(r.status == status for r in results_for(result, rel_path)) + messages(result, rel_path) = join([something(r.message, "") for r in results_for(result, rel_path)], "\n") + static_entries(products) = filter(p -> isa(p, JLLStaticLibraryProduct), products) + entry(products, varname) = only(p for p in static_entries(products) if p.varname == varname) + + mktempdir() do prefix + build_prefix(prefix) + + @testset "inherit from the dynamic variant" begin + products = emit(prefix, [ + LibraryProduct("libplus", :libplus; static = StaticLibraryProduct("libplus")), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult")), + ]) + # One library, once per linkage, loadable entry first + @test [(p.varname, isa(p, JLLStaticLibraryProduct)) for p in products] == + [(:libmult, false), (:libmult, true), (:libplus, false), (:libplus, true)] + libplus = entry(products, :libplus) + @test libplus.path == "lib/libplus.a" + @test isempty(libplus.deps) + @test libplus.system_deps == ["c"] + libmult = entry(products, :libmult) + @test libmult.deps == [JLLLibraryDep(nothing, :libplus)] + @test libmult.system_deps == ["c"] + # The dynamic variant's record is untouched by the archive + dynamic = only(p for p in products if isa(p, JLLLibraryProduct) && p.varname == :libmult) + @test dynamic.deps == [JLLLibraryDep(nothing, :libplus)] + @test dynamic.system_deps == ["c"] + # Record shape + d = generate_toml_dict(libmult) + @test d["type"] == "library" && d["linkage"] == "static" && d["path"] == "lib/libmult.a" + @test d["deps"] == ["libplus"] && d["system_deps"] == ["c"] + end + + @testset "explicit lists replace" begin + products = emit(prefix, [ + LibraryProduct("libplus", :libplus; static = StaticLibraryProduct("libplus"; system_deps = ["c", "m"])), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; deps = [LibraryDependency(:libplus)], system_deps = ["c", "pthread"])), + ]) + @test entry(products, :libplus).system_deps == ["c", "m"] + libmult = entry(products, :libmult) + @test libmult.deps == [JLLLibraryDep(nothing, :libplus)] + @test libmult.system_deps == ["c", "pthread"] + end + + @testset "replacement that drops an inherited entry is pointed out" begin + products = @test_logs (:warn, r"system dependencies of 'lib/libmult\.a'.*: c") match_mode=:any emit(prefix, [ + LibraryProduct("libplus", :libplus), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; system_deps = ["m"])), + ]) + # The declaration wins, and the audit still passes + @test entry(products, :libmult).system_deps == ["m"] + @test entry(products, :libmult).deps == [JLLLibraryDep(nothing, :libplus)] + # A dropped dependency edge is pointed out just the same + products = @test_logs (:warn, r"^Declared dependencies of 'lib/libmult\.a'.*: libplus") match_mode=:any emit(prefix, [ + LibraryProduct("libplus", :libplus), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; deps = LibraryDependency[])), + ]) + @test isempty(entry(products, :libmult).deps) + end + + @testset "standalone archives" begin + # A declared edge is recorded as declared: naming a package, or none for + # a library of this JLL + products = emit(prefix, [LibraryProduct("libplus", :libplus)]; + static_library_products = [StaticLibraryProduct("libmult"; varname = :libmult_a, + deps = [LibraryDependency(:libplus), LibraryDependency(:Foo_jll, :libfoo)], system_deps = ["c"])], + info = AuditInfo(Dict(:Foo_jll => AuditDependencyInfo([JLLLibraryProduct(:libfoo, "lib/libfoo.so.1", [], [])])))) + @test [(p.varname, isa(p, JLLStaticLibraryProduct)) for p in products] == [(:libmult_a, true), (:libplus, false)] + libmult_a = entry(products, :libmult_a) + @test libmult_a.path == "lib/libmult.a" + @test libmult_a.deps == [JLLLibraryDep(nothing, :libplus), JLLLibraryDep(:Foo_jll, :libfoo)] + @test libmult_a.system_deps == ["c"] + end + + @testset "standalone archives cannot inherit" begin + scan = scan_files(prefix, target_platform, AbstractProduct[LibraryProduct("libplus", :libplus), + StaticLibraryProduct("libmult"; varname = :libmult_a, deps = [LibraryDependency(:libplus)])]) + result = AuditResult(scan) + ensure_sonames!(result) + resolve_dynamic_links!(result, AuditInfo()) + resolve_static_libraries!(result, AuditInfo()) + @test !success(result.pass_results) + failure = only(result.pass_results["resolve_static_libraries!"]) + @test failure.status == :fail && failure.identifier == "lib/libmult.a" + # ... and no record is written for it + @test isempty(static_entries(result.jll_lib_products)) + end + + @testset "declared edges must name a library that exists" begin + # A library of this build may be a standalone archive + result = run_passes(prefix, [LibraryProduct("libplus", :libplus)]; + static_library_products = [ + StaticLibraryProduct("libplus"; varname = :libplus_a, deps = LibraryDependency[], system_deps = ["c"]), + StaticLibraryProduct("libmult"; varname = :libmult_a, deps = [LibraryDependency(:libplus_a)], system_deps = ["c"]), + ]) + @test !has_status(result, "lib/libmult.a", :fail) + @test entry(result.jll_lib_products, :libmult_a).deps == [JLLLibraryDep(nothing, :libplus_a)] + # A library this build does not provide + result = run_passes(prefix, [ + LibraryProduct("libplus", :libplus), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; deps = [LibraryDependency(:libnope)])), + ]) + @test has_status(result, "lib/libmult.a", :fail) + @test contains(messages(result, "lib/libmult.a"), "does not name any library product of this build") + # The edge is still recorded as declared + @test entry(result.jll_lib_products, :libmult).deps == [JLLLibraryDep(nothing, :libnope)] + # A JLL this build does not depend on, including the JLL being built, whose + # libraries are named without a package + for pkg in (:Nope_jll, :Foo_jll) + result = run_passes(prefix, [ + LibraryProduct("libplus", :libplus), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; deps = [LibraryDependency(pkg, :libplus)])), + ]) + @test has_status(result, "lib/libmult.a", :fail) + @test contains(messages(result, "lib/libmult.a"), "'$(pkg)', which is not a dependency of this build") + end + # A library that dependency does not provide + zlib = AuditInfo(Dict(:Zlib_jll => AuditDependencyInfo([JLLLibraryProduct(:libz, "lib/libz.so.1", [], [])]))) + result = run_passes(prefix, [ + LibraryProduct("libplus", :libplus), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; deps = [LibraryDependency(:Zlib_jll, :libwrong)])), + ]; info = zlib) + @test has_status(result, "lib/libmult.a", :fail) + @test contains(messages(result, "lib/libmult.a"), "does not name a library product of 'Zlib_jll' (it provides: libz)") + # ... and one it does + products = emit(prefix, [ + LibraryProduct("libplus", :libplus), + LibraryProduct("libmult", :libmult; static = StaticLibraryProduct("libmult"; deps = [LibraryDependency(:libplus), LibraryDependency(:Zlib_jll, :libz)])), + ]; info = zlib) + @test entry(products, :libmult).deps == [JLLLibraryDep(nothing, :libplus), JLLLibraryDep(:Zlib_jll, :libz)] + end + + @testset "archives stay out of the dynamic passes" begin + scan = scan_files(prefix, target_platform, [LibraryProduct("libplus", :libplus; static = StaticLibraryProduct("libplus"))]) + @test "lib/libplus.a" ∉ keys(scan.binary_objects) + # Archives are found by their contents, declared or not + @test scan.static_libraries == Set(["lib/libplus.a", "lib/libmult.a"]) + @test collect(keys(scan.static_library_products)) == ["lib/libplus.a"] + # The archive carries its library's name, which is how the two are paired + @test only(values(scan.static_library_products)).varname == :libplus + end + + @testset "a declared archive must exist" begin + @test_throws ErrorException scan_files(prefix, target_platform, [LibraryProduct("libplus", :libplus; static = StaticLibraryProduct("libnope"))]) + end + + @testset "a declared archive must be an archive" begin + write(joinpath(prefix, "lib", "libfake.a"), "not an archive\n") + @test_throws ErrorException scan_files(prefix, target_platform, [StaticLibraryProduct("libfake"; varname = :libfake)]) + rm(joinpath(prefix, "lib", "libfake.a")) + end + end +end diff --git a/BinaryBuilderAuditor.jl/test/runtests.jl b/BinaryBuilderAuditor.jl/test/runtests.jl index b321344..aa887a1 100644 --- a/BinaryBuilderAuditor.jl/test/runtests.jl +++ b/BinaryBuilderAuditor.jl/test/runtests.jl @@ -6,6 +6,7 @@ include("passes/RelativeSymlinkTests.jl") include("passes/LicenseTests.jl") include("passes/LibrarySONAMETests.jl") include("passes/DynamicLinkageTests.jl") +include("passes/StaticLibrariesTests.jl") @testset "audit!" begin platform = CrossPlatform(BBHostPlatform() => HostPlatform()) @@ -42,6 +43,24 @@ include("passes/DynamicLinkageTests.jl") @test length(result.jll_lib_products) == 2 @test success(result) + # A library shipped both ways is described once per linkage + with_toolchains([toolchain]) do _, env + run(setenv(`$(env["CC"]) -c -o $(joinpath(prefix, "lib", "libplus.o")) $(libplus_c_path)`, env)) + run(setenv(`$(get(env, "AR", "ar")) rcs $(joinpath(prefix, "lib", "libplus.a")) $(joinpath(prefix, "lib", "libplus.o"))`, env)) + rm(joinpath(prefix, "lib", "libplus.o")) + end + both_ways = [ + LibraryProduct("libplus", :libplus; static = StaticLibraryProduct("libplus")), + LibraryProduct("libmult", :libmult), + ] + result = audit!(prefix, both_ways, no_deps; readonly=true) + @test success(result) + @test length(result.jll_lib_products) == 3 + libplus_a = only(p for p in result.jll_lib_products if isa(p, JLLStaticLibraryProduct)) + @test libplus_a.varname == :libplus && libplus_a.path == "lib/libplus.a" + @test libplus_a.system_deps == only(p for p in result.jll_lib_products if isa(p, JLLLibraryProduct) && p.varname == :libplus).system_deps + rm(joinpath(prefix, "lib", "libplus.a")) + # Run audit a second time with `readonly=true`, ensure that the treehash does not change pre_treehash = treehash(prefix) result = audit!(prefix, library_products, no_deps; readonly=true) diff --git a/Project.toml b/Project.toml index d3c3b76..c2088dc 100644 --- a/Project.toml +++ b/Project.toml @@ -64,6 +64,7 @@ JLLGenerator = {path = "JLLGenerator.jl"} KeywordArgumentExtraction = {path = "KeywordArgumentExtraction.jl"} LazyJLLWrappers = {path = "LazyJLLWrappers.jl"} MultiHashParsing = {path = "MultiHashParsing.jl"} +ObjectFile = {rev = "ct/static-combined", url = "https://github.com/topolarity/ObjectFile.jl.git"} ScratchSpaceGarbageCollector = {path = "ScratchSpaceGarbageCollector.jl"} TreeArchival = {path = "TreeArchival.jl"} diff --git a/src/BuildCache.jl b/src/BuildCache.jl index 2ea0c67..d08eaef 100644 --- a/src/BuildCache.jl +++ b/src/BuildCache.jl @@ -29,7 +29,7 @@ that are per build, not per extraction. @struct_hash_equal struct BuildCacheExtractEntry artifact::SHA1Hash log_artifact::SHA1Hash - jll_lib_products::Vector{JLLLibraryProduct} + jll_lib_products::Vector{AbstractJLLProduct} end """ @@ -103,7 +103,7 @@ end function Base.put!(bc::BuildCache, build_hash::SHA1Hash, extract_hash::SHA1Hash, build_log_artifact_hash::SHA1Hash, env::Dict{String,String}, - artifact_hash::SHA1Hash, extract_log_artifact_hash::SHA1Hash, jll_lib_products::Vector{JLLLibraryProduct}) + artifact_hash::SHA1Hash, extract_log_artifact_hash::SHA1Hash, jll_lib_products::Vector{<:AbstractJLLProduct}) put!( bc, build_hash, extract_hash, @@ -230,7 +230,7 @@ function import_archives(bc::BuildCache, dir::String) extract_entry = BuildCacheExtractEntry( extract_artifact, extract_log_artifact, - parse_toml_dict.(JLLLibraryProduct, jll_lib_products), + parse_toml_dict.(AbstractJLLProduct, jll_lib_products), ) catch ex @debug("Unable to import BuildCacheExtractEntry", exception=ex, build_hash=bh, extract_hash=eh) @@ -292,7 +292,7 @@ function load_extract_entries!(extract_entries::Dict{SHA1Hash,BuildCacheExtractE extract_entries[extract_hash] = BuildCacheExtractEntry( artifact_hash, log_artifact_hash, - parse_toml_dict.(JLLLibraryProduct, jll_lib_products), + parse_toml_dict.(AbstractJLLProduct, jll_lib_products), ) catch @debug("skip: extract entry malformed", extract_hash=extract_hash_str) diff --git a/src/build_api/ExtractConfig.jl b/src/build_api/ExtractConfig.jl index 52e71ec..1470081 100644 --- a/src/build_api/ExtractConfig.jl +++ b/src/build_api/ExtractConfig.jl @@ -94,6 +94,12 @@ function extract_spec_hash(build_hash::SHA1Hash, extract_script::String, product println(hash_buffer, "[products]") for product in sort(products; by = p->p.varname) println(hash_buffer, " $(product.varname) = $(product.paths)") + archive = isa(product, LibraryProduct) ? product.static : + isa(product, StaticLibraryProduct) ? product : nothing + if archive !== nothing + deps = archive.deps === nothing ? nothing : ["$(d.pkg):$(d.varname)" for d in archive.deps] + println(hash_buffer, " $(product.varname).static = $(archive.paths); deps = $(deps); system_deps = $(archive.system_deps)") + end end # I think we probably don't need to be sensitive to these, since they're only used in packaging? @@ -209,7 +215,7 @@ end platform = host_if_crossplatform(config.platform) # Get libraries for all JLL dependencies - get_library_products(jart::JLLBuildInfo) = filter(x -> isa(x, JLLLibraryProduct), jart.products) + get_library_products(jart::JLLBuildInfo) = filter((x)->isa(x, JLLLibraryProduct) || isa(x, JLLStaticLibraryProduct), jart.products) get_library_products(jll::JLLInfo, platform::AbstractPlatform) = get_library_products(select_platform(jll, platform)) deps = Dict{Symbol,AuditDependencyInfo}() for dep in dep_jll_infos @@ -221,7 +227,7 @@ end end return audit!( artifact_dir, - LibraryProduct[p for p in config.products if isa(p, LibraryProduct)], + config.products, AuditInfo(deps); prefix_alias, env = config.build.env, @@ -232,13 +238,19 @@ end function find_unlocatable_products(config::ExtractConfig, prefix) unlocatable_products = [] - for product in config.products + function check(product) if locate(product, prefix; env=config.build.env, platform=host_if_crossplatform(config.platform)) === nothing push!(unlocatable_products, product) end end + for product in config.products + check(product) + if isa(product, LibraryProduct) && product.static !== nothing + check(product.static) + end + end return unlocatable_products end @@ -248,7 +260,7 @@ end verbose::Bool = AbstractBuildMeta(config).verbose) local artifact_hash, run_status, run_exception, collector audit_result = nothing - jll_lib_products = JLLLibraryProduct[] + jll_lib_products = AbstractJLLProduct[] build_config = BuildConfig(config) meta = AbstractBuildMeta(config) meta.extractions[config] = nothing diff --git a/src/build_api/ExtractResult.jl b/src/build_api/ExtractResult.jl index c9d71ea..db93c13 100644 --- a/src/build_api/ExtractResult.jl +++ b/src/build_api/ExtractResult.jl @@ -22,7 +22,7 @@ struct ExtractResult audit_result::Union{Nothing,AuditResult} # The JLL library product structure, comes from `audit_result`. - jll_lib_products::Vector{JLLLibraryProduct} + jll_lib_products::Vector{AbstractJLLProduct} # Logs generated during this extraction (audit logs, mostly) extract_log::String @@ -33,7 +33,7 @@ struct ExtractResult artifact::Union{Base.SHA1,SHA1Hash,Nothing}, log_artifact::Union{Base.SHA1,SHA1Hash,Nothing}, audit_result::Union{Nothing,AuditResult}, - jll_lib_products::Vector{JLLLibraryProduct}, + jll_lib_products::Vector{<:AbstractJLLProduct}, extract_log::String) return new( config, @@ -49,7 +49,7 @@ struct ExtractResult end AbstractBuildMeta(result::ExtractResult) = AbstractBuildMeta(result.config) -function ExtractResult_cached(config::ExtractConfig, artifact::Union{Base.SHA1,SHA1Hash}, log_artifact::Union{Base.SHA1,SHA1Hash}, jll_lib_products::Vector{JLLLibraryProduct}) +function ExtractResult_cached(config::ExtractConfig, artifact::Union{Base.SHA1,SHA1Hash}, log_artifact::Union{Base.SHA1,SHA1Hash}, jll_lib_products::Vector{<:AbstractJLLProduct}) build_config = config.build.config extract_log = joinpath(artifact_path(build_config.meta.universe, log_artifact), "$(build_config.src_name)-extract.log") return ExtractResult( @@ -72,7 +72,7 @@ function ExtractResult_skipped(config::ExtractConfig) nothing, nothing, nothing, - JLLLibraryProduct[], + AbstractJLLProduct[], "", ) end @@ -128,7 +128,7 @@ function ExtractResultSource(result::ExtractResult, target::String = "") ) end -function export_jll_lib_products(products::Vector{JLLLibraryProduct}, jlp_path::String) +function export_jll_lib_products(products::Vector{<:AbstractJLLProduct}, jlp_path::String) toml_io = IOBuffer() TOML.print(toml_io, Dict("jll_lib_products" => generate_toml_dict.(products))) jll_lib_product_str = String(take!(toml_io)) diff --git a/src/build_api/PackageConfig.jl b/src/build_api/PackageConfig.jl index 6c837af..d65bca2 100644 --- a/src/build_api/PackageConfig.jl +++ b/src/build_api/PackageConfig.jl @@ -218,7 +218,7 @@ function JLLProducts(result::ExtractResult) products = AbstractJLLProduct[] for product in result.config.products # Skip library products, as those were translated by the auditor - if isa(product, LibraryProduct) + if isa(product, LibraryProduct) || isa(product, StaticLibraryProduct) continue end @@ -236,7 +236,7 @@ function JLLProducts(result::ExtractResult) ) end - # Copy over the LibraryProducts that were translated by the auditor + # Copy over the library products that were translated by the auditor append!(products, result.jll_lib_products) return products end diff --git a/test/BuildCacheTests.jl b/test/BuildCacheTests.jl index 5e45fe8..97133d2 100644 --- a/test/BuildCacheTests.jl +++ b/test/BuildCacheTests.jl @@ -35,7 +35,10 @@ using JLLGenerator build_env1 = Dict("1" => "1") build_env2 = Dict("2" => "2") extract1_jlp = [JLLLibraryProduct(:libfoo, "lib/libfoo.1.dylib", [], [], flags = [:RTLD_LAZY, :RTLD_DEEPBIND])] - extract2_jlp = [JLLLibraryProduct(:libfoo, "lib/libfoo.2.dylib", [], [], flags = [:RTLD_LAZY, :RTLD_DEEPBIND])] + extract2_jlp = AbstractJLLProduct[ + JLLLibraryProduct(:libfoo, "lib/libfoo.2.dylib", [], [], flags = [:RTLD_LAZY, :RTLD_DEEPBIND]), + JLLStaticLibraryProduct(:libfoo_a, "lib/libfoo.a"; system_deps = ["m"]), + ] put!(bc, build1_hash, extract1_hash, build_log1_hash, build_env1, artifact1_hash, extract_log1_hash, extract1_jlp ) put!(bc, build2_hash, extract2_hash, build_log2_hash, build_env2, artifact2_hash, extract_log2_hash, extract2_jlp) @@ -70,7 +73,9 @@ using JLLGenerator patchelf_extract_entry = BuildCacheExtractEntry( patchelf_artifact_hash, SHA1Hash(sha1("patchelf_extract_log")), - JLLLibraryProduct[], + # Include a static archive entry so that `load_cache()` exercises the + # linkage-dispatched deserialization of `jll_lib_products`. + AbstractJLLProduct[JLLStaticLibraryProduct(:libpatchelf_a, "lib/libpatchelf.a"; system_deps = ["c"])], ) put!(bc, patchelf_build_hash, patchelf_extract_hash, patchelf_build_entry, patchelf_extract_entry) save_cache(bc) diff --git a/test/ExtractConfigTests.jl b/test/ExtractConfigTests.jl new file mode 100644 index 0000000..d030ce1 --- /dev/null +++ b/test/ExtractConfigTests.jl @@ -0,0 +1,33 @@ +using Test, BinaryBuilder2, SHA, MultiHashParsing +using BinaryBuilder2: extract_spec_hash + +@testset "extraction spec hash" begin + build_hash = SHA1Hash(sha1("build")) + h(products...) = extract_spec_hash(build_hash, "install libfoo", AbstractProduct[products...]) + + # Products without an archive hash as they always have, so that existing cache + # entries stay valid + legacy = SHA1Hash(sha1(join([ + "[extraction_metadata]", + " build_hash = $(build_hash)", + " script_hash = $(SHA1Hash(sha1("install libfoo")))", + "[products]", + " libfoo = [\"libfoo\"]", + "", + ], "\n"))) + @test h(LibraryProduct("libfoo", :libfoo)) == legacy + + # A library's archive counts, and so does what it declares + lib(; kwargs...) = LibraryProduct("libfoo", :libfoo; static = StaticLibraryProduct("libfoo"; kwargs...)) + @test h(lib()) != legacy + @test h(lib()) != h(lib(; deps = LibraryDependency[])) + @test h(lib(; deps = [LibraryDependency(:libbar)])) != h(lib(; deps = [LibraryDependency(:Bar_jll, :libbar)])) + @test h(lib()) != h(lib(; system_deps = ["m"])) + + # ... and a standalone archive's declarations count just the same + standalone(; deps = LibraryDependency[], system_deps = String[]) = + StaticLibraryProduct("libfoo"; varname = :libfoo_a, deps, system_deps) + @test h(standalone()) == h(standalone()) + @test h(standalone()) != h(standalone(; deps = [LibraryDependency(:libbar)])) + @test h(standalone()) != h(standalone(; system_deps = ["m"])) +end diff --git a/test/runtests.jl b/test/runtests.jl index fd137d8..d5d986c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,7 @@ using Test, Random include("MonorepoCompatTests.jl") include("ContentReflectionTests.jl") include("BuildCacheTests.jl") +include("ExtractConfigTests.jl") include("BuildMetaTests.jl") include("UniversesTests.jl") include("BuildAPITests.jl")