Add basic auditor / pipeline support for static libraries

BinaryBuilderAuditor.jl/Project.toml

@@ -23,6 +23,9 @@ ObjectFile = "d8793406-e978-5875-9003-1fc021f44a92"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b"
 
+[sources]
+ObjectFile = {rev = "ct/static-combined", url = "https://github.com/topolarity/ObjectFile.jl.git"}
+
 [extras]
 BinaryBuilderToolchains = "33566f4c-336c-3150-6d30-4374274e6143"
 JLLPrefixes = "afc68a34-7891-4c5a-9da1-1c62935e7b0d"

BinaryBuilderAuditor.jl/src/AuditInfo.jl

@@ -3,7 +3,7 @@ export AuditInfo, AuditDependencyInfo
 
 struct AuditDependencyInfo
     # library products of the JLL, for the platform being audited
-    libs::Vector{JLLLibraryProduct}
+    libs::Vector{AbstractJLLProduct}
 end
 
 struct AuditLibraryInfo
@@ -24,6 +24,7 @@ end
 function AuditInfo(deps::Dict{Symbol,AuditDependencyInfo})
     sonames = Dict{String,AuditLibraryInfo}()
     for (jll_name, dep) in deps, lib in dep.libs
+        isa(lib, JLLStaticLibraryProduct) && continue # static archives have no SONAME
         sonames[basename(lib.soname)] = AuditLibraryInfo(jll_name, lib.varname, lib.path)
     end
     return AuditInfo(deps, sonames)

BinaryBuilderAuditor.jl/src/AuditResult.jl

@@ -19,10 +19,10 @@ struct AuditResult
     pass_results::Dict{String,Vector{PassResult}}
 
     # These contain the learned interdependency structure of the libraries
-    jll_lib_products::Vector{JLLLibraryProduct}
+    jll_lib_products::Vector{AbstractJLLProduct}
 end
 
-AuditResult(scan::ScanResult) = AuditResult(scan, Dict{String,Vector{PassResult}}(), JLLLibraryProduct[])
+AuditResult(scan::ScanResult) = AuditResult(scan, Dict{String,Vector{PassResult}}(), AbstractJLLProduct[])
 
 Base.success(ar::AuditResult) = success(ar.pass_results)
 function Base.success(pass_results::Dict{String,Vector{PassResult}})

BinaryBuilderAuditor.jl/src/BinaryBuilderAuditor.jl

@@ -13,11 +13,11 @@ include("LdScriptParser.jl")
 include("passes/RelativeSymlink.jl")
 include("passes/LibrarySONAME.jl")
 include("passes/DynamicLinkage.jl")
+include("passes/StaticLibraries.jl")
 include("passes/Licenses.jl")
 
-
 function audit!(prefix::String,
-                library_products::Vector{LibraryProduct},
+                products::Vector{<:AbstractProduct},
                 info::AuditInfo;
                 prefix_alias::String = prefix,
                 platform::AbstractPlatform = HostPlatform(),
@@ -31,7 +31,7 @@ function audit!(prefix::String,
     scan = scan_files(
         prefix,
         platform,
-        library_products,
+        products,
         env,
     )
     result = AuditResult(scan)
@@ -49,6 +49,9 @@ function audit!(prefix::String,
     # Solve dynamic linkage, deriving each library product's record
     resolve_dynamic_links!(result, info)
 
+    # Describe the static archive of each library product, if any, plus any standalone static libraries
+    resolve_static_libraries!(result, info)
+
     # Ensure that all libraries and executables have the correct RPATH setup
     if !readonly
         rpaths_consistent!(result, info)

BinaryBuilderAuditor.jl/src/Scanning.jl

@@ -29,9 +29,16 @@ struct ScanResult
     # and point to the correct `SONAME`.
     soname_forwards::Dict{String,String}
 
+    # List of all static archives contained within the prefix (by `rel_path`)
+    # Note that on Windows this includes import libraries, which are archives too.
+    static_libraries::Set{String}
+
     # This maps `rel_path` to a library product `LibraryProduct`
     library_products::Dict{String,LibraryProduct}
 
+    # This maps the `rel_path` of a static archive to its `StaticLibraryProduct`.
+    static_library_products::Dict{String,StaticLibraryProduct}
+
     # For easy lookup of things by symlink alias
     symlinks::Dict{String,String}
 
@@ -56,18 +63,41 @@ function get_object_handle(path::String, platform::AbstractPlatform)
     end
 end
 
+"""
+    is_static_archive(path::String)
+
+Returns `true` if the file at `path` is a static archive (an `ar` archive),
+judged by its contents rather than its name.  GNU thin archives are not counted,
+as they only refer to their members by path, and so cannot be shipped.
+"""
+function is_static_archive(path::String)
+    open(path) do io
+        try
+            return !isthin(readmeta(io, ArchiveHandle))
+        catch e
+            # Not an archive at all (`MagicMismatch`, `EOFError`), or a malformed one
+            if isa(e, MagicMismatch) || isa(e, EOFError) || isa(e, ArgumentError)
+                return false
+            end
+            rethrow(e)
+        end
+    end
+end
+
 function scan_files(prefix::String, platform::AbstractPlatform,
-                    library_products::Vector{LibraryProduct} = LibraryProduct[],
+                    products::Vector{<:AbstractProduct} = AbstractProduct[],
                     env::Dict{String,String} = Dict{String,String}(
                         "prefix" => prefix,
                         "bb_full_target" => triplet(platform),
                     );
                     prefix_alias::String = prefix)
     prefix = safe_realpath(prefix)
+    library_products = LibraryProduct[p for p in products if isa(p, LibraryProduct)]
 
     # Do a scan over the prefix, find all symlinks, binary objects, etc....
     all_files = Dict{String,StatStruct}()
     binary_objects = Dict{String,ObjectHandle}()
+    static_libraries = Set{String}()
     symlinks = Dict{String,String}()
     for (root, dirs, files) in walkdir(prefix)
         for f in files
@@ -77,15 +107,18 @@ function scan_files(prefix::String, platform::AbstractPlatform,
             # `lstat()` now, as we're going to make use of it multiple times
             all_files[f_key] = lstat(f_path)
 
-            # `readmeta()` on all binary objects, but ignore symlinks, we'll
-            # always resolve to the actual file when dealing with it later on.
+            # `readmeta()` on all binary objects and identify all static archives,
+            # but ignore symlinks, we'll always resolve to the actual file when
+            # dealing with it later on.
             if !islink(all_files[f_key])
                 oh = get_object_handle(f_path, platform)
                 if oh !== nothing
                     if haskey(binary_objects, f_key)
                         throw(ArgumentError("File $(f_key) contains multiple matching object handles"))
                     end
                     binary_objects[f_key] = oh
+                elseif is_static_archive(f_path)
+                    push!(static_libraries, f_key)
                 end
             end
 
@@ -150,6 +183,32 @@ function scan_files(prefix::String, platform::AbstractPlatform,
         library_product_map[relpath_search(symlinks, lib_located_path)] = lib
     end
 
+    # Locate all static library products, each of which must be one of the archives found above
+    static_product_map = Dict{String,StaticLibraryProduct}()
+    function locate_static(slp::StaticLibraryProduct)
+        located_path = locate(slp, prefix; env, platform)
+        if located_path === nothing
+            @error("Unable to locate static library", slp, prefix, platform)
+            error()
+        end
+        rel_path = relpath_search(symlinks, located_path)
+        if rel_path ∉ static_libraries
+            @error("Static library is not an archive", slp, rel_path, prefix, platform)
+            error()
+        end
+        static_product_map[rel_path] = slp
+    end
+    for lib in library_products
+        if lib.static !== nothing
+            locate_static(lib.static)
+        end
+    end
+    for slp in products
+        if isa(slp, StaticLibraryProduct)
+            locate_static(slp)
+        end
+    end
+
     # Get the (memoized) toolchain that holds our third-party tools
     at, at_prefix = deployed_auditor_toolchain(
         CrossPlatform(BBHostPlatform() => host_if_crossplatform(platform)),
@@ -166,7 +225,9 @@ function scan_files(prefix::String, platform::AbstractPlatform,
         soname_locator,
         missing_sonames,
         soname_forwards,
+        static_libraries,
         library_product_map,
+        static_product_map,
         symlinks,
     )
 end

BinaryBuilderAuditor.jl/src/passes/StaticLibraries.jl

@@ -0,0 +1,84 @@
+using BinaryBuilderProducts, JLLGenerator
+
+function resolve_static_libraries!(result::AuditResult, info::AuditInfo)
+    scan, pass_results, products = result.scan, result.pass_results, result.jll_lib_products
+    if isempty(scan.static_library_products)
+        return result
+    end
+    own_libraries = Set{Symbol}(lib.varname for lib in values(scan.library_products))
+    union!(own_libraries, (slp.varname for slp in values(scan.static_library_products)))
+
+    # Unlike their dynamic counterparts, static libraries do not track dependency information
+    # so the only resolution we can do here is to (1) translate the LibraryDependency's the
+    # user provided explicitly, or (2) inherit the info from the dynamic audit.
+    #
+    # TODO: A proper audit would also verify that undefined symbols can be resolved as expected
+    # by these dependencies.
+    dep_string(d::JLLLibraryDep) = d.mod === nothing ? string(d.varname) : string(d.mod, ".", d.varname)
+
+    function resolve(rel_path::String, dep::LibraryDependency)
+        jll_dep = JLLLibraryDep(dep.pkg, dep.varname)
+        if dep.pkg === nothing
+            if dep.varname ∉ own_libraries
+                push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path,
+                             "Dependency '$(dep.varname)' does not name any library product of this build")
+            end
+        else
+            dep_info = get(info.deps, dep.pkg, nothing)
+            if dep_info === nothing
+                push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path,
+                             "Dependency '$(dep_string(jll_dep))' names '$(dep.pkg)', which is not a dependency of this build")
+            elseif !any(lib -> lib.varname == dep.varname, dep_info.libs)
+                available = join(sort(unique(string(lib.varname) for lib in dep_info.libs)), ", ")
+                push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path,
+                             "Dependency '$(dep_string(jll_dep))' does not name a library product of '$(dep.pkg)' (it provides: $(isempty(available) ? "nothing" : available))")
+            end
+        end
+        return jll_dep
+    end
+    resolve(rel_path::String, deps::Vector{LibraryDependency}) = unique!(JLLLibraryDep[resolve(rel_path, dep) for dep in deps])
+
+    dynamic_libs = Dict{Symbol,JLLLibraryProduct}(p.varname => p for p in products if isa(p, JLLLibraryProduct))
+    for (rel_path, slp) in scan.static_library_products
+        shlib = get(dynamic_libs, slp.varname, nothing)
+        if shlib === nothing
+            # no corresponding dynamic library - require explicit deps
+            if slp.deps === nothing || slp.system_deps === nothing
+                push_result!(pass_results, "resolve_static_libraries!", :fail, rel_path,
+                             "Standalone StaticLibraryProduct '$(slp.varname)' has no dynamic variant to inherit from; declare both `deps` and `system_deps` explicitly")
+                continue
+            end
+            deps = resolve(rel_path, slp.deps)
+            system_deps = unique(slp.system_deps)
+        else
+            if slp.deps === nothing
+                deps = unique(shlib.deps) # inherit from dynamic library
+            else
+                deps = resolve(rel_path, slp.deps)
+                # warn if the declared deps do not match the dynamic library
+                note_omissions(rel_path, "dependencies", String[
+                    d.mod === nothing ? string(d.varname) : "$(d.mod).$(d.varname)"
+                    for d in setdiff(shlib.deps, deps)
+                ])
+            end
+            if slp.system_deps === nothing
+                system_deps = unique(shlib.system_deps) # inherit from dynamic library
+            else
+                system_deps = unique(slp.system_deps)
+                # warn if the declared system_deps do not match the dynamic library
+                note_omissions(rel_path, "system dependencies", setdiff(shlib.system_deps, system_deps))
+            end
+        end
+        push!(products, JLLStaticLibraryProduct(slp.varname, rel_path; deps, system_deps))
+    end
+
+    sort!(products; by = p -> (p.varname, isa(p, JLLStaticLibraryProduct)))
+
+    return result
+end
+
+function note_omissions(rel_path::String, what::String, omitted::Vector{String})
+    isempty(omitted) && return nothing
+    @warn("Declared $(what) of '$(rel_path)' omit $(length(omitted)) inherited from the dynamic variant: $(join(omitted, ", "))")
+    return nothing
+end

BinaryBuilderAuditor.jl/test/ScanningTests.jl

@@ -41,3 +41,24 @@ using Test, BinaryBuilderAuditor, ObjectFile, JLLPrefixes, Base.BinaryPlatforms,
     liblzma_soname = scan.soname_forwards[basename(liblzma_relpath)]
     @test haskey(scan.binary_objects, scan.soname_locator[liblzma_soname])
 end
+
+@testset "is_static_archive" begin
+    using BinaryBuilderAuditor: is_static_archive
+    mktempdir() do dir
+        for (name, contents, expected) in (
+                # An archive without any members is still an archive
+                ("empty_archive.a", b"!<arch>\n", true),
+                # Thin archives only refer to their members, so cannot be shipped
+                ("thin.a", b"!<thin>\n", false),
+                ("text.a", b"!<arch> but not really", false),
+                ("short.a", b"!<a", false),
+                ("empty.a", UInt8[], false),
+                # A malformed member header
+                ("malformed.a", vcat(b"!<arch>\n", fill(UInt8(' '), 60)), false),
+            )
+            path = joinpath(dir, name)
+            write(path, contents)
+            @test is_static_archive(path) == expected
+        end
+    end
+end