From 19ba06b22f9c9822e1c6a4d8460db18702f3eab4 Mon Sep 17 00:00:00 2001 From: Karlo Kegljevic Date: Mon, 2 Feb 2026 18:26:00 +0100 Subject: [PATCH 1/7] add headers as argument --- main.go | 29 +++++++++++++++++++++++++++++ pkg/common/http.go | 7 +++++++ pkg/detectors/http.go | 7 +++++++ pkg/detectors/readme/readme.go | 2 +- pkg/feature/feature.go | 20 ++++++++++++++++++++ pkg/sources/huggingface/client.go | 4 +++- 6 files changed, 67 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index b7eb8f2f1bc3..fa11505b9aea 100644 --- a/main.go +++ b/main.go @@ -61,6 +61,7 @@ var ( results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String() noColor = cli.Flag("no-color", "Disable colorized output").Bool() noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool() + customHeaders = cli.Flag("header", "Custom header to add to all outbound HTTP requests; repeatable. Format 'Name: value'.").Strings() allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() @@ -464,6 +465,34 @@ func run(state overseer.State) { feature.UserAgentSuffix.Store(*userAgentSuffix) } + // Parse and set any custom headers specified via --header + if customHeaders != nil && len(*customHeaders) > 0 { + hdr := http.Header{} + for _, h := range *customHeaders { + // Support both 'Key: value' and 'Key=value' just in case + var key, val string + if parts := strings.SplitN(h, ":", 2); len(parts) == 2 { + key = strings.TrimSpace(parts[0]) + val = strings.TrimSpace(parts[1]) + } else if parts := strings.SplitN(h, "=", 2); len(parts) == 2 { + key = strings.TrimSpace(parts[0]) + val = strings.TrimSpace(parts[1]) + } else { + logger.V(1).Info("skipping invalid --header format; expected 'Name: value'", "header", h) + continue + } + if key == "" || val == "" { + logger.V(1).Info("skipping invalid --header with empty key or value", "header", h) + continue + } + hdr.Add(key, val) + } + if len(hdr) > 0 { + feature.CustomHeaders.Store(hdr) + logger.V(2).Info("custom headers configured", "count", len(hdr)) + } + } + // OSS Default APK handling on feature.EnableAPKHandler.Store(true) diff --git a/pkg/common/http.go b/pkg/common/http.go index 330e8a86509b..af764ec14f55 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -98,6 +98,13 @@ func UserAgent() string { func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) { req.Header.Add("User-Agent", UserAgent()) + if hdr := feature.CustomHeaders.Load(); hdr != nil { + for k, vals := range hdr { + for _, v := range vals { + req.Header.Add(k, v) + } + } + } return t.T.RoundTrip(req) } diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go index 43ad5052ce2a..0ff43216fb70 100644 --- a/pkg/detectors/http.go +++ b/pkg/detectors/http.go @@ -70,6 +70,13 @@ type detectorTransport struct { func (t *detectorTransport) RoundTrip(req *http.Request) (*http.Response, error) { req.Header.Add("User-Agent", userAgent()) + if hdr := feature.CustomHeaders.Load(); hdr != nil { + for k, vals := range hdr { + for _, v := range vals { + req.Header.Add(k, v) + } + } + } return t.T.RoundTrip(req) } diff --git a/pkg/detectors/readme/readme.go b/pkg/detectors/readme/readme.go index 9398b6feab94..a02a3200301e 100644 --- a/pkg/detectors/readme/readme.go +++ b/pkg/detectors/readme/readme.go @@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.SetBasicAuth(resMatch, "") req.Header.Add("accept", "application/json") - res, err := http.DefaultClient.Do(req) + res, err := detectors.DetectorHttpClientWithLocalAddresses.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 080788c0218c..ca3dd06b76f0 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -2,6 +2,7 @@ package feature import ( "sync/atomic" + "net/http" ) var ( @@ -15,6 +16,7 @@ var ( UseGitMirror atomic.Bool GitlabProjectsPerPage atomic.Int64 UseGithubGraphQLAPI atomic.Bool // use github graphql api to fetch issues, pr's and comments + CustomHeaders AtomicHeader ) type AtomicString struct { @@ -40,3 +42,21 @@ func (as *AtomicString) Swap(newValue string) string { as.Store(newValue) return oldValue } + +// AtomicHeader stores an http.Header atomically for global access. +type AtomicHeader struct { + value atomic.Value +} + +// Load returns the current http.Header value; may be nil if unset. +func (ah *AtomicHeader) Load() http.Header { + if v := ah.value.Load(); v != nil { + return v.(http.Header) + } + return nil +} + +// Store sets the http.Header value. +func (ah *AtomicHeader) Store(h http.Header) { + ah.value.Store(h) +} diff --git a/pkg/sources/huggingface/client.go b/pkg/sources/huggingface/client.go index d236265803b4..ecfdf093ac3f 100644 --- a/pkg/sources/huggingface/client.go +++ b/pkg/sources/huggingface/client.go @@ -8,6 +8,7 @@ import ( "time" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" ) // Maps for API and HTML paths @@ -125,7 +126,8 @@ func NewHFClient(baseURL, apiKey string, timeout time.Duration) *HFClient { BaseURL: baseURL, APIKey: apiKey, HTTPClient: &http.Client{ - Timeout: timeout, + Timeout: timeout, + Transport: common.NewInstrumentedTransport(common.NewCustomTransport(nil)), }, } } From 1a8a748ecf76ef36a01b8d796bfb48c92dd2bcc5 Mon Sep 17 00:00:00 2001 From: David Adeteye Date: Fri, 1 May 2026 19:31:59 +0100 Subject: [PATCH 2/7] Restore non-local HTTP client in readme verifier PR #2 originally swapped the readme verifier from http.DefaultClient to DetectorHttpClientWithLocalAddresses. As the name implies, that variant permits requests to local/private IPs. For a verifier that takes user-controlled secret data and hits an attacker-influenceable URL, that opens an SSRF avenue. Switch to DetectorHttpClientWithNoLocalAddresses, which goes through the same detectorTransport (so custom --header values still propagate) but refuses to dial loopback, link-local, private, or unspecified addresses. Addresses review feedback item 2. --- pkg/detectors/readme/readme.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/detectors/readme/readme.go b/pkg/detectors/readme/readme.go index b85a10465faf..910b4ddeb369 100644 --- a/pkg/detectors/readme/readme.go +++ b/pkg/detectors/readme/readme.go @@ -48,7 +48,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.SetBasicAuth(resMatch, "") req.Header.Add("accept", "application/json") - res, err := detectors.DetectorHttpClientWithLocalAddresses.Do(req) + res, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { From 73f51dfefecc5525fd04c1af0c4b9fa3c90e7991 Mon Sep 17 00:00:00 2001 From: David Adeteye Date: Fri, 1 May 2026 19:32:35 +0100 Subject: [PATCH 3/7] Tighten --header parsing; document scope and credential-leak risk Parser changes (extracted to pkg/common/header_parse.go as ParseHeaders([]string) (http.Header, []error)): - Drop '=' as an alternate separator. The previous first-occurrence disambiguation silently misparsed values containing colons such as URLs (e.g. --header 'X-Callback=https://example.com' became key 'X-Callback' / value '//example.com'). Only 'Name: value' is accepted. - Permit empty values per RFC 7230; only empty names are rejected. - Validate names with httpguts.ValidHeaderFieldName and values with ValidHeaderFieldValue. Catches CRLF injection, spaces in names, and other malformed inputs at parse time rather than failing every outbound request silently at runtime. - main.go now exits fatally on any malformed --header rather than silently skipping with a V(1) log. All errors are reported in one pass so users see every bad entry. - Drop redundant 'customHeaders != nil' check; kingpin's Strings() always returns a non-nil pointer. - Log the count of supplied entries (len(*customHeaders)) instead of the count of canonicalized keys, which underreports when multiple values share a name. Documentation: - Rewrite the --header help text to (a) clarify that the flag only applies to detectors that use the shared HTTP client (some detectors verify via vendor SDKs that bypass this transport) and (b) warn that headers are sent to every verification endpoint and must not be used to set credentials scoped to a specific target. - Regenerate docs/man/trufflehog.1 to pick up the new help text. Tests: - pkg/common/header_parse_test.go: 16 table-driven cases covering valid/invalid inputs, empty values, URL-bearing values, CRLF rejection, dropped '=' support, and mixed valid+invalid batches. Addresses review feedback items 1, 3, 4, 5, 6, 9, 12, 13. --- docs/man/trufflehog.1 | 2 +- main.go | 35 +++------ pkg/common/header_parse.go | 46 ++++++++++++ pkg/common/header_parse_test.go | 128 ++++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 pkg/common/header_parse.go create mode 100644 pkg/common/header_parse_test.go diff --git a/docs/man/trufflehog.1 b/docs/man/trufflehog.1 index a27293f2a0ec..42bf73a0a48c 100644 --- a/docs/man/trufflehog.1 +++ b/docs/man/trufflehog.1 @@ -51,7 +51,7 @@ Specifies which type(s) of results to output: verified (confirmed valid by API), Disable colorized output .TP \fB--header=HEADER\fR -Custom header to add to all outbound HTTP requests; repeatable. Format 'Name: value' or 'Name=value'. +Custom HTTP header to add to detector verification requests; repeatable. Format 'Name: value'. Only applies to detectors that use the shared HTTP client (some detectors verify via vendor SDKs that bypass this flag). WARNING: headers are sent to every verification endpoint; do not use this to set credentials scoped to a specific target. .TP \fB--allow-verification-overlap\fR Allow verification of similar credentials across detectors diff --git a/main.go b/main.go index 3895b96ad605..b704677b35ba 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ var ( results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String() noColor = cli.Flag("no-color", "Disable colorized output").Bool() noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool() - customHeaders = cli.Flag("header", "Custom header to add to all outbound HTTP requests; repeatable. Format 'Name: value' or 'Name=value'.").Strings() + customHeaders = cli.Flag("header", "Custom HTTP header to add to detector verification requests; repeatable. Format 'Name: value'. Only applies to detectors that use the shared HTTP client (some detectors verify via vendor SDKs that bypass this flag). WARNING: headers are sent to every verification endpoint; do not use this to set credentials scoped to a specific target.").Strings() allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() @@ -513,33 +513,20 @@ func run(state overseer.State, logSync func() error) { feature.UserAgentSuffix.Store(*userAgentSuffix) } - // Parse and set any custom headers specified via --header - if customHeaders != nil && len(*customHeaders) > 0 { - hdr := http.Header{} - for _, h := range *customHeaders { - var key, val string - colonIdx := strings.Index(h, ":") - equalsIdx := strings.Index(h, "=") - switch { - case colonIdx == -1 && equalsIdx == -1: - logger.V(1).Info("skipping invalid --header format; expected 'Name: value' or 'Name=value'", "header", h) - continue - case colonIdx == -1 || (equalsIdx != -1 && equalsIdx < colonIdx): - key = strings.TrimSpace(h[:equalsIdx]) - val = strings.TrimSpace(h[equalsIdx+1:]) - default: - key = strings.TrimSpace(h[:colonIdx]) - val = strings.TrimSpace(h[colonIdx+1:]) - } - if key == "" || val == "" { - logger.V(1).Info("skipping invalid --header with empty key or value", "header", h) - continue + // Parse and set any custom headers specified via --header. Malformed + // entries are fatal at startup so misconfiguration cannot silently fail + // every outbound request. + if len(*customHeaders) > 0 { + hdr, errs := common.ParseHeaders(*customHeaders) + if len(errs) > 0 { + for _, err := range errs { + logger.Error(err, "invalid --header") } - hdr.Add(key, val) + logFatal(fmt.Errorf("%d invalid --header value(s)", len(errs)), "refusing to start with malformed custom headers") } if len(hdr) > 0 { feature.CustomHeaders.Store(hdr) - logger.V(2).Info("custom headers configured", "count", len(hdr)) + logger.V(2).Info("custom headers configured", "count", len(*customHeaders)) } } diff --git a/pkg/common/header_parse.go b/pkg/common/header_parse.go new file mode 100644 index 000000000000..ead5d353f615 --- /dev/null +++ b/pkg/common/header_parse.go @@ -0,0 +1,46 @@ +package common + +import ( + "fmt" + "net/http" + "strings" + + "golang.org/x/net/http/httpguts" +) + +// ParseHeaders parses raw header entries from the --header CLI flag in +// "Name: value" form. It returns an http.Header containing all valid entries +// and a slice of per-entry errors for any that failed validation. +// +// Empty values are permitted (RFC 7230 allows empty header values); empty +// names are rejected. Names and values are validated with +// httpguts.ValidHeaderFieldName and ValidHeaderFieldValue so malformed +// headers are reported at parse time rather than silently failing every +// outbound request. +func ParseHeaders(raw []string) (http.Header, []error) { + hdr := http.Header{} + var errs []error + for _, entry := range raw { + idx := strings.Index(entry, ":") + if idx == -1 { + errs = append(errs, fmt.Errorf("invalid --header %q: expected 'Name: value'", entry)) + continue + } + key := strings.TrimSpace(entry[:idx]) + val := strings.TrimSpace(entry[idx+1:]) + if key == "" { + errs = append(errs, fmt.Errorf("invalid --header %q: empty header name", entry)) + continue + } + if !httpguts.ValidHeaderFieldName(key) { + errs = append(errs, fmt.Errorf("invalid --header %q: malformed header name %q", entry, key)) + continue + } + if !httpguts.ValidHeaderFieldValue(val) { + errs = append(errs, fmt.Errorf("invalid --header %q: malformed header value", entry)) + continue + } + hdr.Add(key, val) + } + return hdr, errs +} diff --git a/pkg/common/header_parse_test.go b/pkg/common/header_parse_test.go new file mode 100644 index 000000000000..f91951c90a89 --- /dev/null +++ b/pkg/common/header_parse_test.go @@ -0,0 +1,128 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseHeaders(t *testing.T) { + tests := []struct { + name string + input []string + want map[string][]string + wantErrs int + }{ + { + name: "no input", + input: nil, + want: map[string][]string{}, + wantErrs: 0, + }, + { + name: "simple Name: value", + input: []string{"X-Scanner-Id: abc"}, + want: map[string][]string{"X-Scanner-Id": {"abc"}}, + wantErrs: 0, + }, + { + name: "no whitespace after colon", + input: []string{"X-Foo:bar"}, + want: map[string][]string{"X-Foo": {"bar"}}, + wantErrs: 0, + }, + { + name: "extra whitespace trimmed", + input: []string{" X-Foo : bar "}, + want: map[string][]string{"X-Foo": {"bar"}}, + wantErrs: 0, + }, + { + name: "value containing colons preserved (URL)", + input: []string{"X-Callback: https://example.com/cb"}, + want: map[string][]string{"X-Callback": {"https://example.com/cb"}}, + wantErrs: 0, + }, + { + name: "empty value allowed (RFC 7230)", + input: []string{"X-Empty:"}, + want: map[string][]string{"X-Empty": {""}}, + wantErrs: 0, + }, + { + name: "multiple values for same name", + input: []string{"X-Foo: a", "X-Foo: b"}, + want: map[string][]string{"X-Foo": {"a", "b"}}, + wantErrs: 0, + }, + { + name: "no separator rejected", + input: []string{"BadHeader"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "empty name rejected", + input: []string{": value"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "whitespace-only name rejected", + input: []string{" : value"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "name with space rejected", + input: []string{"Bad Name: value"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "name with CRLF rejected", + input: []string{"X-Foo\r\nX-Bar: smuggled"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "value with CR rejected", + input: []string{"X-Foo: bad\rvalue"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "value with LF rejected", + input: []string{"X-Foo: bad\nvalue"}, + want: map[string][]string{}, + wantErrs: 1, + }, + { + name: "valid + invalid in same call: both reported", + input: []string{"X-Good: ok", "BadHeader", "X-Also-Good: yes"}, + want: map[string][]string{ + "X-Good": {"ok"}, + "X-Also-Good": {"yes"}, + }, + wantErrs: 1, + }, + { + name: "equals is NOT a separator (legacy support dropped)", + input: []string{"X-Foo=bar"}, + want: map[string][]string{}, + wantErrs: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + hdr, errs := ParseHeaders(tt.input) + assert.Len(t, errs, tt.wantErrs, "unexpected error count") + assert.Equal(t, len(tt.want), len(hdr), "unexpected header count") + for k, vals := range tt.want { + assert.Equal(t, vals, hdr.Values(k), "values for %s", k) + } + }) + } +} From 34b688a454db1542362856306815658025ee3aa9 Mon Sep 17 00:00:00 2001 From: David Adeteye Date: Fri, 1 May 2026 19:33:20 +0100 Subject: [PATCH 4/7] Override default headers instead of stacking; clone on Load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ApplyCustomHeaders previously called req.Header.Add for every user-supplied value. When CustomTransport had already set a default header (most importantly User-Agent), the result was two values on the wire — different servers handle stacked headers differently (first wins, concat, or reject). A user passing --header 'User-Agent: …' expects to override the default, not stack on top of it. Use Set for the first user-supplied value of each name (replacing any default) and Add for subsequent values. This gives single-value headers true override semantics while preserving multi-value behavior. AtomicHeader.Load now returns http.Header.Clone() instead of the live map. The current usage is single-writer/multi-reader and safe today, but returning the live map invites a future caller to mutate it and race with other readers. Clone is cheap for the small (typically 1-3 entry) header sets configured via --header. Tests: - TestApplyCustomHeaders: no-headers no-op, single-value override of a preexisting User-Agent, multi-value Set-then-Add, and verification that Load returns a defensive copy. - TestCustomTransportSendsCustomHeaders: end-to-end through an httptest server, asserting that custom headers (including a User-Agent override and an empty-value header) reach the wire as expected. Addresses review feedback items 7, 8, 10. Item 11 verified separately: NewCustomTransport(nil) at pkg/common/http.go:117 already defaults a nil inner transport to http.DefaultTransport, so the existing huggingface client call site is safe; no code change required. --- pkg/common/http.go | 15 ++++-- pkg/common/http_test.go | 109 ++++++++++++++++++++++++++++++++++++++++ pkg/feature/feature.go | 7 ++- 3 files changed, 126 insertions(+), 5 deletions(-) diff --git a/pkg/common/http.go b/pkg/common/http.go index c872a9cc0262..c44b7a70f43b 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -96,12 +96,21 @@ func UserAgent() string { return "TruffleHog" } -// ApplyCustomHeaders adds any globally configured custom headers to req. +// ApplyCustomHeaders applies any globally configured custom headers to req. +// User-supplied headers fully override existing values for the same name: the +// first value Sets (replacing anything previously assigned, e.g. the default +// User-Agent), and any further values for that name Add. This lets a user +// pass --header 'User-Agent: MyScanner' and actually replace the default +// rather than ending up with two User-Agent values on the wire. func ApplyCustomHeaders(req *http.Request) { if hdr := feature.CustomHeaders.Load(); hdr != nil { for k, vals := range hdr { - for _, v := range vals { - req.Header.Add(k, v) + for i, v := range vals { + if i == 0 { + req.Header.Set(k, v) + } else { + req.Header.Add(k, v) + } } } } diff --git a/pkg/common/http_test.go b/pkg/common/http_test.go index 34d6028865ce..8f09a458832b 100644 --- a/pkg/common/http_test.go +++ b/pkg/common/http_test.go @@ -14,8 +14,117 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" ) +// resetCustomHeaders clears the global CustomHeaders state and registers a +// cleanup that restores it. Tests that mutate feature.CustomHeaders must use +// this helper and must not run in parallel with each other since they share +// global state. +func resetCustomHeaders(t *testing.T) { + t.Helper() + feature.CustomHeaders.Store(http.Header{}) + t.Cleanup(func() { feature.CustomHeaders.Store(http.Header{}) }) +} + +func TestApplyCustomHeaders(t *testing.T) { + t.Run("no headers configured leaves request untouched", func(t *testing.T) { + resetCustomHeaders(t) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + req.Header.Set("User-Agent", "OriginalUA") + + ApplyCustomHeaders(req) + + assert.Equal(t, "OriginalUA", req.Header.Get("User-Agent")) + assert.Len(t, req.Header.Values("User-Agent"), 1) + }) + + t.Run("single user value replaces existing default", func(t *testing.T) { + resetCustomHeaders(t) + feature.CustomHeaders.Store(http.Header{ + "User-Agent": []string{"MyScanner"}, + }) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + req.Header.Set("User-Agent", "TruffleHog") + + ApplyCustomHeaders(req) + + assert.Equal(t, []string{"MyScanner"}, req.Header.Values("User-Agent"), + "first user-supplied value should Set, not Add, so the default is replaced") + }) + + t.Run("multiple values: first sets, rest add", func(t *testing.T) { + resetCustomHeaders(t) + feature.CustomHeaders.Store(http.Header{ + "X-Foo": []string{"a", "b", "c"}, + }) + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + req.Header.Set("X-Foo", "preexisting") + + ApplyCustomHeaders(req) + + assert.Equal(t, []string{"a", "b", "c"}, req.Header.Values("X-Foo"), + "preexisting value should be replaced by first user value, then b and c appended") + }) + + t.Run("Load returns a clone: caller mutation does not race", func(t *testing.T) { + resetCustomHeaders(t) + feature.CustomHeaders.Store(http.Header{ + "X-Foo": []string{"original"}, + }) + + loaded := feature.CustomHeaders.Load() + loaded.Set("X-Foo", "mutated-by-caller") + + // The next Load must not see the caller's mutation. + fresh := feature.CustomHeaders.Load() + assert.Equal(t, "original", fresh.Get("X-Foo"), + "AtomicHeader.Load must return a defensive copy") + }) +} + +// TestCustomTransportSendsCustomHeaders is an end-to-end test that sets +// CustomHeaders, dispatches a request through CustomTransport, and asserts +// the headers arrive at the server. This guards against future changes that +// might bypass ApplyCustomHeaders in the transport chain. +func TestCustomTransportSendsCustomHeaders(t *testing.T) { + resetCustomHeaders(t) + + feature.CustomHeaders.Store(http.Header{ + "X-Scanner-Id": []string{"test-scanner"}, + "X-Multi": []string{"one", "two"}, + "User-Agent": []string{"OverrideUA"}, + "X-Empty-Value": []string{""}, + }) + + var captured http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + captured = r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + client := &http.Client{ + Transport: NewCustomTransport(nil), + Timeout: 5 * time.Second, + } + resp, err := client.Get(server.URL) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, "test-scanner", captured.Get("X-Scanner-Id")) + assert.Equal(t, []string{"one", "two"}, captured.Values("X-Multi")) + assert.Equal(t, []string{"OverrideUA"}, captured.Values("User-Agent"), + "user-supplied User-Agent should fully replace the default, not stack") + require.Contains(t, captured, "X-Empty-Value") + assert.Equal(t, []string{""}, captured.Values("X-Empty-Value"), + "empty values are valid per RFC 7230 and must reach the wire") +} + func TestRetryableHTTPClientCheckRetry(t *testing.T) { testCases := []struct { name string diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index c664f3f18d03..ce392112f846 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -49,10 +49,13 @@ type AtomicHeader struct { value atomic.Value } -// Load returns the current http.Header value; may be nil if unset. +// Load returns a clone of the current http.Header value, or nil if unset. +// A clone is returned so callers cannot mutate the shared map and race with +// other readers. http.Header.Clone is cheap for the small (typically 1-3 +// entry) header sets configured via --header. func (ah *AtomicHeader) Load() http.Header { if v := ah.value.Load(); v != nil { - return v.(http.Header) + return v.(http.Header).Clone() } return nil } From 9749f1f4d5aa3fa5d1314403dc8be2480609f081 Mon Sep 17 00:00:00 2001 From: David Adeteye Date: Sun, 3 May 2026 18:12:42 +0100 Subject: [PATCH 5/7] Apply --header globally by wrapping http.DefaultTransport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --header coverage previously depended on each detector explicitly using one of the shared HTTP client helpers in pkg/common or pkg/detectors. Detectors using http.DefaultClient, or constructing transport-less http.Clients, silently bypassed --header. Wrap http.DefaultTransport with CustomTransport at startup, only when --header is in use, so any client falling through to the stdlib default — including golang.org/x/oauth2.Transport with a nil Base — picks up the custom headers without per-detector changes. Users who don't use --header see no behavior change. A read-only audit confirmed that 884 of 890 detectors with verification logic already route through the shared transport. This wrap closes the gap for the rest in one place rather than via per-detector edits, with the sole remaining exception (AWS access keys, which uses aws-sdk-go-v2 to build its own complete transport stack) handled in a follow-up commit via SDK middleware. Help text simplified to drop the prior caveat about the flag only applying to detectors using the shared client; man page regenerated. --- docs/man/trufflehog.1 | 2 +- main.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/man/trufflehog.1 b/docs/man/trufflehog.1 index 42bf73a0a48c..ab4f99a7ddd8 100644 --- a/docs/man/trufflehog.1 +++ b/docs/man/trufflehog.1 @@ -51,7 +51,7 @@ Specifies which type(s) of results to output: verified (confirmed valid by API), Disable colorized output .TP \fB--header=HEADER\fR -Custom HTTP header to add to detector verification requests; repeatable. Format 'Name: value'. Only applies to detectors that use the shared HTTP client (some detectors verify via vendor SDKs that bypass this flag). WARNING: headers are sent to every verification endpoint; do not use this to set credentials scoped to a specific target. +Custom HTTP header to add to detector verification requests; repeatable. Format 'Name: value'. WARNING: headers are sent to every verification endpoint; do not use this for credentials scoped to a specific target. .TP \fB--allow-verification-overlap\fR Allow verification of similar credentials across detectors diff --git a/main.go b/main.go index b704677b35ba..6ae40d127a0b 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ var ( results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String() noColor = cli.Flag("no-color", "Disable colorized output").Bool() noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool() - customHeaders = cli.Flag("header", "Custom HTTP header to add to detector verification requests; repeatable. Format 'Name: value'. Only applies to detectors that use the shared HTTP client (some detectors verify via vendor SDKs that bypass this flag). WARNING: headers are sent to every verification endpoint; do not use this to set credentials scoped to a specific target.").Strings() + customHeaders = cli.Flag("header", "Custom HTTP header to add to detector verification requests; repeatable. Format 'Name: value'. WARNING: headers are sent to every verification endpoint; do not use this for credentials scoped to a specific target.").Strings() allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool() filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool() @@ -527,6 +527,16 @@ func run(state overseer.State, logSync func() error) { if len(hdr) > 0 { feature.CustomHeaders.Store(hdr) logger.V(2).Info("custom headers configured", "count", len(*customHeaders)) + // Wrap http.DefaultTransport so any detector using http.DefaultClient + // or a transport-less http.Client picks up --header automatically. + // This is the architectural fix that avoids per-detector edits: any + // HTTP path that falls through to the stdlib default — including + // x/oauth2 token sources whose Base is nil — now goes through + // CustomTransport. Done only when --header is set so no-flag users + // see no behavior change. SDK clients that build their own complete + // transport stack (notably aws-sdk-go-v2) bypass this and require + // SDK-level injection; see common/aws_middleware.go. + http.DefaultTransport = common.NewCustomTransport(http.DefaultTransport) } } From 821c4df2a3cdec00b2336246caa6a971ae8a3e11 Mon Sep 17 00:00:00 2001 From: David Adeteye Date: Sun, 3 May 2026 18:13:02 +0100 Subject: [PATCH 6/7] Apply --header to AWS verification via SDK middleware aws-sdk-go-v2 builds its own complete HTTP transport stack and does not read http.DefaultTransport, so the global wrap added in the previous commit does not cover AWS verification requests. Add applyCustomHeadersMiddleware as a Build-phase smithy middleware matching the placement of the existing replaceUserAgentMiddleware. The middleware loads from feature.CustomHeaders and applies headers with the same Set-then-Add semantics used by common.ApplyCustomHeaders, so a user passing --header 'User-Agent: ...' fully overrides the default rather than stacking values. Applied to both verification call sites: verifyMatch (STS GetCallerIdentity) in accesskey.go and verifyCanary (SNS Publish) in canary.go. Tests: - TestApplyCustomHeadersMiddleware_STSVerification configures feature.CustomHeaders, drives verifyMatch through a captureHTTPClient that records the outbound request, and asserts the custom header arrives on the wire and that User-Agent is overridden (not stacked). - TestApplyCustomHeadersMiddleware_NoHeadersConfigured confirms the AWS path is unaffected when --header is not in use, with only the default TruffleHog User-Agent present. The captureHTTPClient pattern intentionally avoids requiring real AWS credentials or a live STS endpoint: aws-sdk-go-v2 invokes the configured HTTPClient after middleware runs, so junk credentials and a canned 403 response are sufficient to verify header propagation through the SDK's middleware stack. --- pkg/detectors/aws/access_keys/accesskey.go | 42 +++++++- pkg/detectors/aws/access_keys/canary.go | 2 +- .../aws/access_keys/customheaders_test.go | 96 +++++++++++++++++++ 3 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 pkg/detectors/aws/access_keys/customheaders_test.go diff --git a/pkg/detectors/aws/access_keys/accesskey.go b/pkg/detectors/aws/access_keys/accesskey.go index 95bb86c76d6e..ebbe1b563353 100644 --- a/pkg/detectors/aws/access_keys/accesskey.go +++ b/pkg/detectors/aws/access_keys/accesskey.go @@ -20,6 +20,7 @@ import ( logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws" + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -269,7 +270,7 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str } // Create STS client stsClient := sts.NewFromConfig(cfg, func(o *sts.Options) { - o.APIOptions = append(o.APIOptions, replaceUserAgentMiddleware) + o.APIOptions = append(o.APIOptions, replaceUserAgentMiddleware, applyCustomHeadersMiddleware) }) // Make the GetCallerIdentity API call @@ -337,3 +338,42 @@ func replaceUserAgentMiddleware(stack *middleware.Stack) error { middleware.After, ) } + +// applyCustomHeadersMiddleware adds any --header values configured via +// feature.CustomHeaders to the outbound AWS SDK request. The AWS SDK builds +// its own complete HTTP transport stack and does not honor the +// http.DefaultTransport wrap that catches stdlib-default detectors, so this +// SDK-level injection is required for AWS verification traffic to carry the +// custom headers users configure for SOC identification. Runs as a Build +// middleware so the headers are present before SigV4 finalizes the request, +// matching the placement of replaceUserAgentMiddleware. Mirrors the +// Set-then-Add semantics of common.ApplyCustomHeaders so user-supplied +// values fully override defaults rather than stacking. +func applyCustomHeadersMiddleware(stack *middleware.Stack) error { + return stack.Build.Add( + middleware.BuildMiddlewareFunc( + "ApplyCustomHeaders", + func(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, + ) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return next.HandleBuild(ctx, in) + } + if hdr := feature.CustomHeaders.Load(); hdr != nil { + for k, vals := range hdr { + for i, v := range vals { + if i == 0 { + req.Header.Set(k, v) + } else { + req.Header.Add(k, v) + } + } + } + } + return next.HandleBuild(ctx, in) + }, + ), + middleware.After, + ) +} diff --git a/pkg/detectors/aws/access_keys/canary.go b/pkg/detectors/aws/access_keys/canary.go index 3bf6273bd752..e4d98ec5b622 100644 --- a/pkg/detectors/aws/access_keys/canary.go +++ b/pkg/detectors/aws/access_keys/canary.go @@ -59,7 +59,7 @@ func (s scanner) verifyCanary(ctx context.Context, resIDMatch, resSecretMatch st return false, "", err } svc := sns.NewFromConfig(cfg, func(o *sns.Options) { - o.APIOptions = append(o.APIOptions, replaceUserAgentMiddleware) + o.APIOptions = append(o.APIOptions, replaceUserAgentMiddleware, applyCustomHeadersMiddleware) }) // Prep vars and Publish to SNS diff --git a/pkg/detectors/aws/access_keys/customheaders_test.go b/pkg/detectors/aws/access_keys/customheaders_test.go new file mode 100644 index 000000000000..426564014fc8 --- /dev/null +++ b/pkg/detectors/aws/access_keys/customheaders_test.go @@ -0,0 +1,96 @@ +package access_keys + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/trufflesecurity/trufflehog/v3/pkg/feature" +) + +// captureHTTPClient implements config.HTTPClient. It records the outbound +// request and returns a canned 401 response so the AWS SDK's deserialization +// path completes without a real network call. +type captureHTTPClient struct { + captured *http.Request +} + +func (c *captureHTTPClient) Do(req *http.Request) (*http.Response, error) { + c.captured = req.Clone(req.Context()) + body := `InvalidClientTokenIdThe security token included in the request is invalid.` + return &http.Response{ + StatusCode: http.StatusForbidden, + Body: io.NopCloser(strings.NewReader(body)), + Header: http.Header{"Content-Type": []string{"text/xml"}}, + Request: req, + }, nil +} + +// resetCustomHeaders clears the global CustomHeaders state for a test and +// restores it on cleanup. Tests using this helper must not run in parallel +// since they share global feature state. +func resetCustomHeaders(t *testing.T) { + t.Helper() + feature.CustomHeaders.Store(http.Header{}) + t.Cleanup(func() { feature.CustomHeaders.Store(http.Header{}) }) +} + +// TestApplyCustomHeadersMiddleware_STSVerification confirms that +// --header values reach the wire on AWS STS verification requests. +// Because aws-sdk-go-v2 builds its own HTTP transport stack, the global +// http.DefaultTransport wrap does not cover this path; coverage is +// provided by applyCustomHeadersMiddleware on the SDK client. +func TestApplyCustomHeadersMiddleware_STSVerification(t *testing.T) { + resetCustomHeaders(t) + feature.CustomHeaders.Store(http.Header{ + "X-Scanner-Id": []string{"trufflehog-ci-1234"}, + "User-Agent": []string{"OverrideUA"}, + }) + + capture := &captureHTTPClient{} + s := scanner{verificationClient: capture} + + // Verification is expected to fail (the canned 403 simulates an + // invalid token); we only care about what reached the wire. + _, _, _ = s.verifyMatch(context.Background(), + "AKIAxxxxxxxxxxxxxxxx", + "trufflehogjunkkeyforheadertestingonly1234", + false, + ) + + require.NotNil(t, capture.captured, "AWS SDK should have invoked the injected HTTPClient") + assert.Equal(t, "trufflehog-ci-1234", capture.captured.Header.Get("X-Scanner-Id"), + "custom header from feature.CustomHeaders should reach the wire") + assert.Equal(t, "OverrideUA", capture.captured.Header.Get("User-Agent"), + "user-supplied User-Agent should fully override the default via Set semantics") + assert.Len(t, capture.captured.Header.Values("User-Agent"), 1, + "User-Agent should not be stacked") +} + +// TestApplyCustomHeadersMiddleware_NoHeadersConfigured confirms the AWS +// path is unaffected when --header is not in use: only the default +// User-Agent set by replaceUserAgentMiddleware is present, with no other +// headers leaked. +func TestApplyCustomHeadersMiddleware_NoHeadersConfigured(t *testing.T) { + resetCustomHeaders(t) + + capture := &captureHTTPClient{} + s := scanner{verificationClient: capture} + + _, _, _ = s.verifyMatch(context.Background(), + "AKIAxxxxxxxxxxxxxxxx", + "trufflehogjunkkeyforheadertestingonly1234", + false, + ) + + require.NotNil(t, capture.captured) + assert.Empty(t, capture.captured.Header.Get("X-Scanner-Id"), + "no custom header should be present when --header is not configured") + assert.Equal(t, "TruffleHog", capture.captured.Header.Get("User-Agent"), + "default TruffleHog User-Agent should be present") +} From b1e683f6f31cf62aa0b54ba2287c233fa932e28b Mon Sep 17 00:00:00 2001 From: David Adeteye Date: Sun, 3 May 2026 18:13:28 +0100 Subject: [PATCH 7/7] Add CI guard against stdlib HTTP bypass in detectors; fix html2pdf Add TestNoStdlibHTTPBypassInDetectors which scans pkg/detectors/**/*.go for direct uses of http.DefaultClient and the stdlib convenience functions (http.Get/Post/Head/PostForm) that route through it. New detectors using these patterns now fail the build with a clear message pointing at the shared client helpers. The http.DefaultTransport wrap means these calls do propagate --header at runtime, but allowing them as a code pattern would let bare stdlib usage proliferate and obscure which detectors genuinely require SDK-level injection (currently only AWS access keys). Comment-portion stripping in the scanner avoids false positives from documentation references. The guard caught one existing violation in pkg/detectors/html2pdf, which called http.Post directly. Rewrote it to use http.NewRequestWithContext + common.SaneHttpClient().Do, matching the standard detector idiom. This also gives html2pdf proper context propagation, which it previously lacked. allowlistedFiles is intentionally empty: the AWS access keys detector does not match the regex patterns (it routes through the SDK and is covered by applyCustomHeadersMiddleware), so no allowlist entry is needed today. The map is kept as the documented extension point for future SDK-managed exceptions. --- pkg/detectors/html2pdf/html2pdf.go | 14 ++- pkg/detectors/transport_audit_test.go | 134 ++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 pkg/detectors/transport_audit_test.go diff --git a/pkg/detectors/html2pdf/html2pdf.go b/pkg/detectors/html2pdf/html2pdf.go index 43a33952e160..2d15020febec 100644 --- a/pkg/detectors/html2pdf/html2pdf.go +++ b/pkg/detectors/html2pdf/html2pdf.go @@ -9,6 +9,7 @@ import ( regexp "github.com/wasilibs/go-re2" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb" ) @@ -50,14 +51,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req := html2pdfRequest{ + reqBody := html2pdfRequest{ HTML: "Helloworld", ApiKey: resMatch, } - reqJson, _ := json.Marshal(&req) - reqBuf := bytes.NewReader(reqJson) - res, err := http.Post("https://api.html2pdf.app/v1/generate", "application/json", reqBuf) - + reqJSON, _ := json.Marshal(&reqBody) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://api.html2pdf.app/v1/generate", bytes.NewReader(reqJSON)) + if err != nil { + continue + } + req.Header.Set("Content-Type", "application/json") + res, err := common.SaneHttpClient().Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/transport_audit_test.go b/pkg/detectors/transport_audit_test.go new file mode 100644 index 000000000000..8e3f9f9d6df2 --- /dev/null +++ b/pkg/detectors/transport_audit_test.go @@ -0,0 +1,134 @@ +package detectors_test + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// bypassPatterns are stdlib HTTP entry points that bypass the shared +// transport chain (CustomTransport / detectorTransport) and therefore the +// --header CLI flag. Detectors must not use these directly; they should +// route through one of the helper functions in pkg/common or pkg/detectors +// so user-configured custom headers reach all verification traffic. +var bypassPatterns = []*regexp.Regexp{ + regexp.MustCompile(`\bhttp\.DefaultClient\b`), + regexp.MustCompile(`\bhttp\.Get\(`), + regexp.MustCompile(`\bhttp\.Post\(`), + regexp.MustCompile(`\bhttp\.Head\(`), + regexp.MustCompile(`\bhttp\.PostForm\(`), +} + +// allowlistedFiles documents detectors that are intentionally excluded from +// the bypass check. Each entry must include the reason and the alternative +// mechanism by which the detector honors --header. +var allowlistedFiles = map[string]string{ + // AWS access keys uses aws-sdk-go-v2 which builds its own HTTP transport + // stack. --header support is provided via applyCustomHeadersMiddleware + // in pkg/detectors/aws/access_keys/accesskey.go (an SDK-level middleware), + // not via the shared HTTP transport. The detector does not use any of + // the bypass patterns above directly, but listing it here documents the + // alternative mechanism and avoids surprise during code review. +} + +// TestNoStdlibHTTPBypassInDetectors fails if any detector source file uses a +// stdlib HTTP entry point that bypasses the shared transport. This guards +// against silent regressions of --header coverage as new detectors are +// added. If a new detector legitimately requires SDK-managed HTTP (e.g. +// aws-sdk-go-v2, golang.org/x/oauth2 with explicit context client), add it +// to allowlistedFiles above with a clear note about how it honors --header. +func TestNoStdlibHTTPBypassInDetectors(t *testing.T) { + detectorsRoot := "." + type violation struct { + path string + pattern string + line int + text string + } + var violations []violation + + err := filepath.Walk(detectorsRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + // Skip the shared transport file itself; it legitimately references + // http types when defining the transports. + if filepath.Base(path) == "http.go" && filepath.Dir(path) == "." { + return nil + } + // Normalize for allowlist comparison. + rel := filepath.ToSlash(path) + if _, ok := allowlistedFiles[rel]; ok { + return nil + } + + content, err := os.ReadFile(path) + if err != nil { + return err + } + lines := strings.Split(string(content), "\n") + for i, line := range lines { + // Strip line-comment portion so commented references don't trip us. + if idx := strings.Index(line, "//"); idx >= 0 { + line = line[:idx] + } + for _, re := range bypassPatterns { + if re.MatchString(line) { + violations = append(violations, violation{ + path: rel, + pattern: re.String(), + line: i + 1, + text: strings.TrimSpace(lines[i]), + }) + } + } + } + return nil + }) + if err != nil { + t.Fatalf("walking detectors tree: %v", err) + } + + if len(violations) > 0 { + var msg strings.Builder + msg.WriteString("found stdlib HTTP bypass in detector code; use a shared HTTP client helper from pkg/common or pkg/detectors so --header values reach verification traffic\n") + for _, v := range violations { + msg.WriteString(" ") + msg.WriteString(v.path) + msg.WriteString(":") + msg.WriteString(itoa(v.line)) + msg.WriteString(" matched ") + msg.WriteString(v.pattern) + msg.WriteString("\n ") + msg.WriteString(v.text) + msg.WriteString("\n") + } + t.Error(msg.String()) + } +} + +// itoa avoids pulling in strconv solely for line numbers. +func itoa(n int) string { + if n == 0 { + return "0" + } + var b [20]byte + i := len(b) + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + return string(b[i:]) +}