From d9a72dfff172267c8eb248ee86d62f890c544618 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 09:30:57 -0500 Subject: [PATCH 01/19] Support continuous profiling with declarative configuration --- README.md | 7 +- docs/profiling.md | 30 +++++++- src/splunk_otel/configurator.py | 5 +- src/splunk_otel/profiling_config.py | 37 ++++++++++ tests/integration/declarative_profile.py | 89 ++++++++++++++++++++++++ tests/test_profiling_config.py | 81 +++++++++++++++++++++ 6 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 src/splunk_otel/profiling_config.py create mode 100644 tests/integration/declarative_profile.py create mode 100644 tests/test_profiling_config.py diff --git a/README.md b/README.md index b61d2961..a10af0df 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ which exports traces, metrics, and logs to a local Collector at `http://localhos This support configures upstream OpenTelemetry SDK components. Configure exporters, including their endpoints and authentication headers, in the YAML file. -Support for the GDI `distribution.splunk` configuration block is limited to -`instrumentations.http.trace_response_header_enabled`, which defaults to `true`. +The GDI `distribution.splunk` configuration block supports continuous CPU profiling +and `instrumentations.http.trace_response_header_enabled`, which defaults to `true`. To disable the trace response header, add this block to the YAML file: ```yaml @@ -93,8 +93,9 @@ distribution: trace_response_header_enabled: false ``` +To enable continuous profiling, see [profiling configuration](docs/profiling.md#yaml-configuration). When `OTEL_CONFIG_FILE` is set, the distribution does not enable -[continuous or call graph profiling](docs/profiling.md) or +[call graph profiling](docs/profiling.md#call-graph-profiling) or [PostgreSQL trace correlation](docs/postgresql.md). OpAMP does not report the declarative file as effective configuration and cannot diff --git a/docs/profiling.md b/docs/profiling.md index 8e192f4b..481e2e80 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -5,9 +5,8 @@ profiling** (also called snapshot profiling). Both modes collect CPU stack trace export them as `pprof` data via the OTel logs exporter, with trace and span IDs embedded so profiles can be correlated with traces in Splunk APM. -Both profiling modes require `OTEL_CONFIG_FILE` to be unset. When it is set, the -distribution does not start either profiler, even if its environment variable is -set to `true`. +Continuous profiling supports environment variables and YAML configuration. +Call graph profiling requires `OTEL_CONFIG_FILE` to be unset. ## Continuous profiling @@ -31,6 +30,31 @@ opentelemetry-instrument python app.py | `SPLUNK_PROFILER_CALL_STACK_INTERVAL` | `1000` | How often (in milliseconds) to collect a stack sample from all threads. | | `SPLUNK_PROFILER_LOGS_ENDPOINT` | _(uses `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`)_ | Override the endpoint where profiling data is sent. Applies to both profiling modes. | +### YAML configuration + +When `OTEL_CONFIG_FILE` is set, add the following block to enable continuous CPU +profiling. An empty `cpu_profiler: {}` block uses the default sampling interval +of 1000 milliseconds. Omit `cpu_profiler` to disable continuous profiling. + +```yaml +distribution: + splunk: + profiling: + always_on: + cpu_profiler: + sampling_interval: 1000 +``` + +`sampling_interval` must be a positive integer in milliseconds. Profiles use the +file's `resource` attributes and export through the file's `logger_provider`. +Configure the log export endpoint and authentication headers there, as shown in +the [declarative configuration example](examples/declarative-config.yaml). +The GDI `profiling.exporter` block is not supported. + +`SPLUNK_PROFILER_ENABLED`, `SPLUNK_PROFILER_CALL_STACK_INTERVAL`, and +`SPLUNK_PROFILER_LOGS_ENDPOINT` do not control profiling when `OTEL_CONFIG_FILE` +is set. To use environment values, reference them explicitly in the YAML file. + ### How it works A background thread fires at each `SPLUNK_PROFILER_CALL_STACK_INTERVAL` tick, collects diff --git a/src/splunk_otel/configurator.py b/src/splunk_otel/configurator.py index 32d103f0..a530110b 100644 --- a/src/splunk_otel/configurator.py +++ b/src/splunk_otel/configurator.py @@ -64,8 +64,7 @@ def _configure(self, **kwargs): set_global_response_propagator(ServerTimingResponsePropagator()) configure_logging_instrumentation(env) - # A later change will configure profiling and callgraphs from OTEL_CONFIG_FILE. - # For now, start them from environment variables only when no config file is set. + # Call graphs and PostgreSQL trace correlation do not support file configuration yet. if not config_file: configure_postgresql_trace_context_propagation(env) _start_profiling_if_enabled() @@ -79,6 +78,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi from opentelemetry.configuration._propagator import configure_propagator from opentelemetry.configuration._tracer_provider import configure_tracer_provider from opentelemetry.configuration.instrumentation import configure_instrumentation + from splunk_otel.profiling_config import configure_continuous_profiling if kwargs: _logger.warning( @@ -98,6 +98,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi configure_logger_provider(config.logger_provider, resource) configure_propagator(config.propagator) configure_instrumentation(config.instrumentation_development) + configure_continuous_profiling(config.distribution, resource) return config diff --git a/src/splunk_otel/profiling_config.py b/src/splunk_otel/profiling_config.py new file mode 100644 index 00000000..55e07a24 --- /dev/null +++ b/src/splunk_otel/profiling_config.py @@ -0,0 +1,37 @@ +# Copyright Splunk Inc. +# SPDX-License-Identifier: Apache-2.0 + +from opentelemetry.configuration import ConfigurationError +from opentelemetry.sdk.resources import Resource + +from splunk_otel.profile import ( + _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS, + _get_profiling_logger, + ProfilingContext, +) + + +def configure_continuous_profiling(distribution: dict | None, resource: Resource) -> None: + config = _mapping(distribution, "distribution") + for key in ("splunk", "profiling", "always_on"): + config = _mapping(config.get(key), key) + if "cpu_profiler" not in config: + return + + cpu_profiler = _mapping(config["cpu_profiler"], "cpu_profiler") + interval = cpu_profiler.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) + if type(interval) is not int or interval <= 0: + message = "cpu_profiler.sampling_interval must be a positive integer in milliseconds" + raise ConfigurationError(message) + + context = ProfilingContext(resource, interval, _get_profiling_logger()) + context.start() + + +def _mapping(value: object, name: str) -> dict: + if value is None: + return {} + if not isinstance(value, dict): + message = f"{name} must be a mapping" + raise ConfigurationError(message) + return value diff --git a/tests/integration/declarative_profile.py b/tests/integration/declarative_profile.py new file mode 100644 index 00000000..e1748608 --- /dev/null +++ b/tests/integration/declarative_profile.py @@ -0,0 +1,89 @@ +import base64 +import gzip +import importlib.util +from pathlib import Path + +from lib import project_path, trace_loop + + +if __name__ == "__main__": + from opentelemetry import _logs + + trace_loop(3) + _logs.get_logger_provider().force_flush() + + +class DeclarativeProfileOtelTest: + def requirements(self): + return (project_path(),) + + def declarative_configuration(self): + config = Path(project_path(), "docs", "examples", "declarative-config.yaml").read_text() + config = config.replace("${OTEL_SERVICE_NAME:-my-python-service}", "yaml-profile-service") + return ( + config + + """ +distribution: + splunk: + profiling: + always_on: + cpu_profiler: + sampling_interval: 250 +""" + ) + + def environment_variables(self): + return { + "OTEL_SERVICE_NAME": "environment-service", + "SPLUNK_PROFILER_ENABLED": "false", + "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "9999", + "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS": "system_metrics", + } + + def wrapper_command(self): + return "opentelemetry-instrument" + + def is_http(self): + return False + + def on_start(self): + pass + + def on_stop(self, telemetry, stdout, stderr, returncode): + from oteltest.telemetry import extract_leaves, get_attribute + + profile_path = Path(project_path(), "src", "splunk_otel", "profile_pb2.py") + spec = importlib.util.spec_from_file_location("profile_pb2", profile_path) + profile_pb2 = importlib.util.module_from_spec(spec) + spec.loader.exec_module(profile_pb2) + + assert returncode == 0, stderr + resources = extract_leaves(telemetry, "log_requests", "pbreq", "resource_logs") + profiles = [] + for resource_logs in resources: + for scope_logs in resource_logs.scope_logs: + if scope_logs.scope.name != "otel.profiling": + continue + assert ( + get_attribute(resource_logs.resource.attributes, "service.name").value.string_value + == "yaml-profile-service" + ) + assert ( + get_attribute(resource_logs.resource.attributes, "telemetry.distro.name").value.string_value + == "splunk-opentelemetry" + ) + profiles.extend(scope_logs.log_records) + + assert profiles + for record in profiles: + assert ( + get_attribute(record.attributes, "profiling.instrumentation.source").value.string_value == "continuous" + ) + profile = profile_pb2.Profile() + profile.ParseFromString(gzip.decompress(base64.b64decode(record.body.string_value))) + assert profile.sample + for sample in profile.sample: + (period,) = [ + label.num for label in sample.label if profile.string_table[label.key] == "source.event.period" + ] + assert period == 250 diff --git a/tests/test_profiling_config.py b/tests/test_profiling_config.py new file mode 100644 index 00000000..1c0db6e7 --- /dev/null +++ b/tests/test_profiling_config.py @@ -0,0 +1,81 @@ +import pytest +from opentelemetry.configuration import ConfigurationError +from opentelemetry.sdk.resources import Resource + +from splunk_otel.profiling_config import configure_continuous_profiling + + +@pytest.fixture +def profiling_contexts(monkeypatch): + contexts = [] + + class FakeProfilingContext: + def __init__(self, resource, interval_millis, logger): + self.resource = resource + self.interval_millis = interval_millis + self.logger = logger + self.started = False + contexts.append(self) + + def start(self): + self.started = True + + monkeypatch.setattr("splunk_otel.profiling_config.ProfilingContext", FakeProfilingContext) + return contexts + + +@pytest.mark.parametrize(("cpu_profiler", "interval"), [({}, 1000), (None, 1000), ({"sampling_interval": 250}, 250)]) +def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_profiler, interval): + monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "false") + monkeypatch.setenv("SPLUNK_PROFILER_CALL_STACK_INTERVAL", "9999") + resource = Resource({"service.name": "yaml-service", "deployment.environment.name": "test"}) + + configure_continuous_profiling({"splunk": {"profiling": {"always_on": {"cpu_profiler": cpu_profiler}}}}, resource) + + (context,) = profiling_contexts + assert context.started + assert context.interval_millis == interval + assert context.resource is resource + + +@pytest.mark.parametrize( + "distribution", + [ + None, + {}, + {"splunk": None}, + {"splunk": {"profiling": {}}}, + {"splunk": {"profiling": {"always_on": {}}}}, + {"splunk": {"profiling": {"callgraphs": {}}}}, + ], +) +def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profiling_contexts, distribution): + monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "true") + + configure_continuous_profiling(distribution, Resource({})) + + assert not profiling_contexts + + +@pytest.mark.parametrize("interval", [0, -1, True, 1.5, "1000", None]) +def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval): + with pytest.raises(ConfigurationError, match="sampling_interval must be a positive integer"): + configure_continuous_profiling( + {"splunk": {"profiling": {"always_on": {"cpu_profiler": {"sampling_interval": interval}}}}}, + Resource({}), + ) + + assert not profiling_contexts + + +@pytest.mark.parametrize("key", ["splunk", "profiling", "always_on", "cpu_profiler"]) +def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): + config = {key: False} + keys = ["splunk", "profiling", "always_on", "cpu_profiler"] + for parent in reversed(keys[: keys.index(key)]): + config = {parent: config} + + with pytest.raises(ConfigurationError, match=f"{key} must be a mapping"): + configure_continuous_profiling(config, Resource({})) + + assert not profiling_contexts From 1effbafff10cc5c304f4bd12c408add75cdc45f3 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 10:05:05 -0500 Subject: [PATCH 02/19] Support a dedicated declarative profiling exporter --- docs/profiling.md | 31 +++++++- src/splunk_otel/profile.py | 4 +- src/splunk_otel/profiling_config.py | 35 ++++++++- .../declarative_profile_exporter.py | 77 +++++++++++++++++++ tests/test_profiling_config.py | 25 ++++++ 5 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 tests/integration/declarative_profile_exporter.py diff --git a/docs/profiling.md b/docs/profiling.md index 481e2e80..38c723c8 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -46,10 +46,33 @@ distribution: ``` `sampling_interval` must be a positive integer in milliseconds. Profiles use the -file's `resource` attributes and export through the file's `logger_provider`. -Configure the log export endpoint and authentication headers there, as shown in -the [declarative configuration example](examples/declarative-config.yaml). -The GDI `profiling.exporter` block is not supported. +file's `resource` attributes. By default, profiles export through the file's +`logger_provider`, as shown in the +[declarative configuration example](examples/declarative-config.yaml). + +To send profiles to a separate OTLP HTTP endpoint, add `exporter` under +`distribution.splunk.profiling`: + +```yaml +distribution: + splunk: + profiling: + exporter: + otlp_log_http: + endpoint: http://localhost:4318/v1/logs + headers: + - name: X-SF-TOKEN + value: ${SPLUNK_ACCESS_TOKEN} + always_on: + cpu_profiler: + sampling_interval: 1000 +``` + +The dedicated exporter receives only profile logs. Application logs continue +through `logger_provider`. An empty `otlp_log_http: {}` block uses +`http://localhost:4318/v1/logs`. The block accepts the standard OTLP HTTP log +exporter settings, including `headers`, `timeout` (in milliseconds), and +`compression`. The profiling thread stops and its exporter flushes at process exit. `SPLUNK_PROFILER_ENABLED`, `SPLUNK_PROFILER_CALL_STACK_INTERVAL`, and `SPLUNK_PROFILER_LOGS_ENDPOINT` do not control profiling when `OTEL_CONFIG_FILE` diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index 6fb5995f..ae743f8f 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -92,8 +92,8 @@ def start_profiling(env=None): return ctx -def _get_profiling_logger() -> Logger: - return get_logger(_SCOPE_NAME, _SCOPE_VERSION) +def _get_profiling_logger(logger_provider=None) -> Logger: + return get_logger(_SCOPE_NAME, _SCOPE_VERSION, logger_provider=logger_provider) def _mk_resource(service_name) -> Resource: diff --git a/src/splunk_otel/profiling_config.py b/src/splunk_otel/profiling_config.py index 55e07a24..40ef382e 100644 --- a/src/splunk_otel/profiling_config.py +++ b/src/splunk_otel/profiling_config.py @@ -1,7 +1,13 @@ # Copyright Splunk Inc. # SPDX-License-Identifier: Apache-2.0 +import atexit + from opentelemetry.configuration import ConfigurationError +from opentelemetry.configuration._conversion import _dict_to_dataclass +from opentelemetry.configuration._logger_provider import create_logger_provider +from opentelemetry.configuration.file._loader import _validate_schema +from opentelemetry.configuration.models import LoggerProvider as LoggerProviderConfig from opentelemetry.sdk.resources import Resource from splunk_otel.profile import ( @@ -13,19 +19,40 @@ def configure_continuous_profiling(distribution: dict | None, resource: Resource) -> None: config = _mapping(distribution, "distribution") - for key in ("splunk", "profiling", "always_on"): + for key in ("splunk", "profiling"): config = _mapping(config.get(key), key) - if "cpu_profiler" not in config: + always_on = _mapping(config.get("always_on"), "always_on") + if "cpu_profiler" not in always_on: return - cpu_profiler = _mapping(config["cpu_profiler"], "cpu_profiler") + cpu_profiler = _mapping(always_on["cpu_profiler"], "cpu_profiler") interval = cpu_profiler.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) if type(interval) is not int or interval <= 0: message = "cpu_profiler.sampling_interval must be a positive integer in milliseconds" raise ConfigurationError(message) - context = ProfilingContext(resource, interval, _get_profiling_logger()) + logger = _create_profiling_logger(config, resource) + context = ProfilingContext(resource, interval, logger) context.start() + # The provider registers shutdown first; stop sampling before it flushes at exit. + atexit.register(context.stop) + + +def _create_profiling_logger(profiling: dict, resource: Resource): + if "exporter" not in profiling: + return _get_profiling_logger() + + exporter = _mapping(profiling["exporter"], "profiling.exporter") + if exporter and set(exporter) != {"otlp_log_http"}: + message = "profiling.exporter supports only otlp_log_http" + raise ConfigurationError(message) + http = _mapping(exporter.get("otlp_log_http"), "profiling.exporter.otlp_log_http") + http = {"endpoint": "http://localhost:4318/v1/logs", **http} + provider_config = {"processors": [{"batch": {"exporter": {"otlp_http": http}}}]} + # Apply the same schema and model conversion as the standard YAML log exporter. + _validate_schema({"file_format": "1.0", "logger_provider": provider_config}) + provider = create_logger_provider(_dict_to_dataclass(provider_config, LoggerProviderConfig), resource) + return _get_profiling_logger(provider) def _mapping(value: object, name: str) -> dict: diff --git a/tests/integration/declarative_profile_exporter.py b/tests/integration/declarative_profile_exporter.py new file mode 100644 index 00000000..923dc48d --- /dev/null +++ b/tests/integration/declarative_profile_exporter.py @@ -0,0 +1,77 @@ +import logging +from pathlib import Path + +from lib import project_path, trace_loop, wait_for_server + + +if __name__ == "__main__": + from opentelemetry._logs import LogRecord, get_logger + + get_logger("application").emit(LogRecord(body="application log")) + trace_loop(1) + + +class DeclarativeProfileExporterOtelTest: + def requirements(self): + return (project_path(),) + + def declarative_configuration(self): + config = Path(project_path(), "docs", "examples", "declarative-config.yaml").read_text() + return ( + config + + """ +distribution: + splunk: + profiling: + exporter: + otlp_log_http: + endpoint: http://localhost:4318/v1/logs + headers: + - name: x-profile-test + value: yaml-token + timeout: 5000 + always_on: + cpu_profiler: + sampling_interval: 100 +""" + ) + + def environment_variables(self): + return { + "SPLUNK_PROFILER_LOGS_ENDPOINT": "http://localhost:1/unused", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://localhost:1/unused", + "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS": "system_metrics", + } + + def wrapper_command(self): + return "opentelemetry-instrument" + + def is_http(self): + return False + + def on_start(self): + from oteltest.sink import HttpSink, raise_if_port_in_use + from oteltest.sink.handler import AccumulatingHandler + + raise_if_port_in_use(4318) + self.profile_handler = AccumulatingHandler() + self.profile_sink = HttpSink(self.profile_handler, logging.getLogger(__name__)) + self.profile_sink.start() + wait_for_server("localhost", 4318) + + def on_stop(self, telemetry, stdout, stderr, returncode): + from oteltest.telemetry import extract_leaves, get_logs + + self.profile_sink.stop() + assert returncode == 0, stderr + assert any(record.body.string_value == "application log" for record in get_logs(telemetry)) + application_scopes = extract_leaves(telemetry, "log_requests", "pbreq", "resource_logs", "scope_logs") + assert all(scope.scope.name != "otel.profiling" for scope in application_scopes) + + profile_telemetry = self.profile_handler.telemetry + profile_scopes = extract_leaves(profile_telemetry, "log_requests", "pbreq", "resource_logs", "scope_logs") + assert profile_scopes + assert all(scope.scope.name == "otel.profiling" and scope.log_records for scope in profile_scopes) + for request in profile_telemetry.log_requests: + headers = {key.lower(): value for key, value in request.headers.items()} + assert headers["x-profile-test"] == "yaml-token" diff --git a/tests/test_profiling_config.py b/tests/test_profiling_config.py index 1c0db6e7..45aba196 100644 --- a/tests/test_profiling_config.py +++ b/tests/test_profiling_config.py @@ -20,6 +20,9 @@ def __init__(self, resource, interval_millis, logger): def start(self): self.started = True + def stop(self): + self.started = False + monkeypatch.setattr("splunk_otel.profiling_config.ProfilingContext", FakeProfilingContext) return contexts @@ -79,3 +82,25 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): configure_continuous_profiling(config, Resource({})) assert not profiling_contexts + + +@pytest.mark.parametrize( + "exporter", + [ + False, + {"otlp_log_grpc": {}}, + {"otlp_log_http": False}, + {"otlp_log_http": {}, "console": {}}, + {"otlp_log_http": {"endpoint": 123}}, + {"otlp_log_http": {"headers": "invalid"}}, + {"otlp_log_http": {"timeout": "invalid"}}, + ], +) +def test_invalid_profile_exporter_fails_before_starting(profiling_contexts, exporter): + with pytest.raises(ConfigurationError): + configure_continuous_profiling( + {"splunk": {"profiling": {"exporter": exporter, "always_on": {"cpu_profiler": {}}}}}, + Resource({}), + ) + + assert not profiling_contexts From 1bba22ac84ee7f06202d1612c468b63bd5ae8b36 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 10:16:38 -0500 Subject: [PATCH 03/19] Use a dedicated default exporter for declarative profiling --- docs/profiling.md | 8 ++++---- src/splunk_otel/profiling_config.py | 5 +---- tests/integration/declarative_profile.py | 19 +++++++++---------- .../declarative_profile_exporter.py | 8 ++++---- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/docs/profiling.md b/docs/profiling.md index 38c723c8..796f8563 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -46,11 +46,11 @@ distribution: ``` `sampling_interval` must be a positive integer in milliseconds. Profiles use the -file's `resource` attributes. By default, profiles export through the file's -`logger_provider`, as shown in the -[declarative configuration example](examples/declarative-config.yaml). +file's `resource` attributes and a dedicated OTLP HTTP exporter. When no profiling +exporter is configured, profiles go to `http://localhost:4318/v1/logs`. +Profile export does not require a `logger_provider` block. -To send profiles to a separate OTLP HTTP endpoint, add `exporter` under +To configure the profile export endpoint or headers, add `exporter` under `distribution.splunk.profiling`: ```yaml diff --git a/src/splunk_otel/profiling_config.py b/src/splunk_otel/profiling_config.py index 40ef382e..9c26260f 100644 --- a/src/splunk_otel/profiling_config.py +++ b/src/splunk_otel/profiling_config.py @@ -39,10 +39,7 @@ def configure_continuous_profiling(distribution: dict | None, resource: Resource def _create_profiling_logger(profiling: dict, resource: Resource): - if "exporter" not in profiling: - return _get_profiling_logger() - - exporter = _mapping(profiling["exporter"], "profiling.exporter") + exporter = _mapping(profiling.get("exporter"), "profiling.exporter") if exporter and set(exporter) != {"otlp_log_http"}: message = "profiling.exporter supports only otlp_log_http" raise ConfigurationError(message) diff --git a/tests/integration/declarative_profile.py b/tests/integration/declarative_profile.py index e1748608..38f33a29 100644 --- a/tests/integration/declarative_profile.py +++ b/tests/integration/declarative_profile.py @@ -7,10 +7,7 @@ if __name__ == "__main__": - from opentelemetry import _logs - trace_loop(3) - _logs.get_logger_provider().force_flush() class DeclarativeProfileOtelTest: @@ -18,11 +15,12 @@ def requirements(self): return (project_path(),) def declarative_configuration(self): - config = Path(project_path(), "docs", "examples", "declarative-config.yaml").read_text() - config = config.replace("${OTEL_SERVICE_NAME:-my-python-service}", "yaml-profile-service") - return ( - config - + """ + return """ +file_format: "1.0" +resource: + attributes: + - name: service.name + value: yaml-profile-service distribution: splunk: profiling: @@ -30,13 +28,14 @@ def declarative_configuration(self): cpu_profiler: sampling_interval: 250 """ - ) def environment_variables(self): return { "OTEL_SERVICE_NAME": "environment-service", "SPLUNK_PROFILER_ENABLED": "false", "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "9999", + "SPLUNK_PROFILER_LOGS_ENDPOINT": "http://localhost:1/unused", + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": "http://localhost:1/unused", "OTEL_PYTHON_DISABLED_INSTRUMENTATIONS": "system_metrics", } @@ -44,7 +43,7 @@ def wrapper_command(self): return "opentelemetry-instrument" def is_http(self): - return False + return True def on_start(self): pass diff --git a/tests/integration/declarative_profile_exporter.py b/tests/integration/declarative_profile_exporter.py index 923dc48d..a0a49c5e 100644 --- a/tests/integration/declarative_profile_exporter.py +++ b/tests/integration/declarative_profile_exporter.py @@ -25,7 +25,7 @@ def declarative_configuration(self): profiling: exporter: otlp_log_http: - endpoint: http://localhost:4318/v1/logs + endpoint: http://localhost:14318/v1/logs headers: - name: x-profile-test value: yaml-token @@ -53,11 +53,11 @@ def on_start(self): from oteltest.sink import HttpSink, raise_if_port_in_use from oteltest.sink.handler import AccumulatingHandler - raise_if_port_in_use(4318) + raise_if_port_in_use(14318) self.profile_handler = AccumulatingHandler() - self.profile_sink = HttpSink(self.profile_handler, logging.getLogger(__name__)) + self.profile_sink = HttpSink(self.profile_handler, logging.getLogger(__name__), port=14318) self.profile_sink.start() - wait_for_server("localhost", 4318) + wait_for_server("localhost", 14318) def on_stop(self, telemetry, stdout, stderr, returncode): from oteltest.telemetry import extract_leaves, get_logs From c494b67cd7abb3946754bb68c280d02f0f46d40a Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 10:18:06 -0500 Subject: [PATCH 04/19] Make declarative profiling provider setup explicit --- src/splunk_otel/profile.py | 4 ++-- src/splunk_otel/profiling_config.py | 18 ++++++++++-------- tests/test_profiling_config.py | 6 ++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index ae743f8f..832d505a 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -11,7 +11,7 @@ import opentelemetry.context import wrapt -from opentelemetry._logs import Logger, LogRecord, SeverityNumber, get_logger +from opentelemetry._logs import Logger, LoggerProvider, LogRecord, SeverityNumber, get_logger from opentelemetry.context import Context from opentelemetry.instrumentation.version import __version__ as version from opentelemetry.sdk._logs import ReadWriteLogRecord @@ -92,7 +92,7 @@ def start_profiling(env=None): return ctx -def _get_profiling_logger(logger_provider=None) -> Logger: +def _get_profiling_logger(logger_provider: LoggerProvider | None = None) -> Logger: return get_logger(_SCOPE_NAME, _SCOPE_VERSION, logger_provider=logger_provider) diff --git a/src/splunk_otel/profiling_config.py b/src/splunk_otel/profiling_config.py index 9c26260f..87e2c1da 100644 --- a/src/splunk_otel/profiling_config.py +++ b/src/splunk_otel/profiling_config.py @@ -8,6 +8,7 @@ from opentelemetry.configuration._logger_provider import create_logger_provider from opentelemetry.configuration.file._loader import _validate_schema from opentelemetry.configuration.models import LoggerProvider as LoggerProviderConfig +from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.resources import Resource from splunk_otel.profile import ( @@ -18,10 +19,10 @@ def configure_continuous_profiling(distribution: dict | None, resource: Resource) -> None: - config = _mapping(distribution, "distribution") - for key in ("splunk", "profiling"): - config = _mapping(config.get(key), key) - always_on = _mapping(config.get("always_on"), "always_on") + distribution = _mapping(distribution, "distribution") + splunk = _mapping(distribution.get("splunk"), "splunk") + profiling = _mapping(splunk.get("profiling"), "profiling") + always_on = _mapping(profiling.get("always_on"), "always_on") if "cpu_profiler" not in always_on: return @@ -31,14 +32,15 @@ def configure_continuous_profiling(distribution: dict | None, resource: Resource message = "cpu_profiler.sampling_interval must be a positive integer in milliseconds" raise ConfigurationError(message) - logger = _create_profiling_logger(config, resource) + provider = _create_profiling_logger_provider(profiling, resource) + logger = _get_profiling_logger(provider) context = ProfilingContext(resource, interval, logger) context.start() # The provider registers shutdown first; stop sampling before it flushes at exit. atexit.register(context.stop) -def _create_profiling_logger(profiling: dict, resource: Resource): +def _create_profiling_logger_provider(profiling: dict, resource: Resource) -> LoggerProvider: exporter = _mapping(profiling.get("exporter"), "profiling.exporter") if exporter and set(exporter) != {"otlp_log_http"}: message = "profiling.exporter supports only otlp_log_http" @@ -48,8 +50,8 @@ def _create_profiling_logger(profiling: dict, resource: Resource): provider_config = {"processors": [{"batch": {"exporter": {"otlp_http": http}}}]} # Apply the same schema and model conversion as the standard YAML log exporter. _validate_schema({"file_format": "1.0", "logger_provider": provider_config}) - provider = create_logger_provider(_dict_to_dataclass(provider_config, LoggerProviderConfig), resource) - return _get_profiling_logger(provider) + config = _dict_to_dataclass(provider_config, LoggerProviderConfig) + return create_logger_provider(config, resource) def _mapping(value: object, name: str) -> dict: diff --git a/tests/test_profiling_config.py b/tests/test_profiling_config.py index 45aba196..d0f4ee32 100644 --- a/tests/test_profiling_config.py +++ b/tests/test_profiling_config.py @@ -1,4 +1,5 @@ import pytest +from opentelemetry._logs import NoOpLoggerProvider from opentelemetry.configuration import ConfigurationError from opentelemetry.sdk.resources import Resource @@ -24,6 +25,11 @@ def stop(self): self.started = False monkeypatch.setattr("splunk_otel.profiling_config.ProfilingContext", FakeProfilingContext) + + def _fake_create_logger_provider(_config, _resource): + return NoOpLoggerProvider() + + monkeypatch.setattr("splunk_otel.profiling_config.create_logger_provider", _fake_create_logger_provider) return contexts From 190d9fd721a9574c8ba8bd79b213a88e00073d73 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 10:32:29 -0500 Subject: [PATCH 05/19] Share continuous profiling startup across configuration paths --- src/splunk_otel/configurator.py | 4 +-- src/splunk_otel/profile.py | 22 ++++++++----- src/splunk_otel/profiling_config.py | 11 ++----- tests/test_profiling_config.py | 51 +++++++++++++++++++++++++---- 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/splunk_otel/configurator.py b/src/splunk_otel/configurator.py index a530110b..70ecc22f 100644 --- a/src/splunk_otel/configurator.py +++ b/src/splunk_otel/configurator.py @@ -78,7 +78,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi from opentelemetry.configuration._propagator import configure_propagator from opentelemetry.configuration._tracer_provider import configure_tracer_provider from opentelemetry.configuration.instrumentation import configure_instrumentation - from splunk_otel.profiling_config import configure_continuous_profiling + from splunk_otel.profiling_config import configure_declarative_continuous_profiling if kwargs: _logger.warning( @@ -98,7 +98,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi configure_logger_provider(config.logger_provider, resource) configure_propagator(config.propagator) configure_instrumentation(config.instrumentation_development) - configure_continuous_profiling(config.distribution, resource) + configure_declarative_continuous_profiling(config.distribution, resource) return config diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index 832d505a..edd7752b 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -1,3 +1,4 @@ +import atexit import base64 import gzip import sys @@ -81,15 +82,20 @@ def _start_profiling_if_enabled(env=None): def start_profiling(env=None): env = env or Env() interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) - svcname = env.getval(OTEL_SERVICE_NAME) - - ctx = ProfilingContext( - _mk_resource(svcname), - interval_millis, - get_logger(_SCOPE_NAME, _SCOPE_VERSION), + service_name = env.getval(OTEL_SERVICE_NAME) + return start_continuous_profiling( + resource=_mk_resource(service_name), + interval_millis=interval_millis, + logger=_get_profiling_logger(), ) - ctx.start() - return ctx + + +def start_continuous_profiling(resource: Resource, interval_millis: int, logger: Logger) -> ProfilingContext: + context = ProfilingContext(resource, interval_millis, logger) + context.start() + # Providers register shutdown first; stop sampling before they flush at exit. + atexit.register(context.stop) + return context def _get_profiling_logger(logger_provider: LoggerProvider | None = None) -> Logger: diff --git a/src/splunk_otel/profiling_config.py b/src/splunk_otel/profiling_config.py index 87e2c1da..5f54a70a 100644 --- a/src/splunk_otel/profiling_config.py +++ b/src/splunk_otel/profiling_config.py @@ -1,8 +1,6 @@ # Copyright Splunk Inc. # SPDX-License-Identifier: Apache-2.0 -import atexit - from opentelemetry.configuration import ConfigurationError from opentelemetry.configuration._conversion import _dict_to_dataclass from opentelemetry.configuration._logger_provider import create_logger_provider @@ -14,11 +12,11 @@ from splunk_otel.profile import ( _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS, _get_profiling_logger, - ProfilingContext, + start_continuous_profiling, ) -def configure_continuous_profiling(distribution: dict | None, resource: Resource) -> None: +def configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: distribution = _mapping(distribution, "distribution") splunk = _mapping(distribution.get("splunk"), "splunk") profiling = _mapping(splunk.get("profiling"), "profiling") @@ -34,10 +32,7 @@ def configure_continuous_profiling(distribution: dict | None, resource: Resource provider = _create_profiling_logger_provider(profiling, resource) logger = _get_profiling_logger(provider) - context = ProfilingContext(resource, interval, logger) - context.start() - # The provider registers shutdown first; stop sampling before it flushes at exit. - atexit.register(context.stop) + start_continuous_profiling(resource=resource, interval_millis=interval, logger=logger) def _create_profiling_logger_provider(profiling: dict, resource: Resource) -> LoggerProvider: diff --git a/tests/test_profiling_config.py b/tests/test_profiling_config.py index d0f4ee32..122fd4b2 100644 --- a/tests/test_profiling_config.py +++ b/tests/test_profiling_config.py @@ -3,7 +3,9 @@ from opentelemetry.configuration import ConfigurationError from opentelemetry.sdk.resources import Resource -from splunk_otel.profiling_config import configure_continuous_profiling +from splunk_otel.env import Env +from splunk_otel.profile import start_profiling +from splunk_otel.profiling_config import configure_declarative_continuous_profiling @pytest.fixture @@ -24,7 +26,7 @@ def start(self): def stop(self): self.started = False - monkeypatch.setattr("splunk_otel.profiling_config.ProfilingContext", FakeProfilingContext) + monkeypatch.setattr("splunk_otel.profile.ProfilingContext", FakeProfilingContext) def _fake_create_logger_provider(_config, _resource): return NoOpLoggerProvider() @@ -39,7 +41,9 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p monkeypatch.setenv("SPLUNK_PROFILER_CALL_STACK_INTERVAL", "9999") resource = Resource({"service.name": "yaml-service", "deployment.environment.name": "test"}) - configure_continuous_profiling({"splunk": {"profiling": {"always_on": {"cpu_profiler": cpu_profiler}}}}, resource) + configure_declarative_continuous_profiling( + {"splunk": {"profiling": {"always_on": {"cpu_profiler": cpu_profiler}}}}, resource + ) (context,) = profiling_contexts assert context.started @@ -47,6 +51,39 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p assert context.resource is resource +def test_environment_starts_continuous_profiling(profiling_contexts): + environment = Env({"OTEL_SERVICE_NAME": "environment-service", "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "250"}) + + context = start_profiling(environment) + + assert profiling_contexts == [context] + assert context.started + assert context.interval_millis == 250 + assert context.resource.attributes["service.name"] == "environment-service" + + +@pytest.mark.parametrize("configuration_source", ["environment", "declarative"]) +def test_continuous_profiling_stops_at_exit(monkeypatch, profiling_contexts, configuration_source): + shutdown_callbacks = [] + + def _fake_register_shutdown(callback): + shutdown_callbacks.append(callback) + + monkeypatch.setattr("splunk_otel.profile.atexit.register", _fake_register_shutdown) + if configuration_source == "environment": + start_profiling(Env({"OTEL_SERVICE_NAME": "environment-service"})) + else: + configure_declarative_continuous_profiling( + {"splunk": {"profiling": {"always_on": {"cpu_profiler": {}}}}}, Resource({}) + ) + + (context,) = profiling_contexts + assert context.started + (shutdown,) = shutdown_callbacks + shutdown() + assert not context.started + + @pytest.mark.parametrize( "distribution", [ @@ -61,7 +98,7 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profiling_contexts, distribution): monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "true") - configure_continuous_profiling(distribution, Resource({})) + configure_declarative_continuous_profiling(distribution, Resource({})) assert not profiling_contexts @@ -69,7 +106,7 @@ def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profili @pytest.mark.parametrize("interval", [0, -1, True, 1.5, "1000", None]) def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval): with pytest.raises(ConfigurationError, match="sampling_interval must be a positive integer"): - configure_continuous_profiling( + configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {"sampling_interval": interval}}}}}, Resource({}), ) @@ -85,7 +122,7 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): config = {parent: config} with pytest.raises(ConfigurationError, match=f"{key} must be a mapping"): - configure_continuous_profiling(config, Resource({})) + configure_declarative_continuous_profiling(config, Resource({})) assert not profiling_contexts @@ -104,7 +141,7 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): ) def test_invalid_profile_exporter_fails_before_starting(profiling_contexts, exporter): with pytest.raises(ConfigurationError): - configure_continuous_profiling( + configure_declarative_continuous_profiling( {"splunk": {"profiling": {"exporter": exporter, "always_on": {"cpu_profiler": {}}}}}, Resource({}), ) From 1a243b112766ce1ced29ff7097d2abc8346b1cc3 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 10:46:36 -0500 Subject: [PATCH 06/19] Clarify profiling configuration and exporter names --- src/splunk_otel/configurator.py | 6 ++-- ...ing_config.py => declarative_profiling.py} | 36 +++++++++---------- src/splunk_otel/profile.py | 6 ++-- ...=> declarative_custom_profile_exporter.py} | 2 +- ...> declarative_default_profile_exporter.py} | 2 +- tests/test_configurator.py | 2 +- ...fig.py => test_profiling_configuration.py} | 10 +++--- 7 files changed, 32 insertions(+), 32 deletions(-) rename src/splunk_otel/{profiling_config.py => declarative_profiling.py} (52%) rename tests/integration/{declarative_profile_exporter.py => declarative_custom_profile_exporter.py} (98%) rename tests/integration/{declarative_profile.py => declarative_default_profile_exporter.py} (98%) rename tests/{test_profiling_config.py => test_profiling_configuration.py} (92%) diff --git a/src/splunk_otel/configurator.py b/src/splunk_otel/configurator.py index 70ecc22f..e1395027 100644 --- a/src/splunk_otel/configurator.py +++ b/src/splunk_otel/configurator.py @@ -26,7 +26,7 @@ from splunk_otel.callgraphs import _configure_callgraphs_if_enabled from splunk_otel.distro import _DEFAULT_SERVICE_NAME, _DISTRO_NAME from splunk_otel.env import SPLUNK_TRACE_RESPONSE_HEADER_ENABLED, Env -from splunk_otel.profile import _start_profiling_if_enabled +from splunk_otel.profile import configure_environment_continuous_profiling from splunk_otel.propagator import ServerTimingResponsePropagator from splunk_otel.runtime import ( _SERVER_TIMING_DEFAULT_ENABLED, @@ -67,7 +67,7 @@ def _configure(self, **kwargs): # Call graphs and PostgreSQL trace correlation do not support file configuration yet. if not config_file: configure_postgresql_trace_context_propagation(env) - _start_profiling_if_enabled() + configure_environment_continuous_profiling() _configure_callgraphs_if_enabled() @@ -78,7 +78,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi from opentelemetry.configuration._propagator import configure_propagator from opentelemetry.configuration._tracer_provider import configure_tracer_provider from opentelemetry.configuration.instrumentation import configure_instrumentation - from splunk_otel.profiling_config import configure_declarative_continuous_profiling + from splunk_otel.declarative_profiling import configure_declarative_continuous_profiling if kwargs: _logger.warning( diff --git a/src/splunk_otel/profiling_config.py b/src/splunk_otel/declarative_profiling.py similarity index 52% rename from src/splunk_otel/profiling_config.py rename to src/splunk_otel/declarative_profiling.py index 5f54a70a..232be7aa 100644 --- a/src/splunk_otel/profiling_config.py +++ b/src/splunk_otel/declarative_profiling.py @@ -17,39 +17,39 @@ def configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: - distribution = _mapping(distribution, "distribution") - splunk = _mapping(distribution.get("splunk"), "splunk") - profiling = _mapping(splunk.get("profiling"), "profiling") - always_on = _mapping(profiling.get("always_on"), "always_on") + distribution = _mapping_or_empty(distribution, "distribution") + splunk = _mapping_or_empty(distribution.get("splunk"), "splunk") + profiling = _mapping_or_empty(splunk.get("profiling"), "profiling") + always_on = _mapping_or_empty(profiling.get("always_on"), "always_on") if "cpu_profiler" not in always_on: return - cpu_profiler = _mapping(always_on["cpu_profiler"], "cpu_profiler") - interval = cpu_profiler.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) - if type(interval) is not int or interval <= 0: + cpu_profiler = _mapping_or_empty(always_on["cpu_profiler"], "cpu_profiler") + interval_millis = cpu_profiler.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) + if type(interval_millis) is not int or interval_millis <= 0: message = "cpu_profiler.sampling_interval must be a positive integer in milliseconds" raise ConfigurationError(message) - provider = _create_profiling_logger_provider(profiling, resource) - logger = _get_profiling_logger(provider) - start_continuous_profiling(resource=resource, interval_millis=interval, logger=logger) + logger_provider = _create_profiling_logger_provider(profiling, resource) + logger = _get_profiling_logger(logger_provider) + start_continuous_profiling(resource=resource, interval_millis=interval_millis, logger=logger) def _create_profiling_logger_provider(profiling: dict, resource: Resource) -> LoggerProvider: - exporter = _mapping(profiling.get("exporter"), "profiling.exporter") + exporter = _mapping_or_empty(profiling.get("exporter"), "profiling.exporter") if exporter and set(exporter) != {"otlp_log_http"}: message = "profiling.exporter supports only otlp_log_http" raise ConfigurationError(message) - http = _mapping(exporter.get("otlp_log_http"), "profiling.exporter.otlp_log_http") - http = {"endpoint": "http://localhost:4318/v1/logs", **http} - provider_config = {"processors": [{"batch": {"exporter": {"otlp_http": http}}}]} + http_exporter_config = _mapping_or_empty(exporter.get("otlp_log_http"), "profiling.exporter.otlp_log_http") + http_exporter_config = {"endpoint": "http://localhost:4318/v1/logs", **http_exporter_config} + logger_provider_data = {"processors": [{"batch": {"exporter": {"otlp_http": http_exporter_config}}}]} # Apply the same schema and model conversion as the standard YAML log exporter. - _validate_schema({"file_format": "1.0", "logger_provider": provider_config}) - config = _dict_to_dataclass(provider_config, LoggerProviderConfig) - return create_logger_provider(config, resource) + _validate_schema({"file_format": "1.0", "logger_provider": logger_provider_data}) + logger_provider_config = _dict_to_dataclass(logger_provider_data, LoggerProviderConfig) + return create_logger_provider(logger_provider_config, resource) -def _mapping(value: object, name: str) -> dict: +def _mapping_or_empty(value: object, name: str) -> dict: if value is None: return {} if not isinstance(value, dict): diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index edd7752b..9f7a1336 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -73,13 +73,13 @@ def pause_after(self, seconds: float): self._timer.pause_after(seconds) -def _start_profiling_if_enabled(env=None): +def configure_environment_continuous_profiling(env=None): env = env or Env() if env.is_true(SPLUNK_PROFILER_ENABLED): - start_profiling(env) + start_profiling_from_environment(env) -def start_profiling(env=None): +def start_profiling_from_environment(env=None): env = env or Env() interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) service_name = env.getval(OTEL_SERVICE_NAME) diff --git a/tests/integration/declarative_profile_exporter.py b/tests/integration/declarative_custom_profile_exporter.py similarity index 98% rename from tests/integration/declarative_profile_exporter.py rename to tests/integration/declarative_custom_profile_exporter.py index a0a49c5e..3c3bc0d6 100644 --- a/tests/integration/declarative_profile_exporter.py +++ b/tests/integration/declarative_custom_profile_exporter.py @@ -11,7 +11,7 @@ trace_loop(1) -class DeclarativeProfileExporterOtelTest: +class DeclarativeCustomProfileExporterOtelTest: def requirements(self): return (project_path(),) diff --git a/tests/integration/declarative_profile.py b/tests/integration/declarative_default_profile_exporter.py similarity index 98% rename from tests/integration/declarative_profile.py rename to tests/integration/declarative_default_profile_exporter.py index 38f33a29..248341c4 100644 --- a/tests/integration/declarative_profile.py +++ b/tests/integration/declarative_default_profile_exporter.py @@ -10,7 +10,7 @@ trace_loop(3) -class DeclarativeProfileOtelTest: +class DeclarativeDefaultProfileExporterOtelTest: def requirements(self): return (project_path(),) diff --git a/tests/test_configurator.py b/tests/test_configurator.py index eb00a6e4..6d283155 100644 --- a/tests/test_configurator.py +++ b/tests/test_configurator.py @@ -140,7 +140,7 @@ def test_configurator_activates_postgresql_only_without_config_file(monkeypatch, lambda *_a, **_k: OpenTelemetryConfiguration(file_format="1.0"), ) monkeypatch.setattr("splunk_otel.configurator.configure_logging_instrumentation", lambda _env: None) - monkeypatch.setattr("splunk_otel.configurator._start_profiling_if_enabled", lambda: None) + monkeypatch.setattr("splunk_otel.configurator.configure_environment_continuous_profiling", lambda: None) monkeypatch.setattr("splunk_otel.configurator._configure_callgraphs_if_enabled", lambda: None) monkeypatch.setattr( "splunk_otel.runtime.enable_postgresql_trace_context_propagation", diff --git a/tests/test_profiling_config.py b/tests/test_profiling_configuration.py similarity index 92% rename from tests/test_profiling_config.py rename to tests/test_profiling_configuration.py index 122fd4b2..c3e23049 100644 --- a/tests/test_profiling_config.py +++ b/tests/test_profiling_configuration.py @@ -4,8 +4,8 @@ from opentelemetry.sdk.resources import Resource from splunk_otel.env import Env -from splunk_otel.profile import start_profiling -from splunk_otel.profiling_config import configure_declarative_continuous_profiling +from splunk_otel.profile import start_profiling_from_environment +from splunk_otel.declarative_profiling import configure_declarative_continuous_profiling @pytest.fixture @@ -31,7 +31,7 @@ def stop(self): def _fake_create_logger_provider(_config, _resource): return NoOpLoggerProvider() - monkeypatch.setattr("splunk_otel.profiling_config.create_logger_provider", _fake_create_logger_provider) + monkeypatch.setattr("splunk_otel.declarative_profiling.create_logger_provider", _fake_create_logger_provider) return contexts @@ -54,7 +54,7 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p def test_environment_starts_continuous_profiling(profiling_contexts): environment = Env({"OTEL_SERVICE_NAME": "environment-service", "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "250"}) - context = start_profiling(environment) + context = start_profiling_from_environment(environment) assert profiling_contexts == [context] assert context.started @@ -71,7 +71,7 @@ def _fake_register_shutdown(callback): monkeypatch.setattr("splunk_otel.profile.atexit.register", _fake_register_shutdown) if configuration_source == "environment": - start_profiling(Env({"OTEL_SERVICE_NAME": "environment-service"})) + start_profiling_from_environment(Env({"OTEL_SERVICE_NAME": "environment-service"})) else: configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {}}}}}, Resource({}) From 230d54b0cb4bbc2a7254b58fdb530526f3425e1d Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 10:59:25 -0500 Subject: [PATCH 07/19] Name environment profiling startup explicitly --- src/splunk_otel/profile.py | 4 ++-- tests/test_profiling_configuration.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index 9f7a1336..90841856 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -76,10 +76,10 @@ def pause_after(self, seconds: float): def configure_environment_continuous_profiling(env=None): env = env or Env() if env.is_true(SPLUNK_PROFILER_ENABLED): - start_profiling_from_environment(env) + start_continuous_profiling_from_environment(env) -def start_profiling_from_environment(env=None): +def start_continuous_profiling_from_environment(env=None): env = env or Env() interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) service_name = env.getval(OTEL_SERVICE_NAME) diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index c3e23049..e598f1d8 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -4,7 +4,7 @@ from opentelemetry.sdk.resources import Resource from splunk_otel.env import Env -from splunk_otel.profile import start_profiling_from_environment +from splunk_otel.profile import start_continuous_profiling_from_environment from splunk_otel.declarative_profiling import configure_declarative_continuous_profiling @@ -54,7 +54,7 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p def test_environment_starts_continuous_profiling(profiling_contexts): environment = Env({"OTEL_SERVICE_NAME": "environment-service", "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "250"}) - context = start_profiling_from_environment(environment) + context = start_continuous_profiling_from_environment(environment) assert profiling_contexts == [context] assert context.started @@ -71,7 +71,7 @@ def _fake_register_shutdown(callback): monkeypatch.setattr("splunk_otel.profile.atexit.register", _fake_register_shutdown) if configuration_source == "environment": - start_profiling_from_environment(Env({"OTEL_SERVICE_NAME": "environment-service"})) + start_continuous_profiling_from_environment(Env({"OTEL_SERVICE_NAME": "environment-service"})) else: configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {}}}}}, Resource({}) From e1379aa7e2dbc41c666eaf06d65933fb3e75f7fd Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 11:06:59 -0500 Subject: [PATCH 08/19] Restore the public start_profiling entry point --- src/splunk_otel/profile.py | 5 +++-- tests/test_profiling_configuration.py | 23 +++++++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index 90841856..92a63c4f 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -76,10 +76,11 @@ def pause_after(self, seconds: float): def configure_environment_continuous_profiling(env=None): env = env or Env() if env.is_true(SPLUNK_PROFILER_ENABLED): - start_continuous_profiling_from_environment(env) + start_profiling(env) -def start_continuous_profiling_from_environment(env=None): +def start_profiling(env=None): + """Start continuous profiling using environment settings.""" env = env or Env() interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) service_name = env.getval(OTEL_SERVICE_NAME) diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index e598f1d8..7825a062 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -4,7 +4,7 @@ from opentelemetry.sdk.resources import Resource from splunk_otel.env import Env -from splunk_otel.profile import start_continuous_profiling_from_environment +from splunk_otel.profile import start_profiling from splunk_otel.declarative_profiling import configure_declarative_continuous_profiling @@ -51,10 +51,12 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p assert context.resource is resource -def test_environment_starts_continuous_profiling(profiling_contexts): +def test_start_profiling_uses_supplied_environment(monkeypatch, profiling_contexts): + monkeypatch.setenv("OTEL_SERVICE_NAME", "process-service") + monkeypatch.setenv("SPLUNK_PROFILER_CALL_STACK_INTERVAL", "9999") environment = Env({"OTEL_SERVICE_NAME": "environment-service", "SPLUNK_PROFILER_CALL_STACK_INTERVAL": "250"}) - context = start_continuous_profiling_from_environment(environment) + context = start_profiling(environment) assert profiling_contexts == [context] assert context.started @@ -62,6 +64,19 @@ def test_environment_starts_continuous_profiling(profiling_contexts): assert context.resource.attributes["service.name"] == "environment-service" +def test_start_profiling_without_arguments_uses_process_environment(monkeypatch, profiling_contexts): + monkeypatch.setenv("OTEL_SERVICE_NAME", "process-service") + monkeypatch.setenv("SPLUNK_PROFILER_CALL_STACK_INTERVAL", "500") + monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "false") + + context = start_profiling() + + assert profiling_contexts == [context] + assert context.started + assert context.interval_millis == 500 + assert context.resource.attributes["service.name"] == "process-service" + + @pytest.mark.parametrize("configuration_source", ["environment", "declarative"]) def test_continuous_profiling_stops_at_exit(monkeypatch, profiling_contexts, configuration_source): shutdown_callbacks = [] @@ -71,7 +86,7 @@ def _fake_register_shutdown(callback): monkeypatch.setattr("splunk_otel.profile.atexit.register", _fake_register_shutdown) if configuration_source == "environment": - start_continuous_profiling_from_environment(Env({"OTEL_SERVICE_NAME": "environment-service"})) + start_profiling(Env({"OTEL_SERVICE_NAME": "environment-service"})) else: configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {}}}}}, Resource({}) From 914d1880ecd3f7114cd3db41736c337329187c6c Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 11:16:19 -0500 Subject: [PATCH 09/19] Keep profiling implementation helpers private --- ..._profiling.py => _declarative_profiling.py} | 6 +++--- src/splunk_otel/configurator.py | 8 ++++---- src/splunk_otel/profile.py | 15 ++++++++------- tests/test_configurator.py | 2 +- tests/test_profiling_configuration.py | 18 +++++++++--------- 5 files changed, 25 insertions(+), 24 deletions(-) rename src/splunk_otel/{declarative_profiling.py => _declarative_profiling.py} (92%) diff --git a/src/splunk_otel/declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py similarity index 92% rename from src/splunk_otel/declarative_profiling.py rename to src/splunk_otel/_declarative_profiling.py index 232be7aa..085cd152 100644 --- a/src/splunk_otel/declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -12,11 +12,11 @@ from splunk_otel.profile import ( _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS, _get_profiling_logger, - start_continuous_profiling, + _start_continuous_profiling, ) -def configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: +def _configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: distribution = _mapping_or_empty(distribution, "distribution") splunk = _mapping_or_empty(distribution.get("splunk"), "splunk") profiling = _mapping_or_empty(splunk.get("profiling"), "profiling") @@ -32,7 +32,7 @@ def configure_declarative_continuous_profiling(distribution: dict | None, resour logger_provider = _create_profiling_logger_provider(profiling, resource) logger = _get_profiling_logger(logger_provider) - start_continuous_profiling(resource=resource, interval_millis=interval_millis, logger=logger) + _start_continuous_profiling(resource=resource, interval_millis=interval_millis, logger=logger) def _create_profiling_logger_provider(profiling: dict, resource: Resource) -> LoggerProvider: diff --git a/src/splunk_otel/configurator.py b/src/splunk_otel/configurator.py index e1395027..f69144b1 100644 --- a/src/splunk_otel/configurator.py +++ b/src/splunk_otel/configurator.py @@ -26,7 +26,7 @@ from splunk_otel.callgraphs import _configure_callgraphs_if_enabled from splunk_otel.distro import _DEFAULT_SERVICE_NAME, _DISTRO_NAME from splunk_otel.env import SPLUNK_TRACE_RESPONSE_HEADER_ENABLED, Env -from splunk_otel.profile import configure_environment_continuous_profiling +from splunk_otel.profile import _start_profiling_if_enabled from splunk_otel.propagator import ServerTimingResponsePropagator from splunk_otel.runtime import ( _SERVER_TIMING_DEFAULT_ENABLED, @@ -67,7 +67,7 @@ def _configure(self, **kwargs): # Call graphs and PostgreSQL trace correlation do not support file configuration yet. if not config_file: configure_postgresql_trace_context_propagation(env) - configure_environment_continuous_profiling() + _start_profiling_if_enabled() _configure_callgraphs_if_enabled() @@ -78,7 +78,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi from opentelemetry.configuration._propagator import configure_propagator from opentelemetry.configuration._tracer_provider import configure_tracer_provider from opentelemetry.configuration.instrumentation import configure_instrumentation - from splunk_otel.declarative_profiling import configure_declarative_continuous_profiling + from splunk_otel._declarative_profiling import _configure_declarative_continuous_profiling if kwargs: _logger.warning( @@ -98,7 +98,7 @@ def _configure_declarative_sdk(config_file: str, **kwargs) -> OpenTelemetryConfi configure_logger_provider(config.logger_provider, resource) configure_propagator(config.propagator) configure_instrumentation(config.instrumentation_development) - configure_declarative_continuous_profiling(config.distribution, resource) + _configure_declarative_continuous_profiling(config.distribution, resource) return config diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index 92a63c4f..a18300d2 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -1,4 +1,4 @@ -import atexit +import atexit as _atexit import base64 import gzip import sys @@ -12,7 +12,8 @@ import opentelemetry.context import wrapt -from opentelemetry._logs import Logger, LoggerProvider, LogRecord, SeverityNumber, get_logger +from opentelemetry._logs import Logger, LogRecord, SeverityNumber, get_logger +from opentelemetry._logs import LoggerProvider as _LoggerProvider from opentelemetry.context import Context from opentelemetry.instrumentation.version import __version__ as version from opentelemetry.sdk._logs import ReadWriteLogRecord @@ -73,7 +74,7 @@ def pause_after(self, seconds: float): self._timer.pause_after(seconds) -def configure_environment_continuous_profiling(env=None): +def _start_profiling_if_enabled(env=None): env = env or Env() if env.is_true(SPLUNK_PROFILER_ENABLED): start_profiling(env) @@ -84,22 +85,22 @@ def start_profiling(env=None): env = env or Env() interval_millis = env.getint(SPLUNK_PROFILER_CALL_STACK_INTERVAL, _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) service_name = env.getval(OTEL_SERVICE_NAME) - return start_continuous_profiling( + return _start_continuous_profiling( resource=_mk_resource(service_name), interval_millis=interval_millis, logger=_get_profiling_logger(), ) -def start_continuous_profiling(resource: Resource, interval_millis: int, logger: Logger) -> ProfilingContext: +def _start_continuous_profiling(resource: Resource, interval_millis: int, logger: Logger) -> ProfilingContext: context = ProfilingContext(resource, interval_millis, logger) context.start() # Providers register shutdown first; stop sampling before they flush at exit. - atexit.register(context.stop) + _atexit.register(context.stop) return context -def _get_profiling_logger(logger_provider: LoggerProvider | None = None) -> Logger: +def _get_profiling_logger(logger_provider: _LoggerProvider | None = None) -> Logger: return get_logger(_SCOPE_NAME, _SCOPE_VERSION, logger_provider=logger_provider) diff --git a/tests/test_configurator.py b/tests/test_configurator.py index 6d283155..eb00a6e4 100644 --- a/tests/test_configurator.py +++ b/tests/test_configurator.py @@ -140,7 +140,7 @@ def test_configurator_activates_postgresql_only_without_config_file(monkeypatch, lambda *_a, **_k: OpenTelemetryConfiguration(file_format="1.0"), ) monkeypatch.setattr("splunk_otel.configurator.configure_logging_instrumentation", lambda _env: None) - monkeypatch.setattr("splunk_otel.configurator.configure_environment_continuous_profiling", lambda: None) + monkeypatch.setattr("splunk_otel.configurator._start_profiling_if_enabled", lambda: None) monkeypatch.setattr("splunk_otel.configurator._configure_callgraphs_if_enabled", lambda: None) monkeypatch.setattr( "splunk_otel.runtime.enable_postgresql_trace_context_propagation", diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index 7825a062..9caa08c6 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -5,7 +5,7 @@ from splunk_otel.env import Env from splunk_otel.profile import start_profiling -from splunk_otel.declarative_profiling import configure_declarative_continuous_profiling +from splunk_otel._declarative_profiling import _configure_declarative_continuous_profiling @pytest.fixture @@ -31,7 +31,7 @@ def stop(self): def _fake_create_logger_provider(_config, _resource): return NoOpLoggerProvider() - monkeypatch.setattr("splunk_otel.declarative_profiling.create_logger_provider", _fake_create_logger_provider) + monkeypatch.setattr("splunk_otel._declarative_profiling.create_logger_provider", _fake_create_logger_provider) return contexts @@ -41,7 +41,7 @@ def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_p monkeypatch.setenv("SPLUNK_PROFILER_CALL_STACK_INTERVAL", "9999") resource = Resource({"service.name": "yaml-service", "deployment.environment.name": "test"}) - configure_declarative_continuous_profiling( + _configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": cpu_profiler}}}}, resource ) @@ -84,11 +84,11 @@ def test_continuous_profiling_stops_at_exit(monkeypatch, profiling_contexts, con def _fake_register_shutdown(callback): shutdown_callbacks.append(callback) - monkeypatch.setattr("splunk_otel.profile.atexit.register", _fake_register_shutdown) + monkeypatch.setattr("splunk_otel.profile._atexit.register", _fake_register_shutdown) if configuration_source == "environment": start_profiling(Env({"OTEL_SERVICE_NAME": "environment-service"})) else: - configure_declarative_continuous_profiling( + _configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {}}}}}, Resource({}) ) @@ -113,7 +113,7 @@ def _fake_register_shutdown(callback): def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profiling_contexts, distribution): monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "true") - configure_declarative_continuous_profiling(distribution, Resource({})) + _configure_declarative_continuous_profiling(distribution, Resource({})) assert not profiling_contexts @@ -121,7 +121,7 @@ def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profili @pytest.mark.parametrize("interval", [0, -1, True, 1.5, "1000", None]) def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval): with pytest.raises(ConfigurationError, match="sampling_interval must be a positive integer"): - configure_declarative_continuous_profiling( + _configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {"sampling_interval": interval}}}}}, Resource({}), ) @@ -137,7 +137,7 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): config = {parent: config} with pytest.raises(ConfigurationError, match=f"{key} must be a mapping"): - configure_declarative_continuous_profiling(config, Resource({})) + _configure_declarative_continuous_profiling(config, Resource({})) assert not profiling_contexts @@ -156,7 +156,7 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): ) def test_invalid_profile_exporter_fails_before_starting(profiling_contexts, exporter): with pytest.raises(ConfigurationError): - configure_declarative_continuous_profiling( + _configure_declarative_continuous_profiling( {"splunk": {"profiling": {"exporter": exporter, "always_on": {"cpu_profiler": {}}}}}, Resource({}), ) From 26780837330a862de83f6c737a16bedc26391d1e Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 13:28:39 -0500 Subject: [PATCH 10/19] Validate declarative profiling configuration with a schema --- pyproject.toml | 1 + src/splunk_otel/_declarative_profiling.py | 52 ++++++++++++---------- src/splunk_otel/_profiling_schema.json | 54 +++++++++++++++++++++++ tests/test_profiling_configuration.py | 10 +++-- 4 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 src/splunk_otel/_profiling_schema.json diff --git a/pyproject.toml b/pyproject.toml index 8e89db8d..cca44a88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dependencies = [ + "jsonschema>=4.0", "opentelemetry-api==1.45.1", "opentelemetry-sdk==1.45.1", "opentelemetry-propagator-b3==1.45.1", diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index 085cd152..a6ffb716 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -1,10 +1,14 @@ # Copyright Splunk Inc. # SPDX-License-Identifier: Apache-2.0 +import json +from importlib.resources import files + +from jsonschema import Draft202012Validator, ValidationError, validators from opentelemetry.configuration import ConfigurationError from opentelemetry.configuration._conversion import _dict_to_dataclass from opentelemetry.configuration._logger_provider import create_logger_provider -from opentelemetry.configuration.file._loader import _validate_schema +from opentelemetry.configuration.file._loader import _get_schema from opentelemetry.configuration.models import LoggerProvider as LoggerProviderConfig from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.resources import Resource @@ -15,20 +19,27 @@ _start_continuous_profiling, ) +_PROFILING_SCHEMA = json.loads(files("splunk_otel").joinpath("_profiling_schema.json").read_text(encoding="utf-8")) +# Protobuf sampling periods need Python integers; JSON Schema also accepts 1.0. +_ProfilingValidator = validators.extend( + Draft202012Validator, + type_checker=Draft202012Validator.TYPE_CHECKER.redefine("integer", lambda _checker, value: type(value) is int), +) +_PROFILING_VALIDATOR = _ProfilingValidator( + {**_PROFILING_SCHEMA, "$defs": {**_get_schema()["$defs"], **_PROFILING_SCHEMA["$defs"]}} +) + def _configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: - distribution = _mapping_or_empty(distribution, "distribution") - splunk = _mapping_or_empty(distribution.get("splunk"), "splunk") - profiling = _mapping_or_empty(splunk.get("profiling"), "profiling") - always_on = _mapping_or_empty(profiling.get("always_on"), "always_on") + _validate_profiling_config(distribution) + splunk = (distribution or {}).get("splunk") or {} + profiling = splunk.get("profiling") or {} + always_on = profiling.get("always_on") or {} if "cpu_profiler" not in always_on: return - cpu_profiler = _mapping_or_empty(always_on["cpu_profiler"], "cpu_profiler") + cpu_profiler = always_on["cpu_profiler"] or {} interval_millis = cpu_profiler.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) - if type(interval_millis) is not int or interval_millis <= 0: - message = "cpu_profiler.sampling_interval must be a positive integer in milliseconds" - raise ConfigurationError(message) logger_provider = _create_profiling_logger_provider(profiling, resource) logger = _get_profiling_logger(logger_provider) @@ -36,23 +47,18 @@ def _configure_declarative_continuous_profiling(distribution: dict | None, resou def _create_profiling_logger_provider(profiling: dict, resource: Resource) -> LoggerProvider: - exporter = _mapping_or_empty(profiling.get("exporter"), "profiling.exporter") - if exporter and set(exporter) != {"otlp_log_http"}: - message = "profiling.exporter supports only otlp_log_http" - raise ConfigurationError(message) - http_exporter_config = _mapping_or_empty(exporter.get("otlp_log_http"), "profiling.exporter.otlp_log_http") + exporter = profiling.get("exporter") or {} + http_exporter_config = exporter.get("otlp_log_http") or {} http_exporter_config = {"endpoint": "http://localhost:4318/v1/logs", **http_exporter_config} logger_provider_data = {"processors": [{"batch": {"exporter": {"otlp_http": http_exporter_config}}}]} - # Apply the same schema and model conversion as the standard YAML log exporter. - _validate_schema({"file_format": "1.0", "logger_provider": logger_provider_data}) logger_provider_config = _dict_to_dataclass(logger_provider_data, LoggerProviderConfig) return create_logger_provider(logger_provider_config, resource) -def _mapping_or_empty(value: object, name: str) -> dict: - if value is None: - return {} - if not isinstance(value, dict): - message = f"{name} must be a mapping" - raise ConfigurationError(message) - return value +def _validate_profiling_config(distribution: dict | None) -> None: + try: + _PROFILING_VALIDATOR.validate(distribution) + except ValidationError as error: + path = ".".join(["distribution", *(str(part) for part in error.absolute_path)]) + message = f"{path}: {error.message}" + raise ConfigurationError(message) from error diff --git a/src/splunk_otel/_profiling_schema.json b/src/splunk_otel/_profiling_schema.json new file mode 100644 index 00000000..d7a4b426 --- /dev/null +++ b/src/splunk_otel/_profiling_schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["object", "null"], + "properties": { + "splunk": { + "type": ["object", "null"], + "properties": { + "profiling": {"$ref": "#/$defs/SplunkProfiling"} + } + } + }, + "$defs": { + "SplunkProfiling": { + "type": ["object", "null"], + "properties": { + "always_on": { + "type": ["object", "null"], + "properties": { + "cpu_profiler": { + "type": ["object", "null"], + "properties": { + "sampling_interval": {"type": "integer", "minimum": 1} + } + } + } + } + }, + "if": { + "$comment": "Exporter settings are used only when continuous CPU profiling is enabled.", + "type": "object", + "required": ["always_on"], + "properties": { + "always_on": {"type": "object", "required": ["cpu_profiler"]} + } + }, + "then": { + "properties": { + "exporter": { + "type": ["object", "null"], + "additionalProperties": false, + "properties": { + "otlp_log_http": { + "anyOf": [ + {"type": "null"}, + {"$ref": "#/$defs/OtlpHttpExporter"} + ] + } + } + } + } + } + } + } +} diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index 9caa08c6..341cefbb 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -108,6 +108,7 @@ def _fake_register_shutdown(callback): {"splunk": {"profiling": {}}}, {"splunk": {"profiling": {"always_on": {}}}}, {"splunk": {"profiling": {"callgraphs": {}}}}, + {"splunk": {"profiling": {"exporter": {"otlp_log_grpc": {}}}}}, ], ) def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profiling_contexts, distribution): @@ -118,9 +119,12 @@ def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profili assert not profiling_contexts -@pytest.mark.parametrize("interval", [0, -1, True, 1.5, "1000", None]) +@pytest.mark.parametrize("interval", [0, -1, True, 1.0, 1.5, "1000", None]) def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval): - with pytest.raises(ConfigurationError, match="sampling_interval must be a positive integer"): + with pytest.raises( + ConfigurationError, + match=r"distribution\.splunk\.profiling\.always_on\.cpu_profiler\.sampling_interval", + ): _configure_declarative_continuous_profiling( {"splunk": {"profiling": {"always_on": {"cpu_profiler": {"sampling_interval": interval}}}}}, Resource({}), @@ -136,7 +140,7 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): for parent in reversed(keys[: keys.index(key)]): config = {parent: config} - with pytest.raises(ConfigurationError, match=f"{key} must be a mapping"): + with pytest.raises(ConfigurationError, match=key): _configure_declarative_continuous_profiling(config, Resource({})) assert not profiling_contexts From 103dca7c6143a2cf989a60c5798c2652222a1c0c Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 13:40:37 -0500 Subject: [PATCH 11/19] Explain strict integer validation and its scope --- src/splunk_otel/_declarative_profiling.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index a6ffb716..ef7bee70 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -20,7 +20,9 @@ ) _PROFILING_SCHEMA = json.loads(files("splunk_otel").joinpath("_profiling_schema.json").read_text(encoding="utf-8")) -# Protobuf sampling periods need Python integers; JSON Schema also accepts 1.0. +# JSON Schema accepts 1.0 as an integer, but protobuf sampling periods require +# Python int values. The exact-type check rejects floats and bools for every +# integer field in this validator, including HTTP exporter settings. _ProfilingValidator = validators.extend( Draft202012Validator, type_checker=Draft202012Validator.TYPE_CHECKER.redefine("integer", lambda _checker, value: type(value) is int), From a772c5c2feb046a449c67372980b7eba34ec5297 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 13:43:10 -0500 Subject: [PATCH 12/19] Keep profiling validator setup local to its factory --- src/splunk_otel/_declarative_profiling.py | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index ef7bee70..e66eec61 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -19,17 +19,20 @@ _start_continuous_profiling, ) -_PROFILING_SCHEMA = json.loads(files("splunk_otel").joinpath("_profiling_schema.json").read_text(encoding="utf-8")) -# JSON Schema accepts 1.0 as an integer, but protobuf sampling periods require -# Python int values. The exact-type check rejects floats and bools for every -# integer field in this validator, including HTTP exporter settings. -_ProfilingValidator = validators.extend( - Draft202012Validator, - type_checker=Draft202012Validator.TYPE_CHECKER.redefine("integer", lambda _checker, value: type(value) is int), -) -_PROFILING_VALIDATOR = _ProfilingValidator( - {**_PROFILING_SCHEMA, "$defs": {**_get_schema()["$defs"], **_PROFILING_SCHEMA["$defs"]}} -) + +def _create_profiling_validator(): + profiling_schema = json.loads(files("splunk_otel").joinpath("_profiling_schema.json").read_text(encoding="utf-8")) + # JSON Schema accepts 1.0 as an integer, but protobuf sampling periods require + # Python int values. The exact-type check rejects floats and bools for every + # integer field in this validator, including HTTP exporter settings. + validator_class = validators.extend( + Draft202012Validator, + type_checker=Draft202012Validator.TYPE_CHECKER.redefine("integer", lambda _checker, value: type(value) is int), + ) + return validator_class({**profiling_schema, "$defs": {**_get_schema()["$defs"], **profiling_schema["$defs"]}}) + + +_PROFILING_VALIDATOR = _create_profiling_validator() def _configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: From b91b869f72238e040e27a99a80e33f79650ed2e5 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 13:56:25 -0500 Subject: [PATCH 13/19] Name the profiling schema composition steps --- src/splunk_otel/_declarative_profiling.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index e66eec61..1e4e4159 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -29,7 +29,11 @@ def _create_profiling_validator(): Draft202012Validator, type_checker=Draft202012Validator.TYPE_CHECKER.redefine("integer", lambda _checker, value: type(value) is int), ) - return validator_class({**profiling_schema, "$defs": {**_get_schema()["$defs"], **profiling_schema["$defs"]}}) + opentelemetry_definitions = _get_schema()["$defs"] + profiling_definitions = profiling_schema["$defs"] + combined_definitions = {**opentelemetry_definitions, **profiling_definitions} + combined_schema = {**profiling_schema, "$defs": combined_definitions} + return validator_class(combined_schema) _PROFILING_VALIDATOR = _create_profiling_validator() From f71a4d2504c55c60e192e4c9d04442b4e79d7d24 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 14:04:29 -0500 Subject: [PATCH 14/19] Distinguish profiling configuration from runtime objects --- src/splunk_otel/_declarative_profiling.py | 32 ++++++++--------- .../declarative_custom_profile_exporter.py | 14 ++++---- .../declarative_default_profile_exporter.py | 20 +++++------ tests/test_profiling_configuration.py | 36 ++++++++++--------- 4 files changed, 52 insertions(+), 50 deletions(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index 1e4e4159..01210cd7 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -39,35 +39,35 @@ def _create_profiling_validator(): _PROFILING_VALIDATOR = _create_profiling_validator() -def _configure_declarative_continuous_profiling(distribution: dict | None, resource: Resource) -> None: - _validate_profiling_config(distribution) - splunk = (distribution or {}).get("splunk") or {} - profiling = splunk.get("profiling") or {} - always_on = profiling.get("always_on") or {} - if "cpu_profiler" not in always_on: +def _configure_declarative_continuous_profiling(distribution_config: dict | None, resource: Resource) -> None: + _validate_profiling_config(distribution_config) + splunk_config = (distribution_config or {}).get("splunk") or {} + profiling_config = splunk_config.get("profiling") or {} + always_on_config = profiling_config.get("always_on") or {} + if "cpu_profiler" not in always_on_config: return - cpu_profiler = always_on["cpu_profiler"] or {} - interval_millis = cpu_profiler.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) + cpu_profiler_config = always_on_config["cpu_profiler"] or {} + interval_millis = cpu_profiler_config.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) - logger_provider = _create_profiling_logger_provider(profiling, resource) + logger_provider = _create_profiling_logger_provider(profiling_config, resource) logger = _get_profiling_logger(logger_provider) _start_continuous_profiling(resource=resource, interval_millis=interval_millis, logger=logger) -def _create_profiling_logger_provider(profiling: dict, resource: Resource) -> LoggerProvider: - exporter = profiling.get("exporter") or {} - http_exporter_config = exporter.get("otlp_log_http") or {} +def _create_profiling_logger_provider(profiling_config: dict, resource: Resource) -> LoggerProvider: + exporter_config = profiling_config.get("exporter") or {} + http_exporter_config = exporter_config.get("otlp_log_http") or {} http_exporter_config = {"endpoint": "http://localhost:4318/v1/logs", **http_exporter_config} logger_provider_data = {"processors": [{"batch": {"exporter": {"otlp_http": http_exporter_config}}}]} logger_provider_config = _dict_to_dataclass(logger_provider_data, LoggerProviderConfig) return create_logger_provider(logger_provider_config, resource) -def _validate_profiling_config(distribution: dict | None) -> None: +def _validate_profiling_config(distribution_config: dict | None) -> None: try: - _PROFILING_VALIDATOR.validate(distribution) + _PROFILING_VALIDATOR.validate(distribution_config) except ValidationError as error: - path = ".".join(["distribution", *(str(part) for part in error.absolute_path)]) - message = f"{path}: {error.message}" + config_path = ".".join(["distribution", *(str(part) for part in error.absolute_path)]) + message = f"{config_path}: {error.message}" raise ConfigurationError(message) from error diff --git a/tests/integration/declarative_custom_profile_exporter.py b/tests/integration/declarative_custom_profile_exporter.py index 3c3bc0d6..efe562fe 100644 --- a/tests/integration/declarative_custom_profile_exporter.py +++ b/tests/integration/declarative_custom_profile_exporter.py @@ -16,9 +16,9 @@ def requirements(self): return (project_path(),) def declarative_configuration(self): - config = Path(project_path(), "docs", "examples", "declarative-config.yaml").read_text() + config_yaml = Path(project_path(), "docs", "examples", "declarative-config.yaml").read_text() return ( - config + config_yaml + """ distribution: splunk: @@ -65,13 +65,13 @@ def on_stop(self, telemetry, stdout, stderr, returncode): self.profile_sink.stop() assert returncode == 0, stderr assert any(record.body.string_value == "application log" for record in get_logs(telemetry)) - application_scopes = extract_leaves(telemetry, "log_requests", "pbreq", "resource_logs", "scope_logs") - assert all(scope.scope.name != "otel.profiling" for scope in application_scopes) + application_scope_logs = extract_leaves(telemetry, "log_requests", "pbreq", "resource_logs", "scope_logs") + assert all(scope.scope.name != "otel.profiling" for scope in application_scope_logs) profile_telemetry = self.profile_handler.telemetry - profile_scopes = extract_leaves(profile_telemetry, "log_requests", "pbreq", "resource_logs", "scope_logs") - assert profile_scopes - assert all(scope.scope.name == "otel.profiling" and scope.log_records for scope in profile_scopes) + profile_scope_logs = extract_leaves(profile_telemetry, "log_requests", "pbreq", "resource_logs", "scope_logs") + assert profile_scope_logs + assert all(scope.scope.name == "otel.profiling" and scope.log_records for scope in profile_scope_logs) for request in profile_telemetry.log_requests: headers = {key.lower(): value for key, value in request.headers.items()} assert headers["x-profile-test"] == "yaml-token" diff --git a/tests/integration/declarative_default_profile_exporter.py b/tests/integration/declarative_default_profile_exporter.py index 248341c4..dcdaed2f 100644 --- a/tests/integration/declarative_default_profile_exporter.py +++ b/tests/integration/declarative_default_profile_exporter.py @@ -51,15 +51,15 @@ def on_start(self): def on_stop(self, telemetry, stdout, stderr, returncode): from oteltest.telemetry import extract_leaves, get_attribute - profile_path = Path(project_path(), "src", "splunk_otel", "profile_pb2.py") - spec = importlib.util.spec_from_file_location("profile_pb2", profile_path) + profile_proto_path = Path(project_path(), "src", "splunk_otel", "profile_pb2.py") + spec = importlib.util.spec_from_file_location("profile_pb2", profile_proto_path) profile_pb2 = importlib.util.module_from_spec(spec) spec.loader.exec_module(profile_pb2) assert returncode == 0, stderr - resources = extract_leaves(telemetry, "log_requests", "pbreq", "resource_logs") - profiles = [] - for resource_logs in resources: + resource_log_groups = extract_leaves(telemetry, "log_requests", "pbreq", "resource_logs") + profile_records = [] + for resource_logs in resource_log_groups: for scope_logs in resource_logs.scope_logs: if scope_logs.scope.name != "otel.profiling": continue @@ -71,10 +71,10 @@ def on_stop(self, telemetry, stdout, stderr, returncode): get_attribute(resource_logs.resource.attributes, "telemetry.distro.name").value.string_value == "splunk-opentelemetry" ) - profiles.extend(scope_logs.log_records) + profile_records.extend(scope_logs.log_records) - assert profiles - for record in profiles: + assert profile_records + for record in profile_records: assert ( get_attribute(record.attributes, "profiling.instrumentation.source").value.string_value == "continuous" ) @@ -82,7 +82,7 @@ def on_stop(self, telemetry, stdout, stderr, returncode): profile.ParseFromString(gzip.decompress(base64.b64decode(record.body.string_value))) assert profile.sample for sample in profile.sample: - (period,) = [ + (sample_period_millis,) = [ label.num for label in sample.label if profile.string_table[label.key] == "source.event.period" ] - assert period == 250 + assert sample_period_millis == 250 diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index 341cefbb..8d14abbe 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -28,26 +28,28 @@ def stop(self): monkeypatch.setattr("splunk_otel.profile.ProfilingContext", FakeProfilingContext) - def _fake_create_logger_provider(_config, _resource): + def _fake_create_logger_provider(_logger_provider_config, _resource): return NoOpLoggerProvider() monkeypatch.setattr("splunk_otel._declarative_profiling.create_logger_provider", _fake_create_logger_provider) return contexts -@pytest.mark.parametrize(("cpu_profiler", "interval"), [({}, 1000), (None, 1000), ({"sampling_interval": 250}, 250)]) -def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_profiler, interval): +@pytest.mark.parametrize( + ("cpu_profiler_config", "interval_millis"), [({}, 1000), (None, 1000), ({"sampling_interval": 250}, 250)] +) +def test_yaml_starts_continuous_profiling(monkeypatch, profiling_contexts, cpu_profiler_config, interval_millis): monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "false") monkeypatch.setenv("SPLUNK_PROFILER_CALL_STACK_INTERVAL", "9999") resource = Resource({"service.name": "yaml-service", "deployment.environment.name": "test"}) _configure_declarative_continuous_profiling( - {"splunk": {"profiling": {"always_on": {"cpu_profiler": cpu_profiler}}}}, resource + {"splunk": {"profiling": {"always_on": {"cpu_profiler": cpu_profiler_config}}}}, resource ) (context,) = profiling_contexts assert context.started - assert context.interval_millis == interval + assert context.interval_millis == interval_millis assert context.resource is resource @@ -100,7 +102,7 @@ def _fake_register_shutdown(callback): @pytest.mark.parametrize( - "distribution", + "distribution_config", [ None, {}, @@ -111,22 +113,22 @@ def _fake_register_shutdown(callback): {"splunk": {"profiling": {"exporter": {"otlp_log_grpc": {}}}}}, ], ) -def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profiling_contexts, distribution): +def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profiling_contexts, distribution_config): monkeypatch.setenv("SPLUNK_PROFILER_ENABLED", "true") - _configure_declarative_continuous_profiling(distribution, Resource({})) + _configure_declarative_continuous_profiling(distribution_config, Resource({})) assert not profiling_contexts -@pytest.mark.parametrize("interval", [0, -1, True, 1.0, 1.5, "1000", None]) -def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval): +@pytest.mark.parametrize("interval_millis", [0, -1, True, 1.0, 1.5, "1000", None]) +def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval_millis): with pytest.raises( ConfigurationError, match=r"distribution\.splunk\.profiling\.always_on\.cpu_profiler\.sampling_interval", ): _configure_declarative_continuous_profiling( - {"splunk": {"profiling": {"always_on": {"cpu_profiler": {"sampling_interval": interval}}}}}, + {"splunk": {"profiling": {"always_on": {"cpu_profiler": {"sampling_interval": interval_millis}}}}}, Resource({}), ) @@ -135,19 +137,19 @@ def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, int @pytest.mark.parametrize("key", ["splunk", "profiling", "always_on", "cpu_profiler"]) def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): - config = {key: False} + distribution_config = {key: False} keys = ["splunk", "profiling", "always_on", "cpu_profiler"] for parent in reversed(keys[: keys.index(key)]): - config = {parent: config} + distribution_config = {parent: distribution_config} with pytest.raises(ConfigurationError, match=key): - _configure_declarative_continuous_profiling(config, Resource({})) + _configure_declarative_continuous_profiling(distribution_config, Resource({})) assert not profiling_contexts @pytest.mark.parametrize( - "exporter", + "exporter_config", [ False, {"otlp_log_grpc": {}}, @@ -158,10 +160,10 @@ def test_invalid_profiling_block_fails_before_starting(profiling_contexts, key): {"otlp_log_http": {"timeout": "invalid"}}, ], ) -def test_invalid_profile_exporter_fails_before_starting(profiling_contexts, exporter): +def test_invalid_profile_exporter_fails_before_starting(profiling_contexts, exporter_config): with pytest.raises(ConfigurationError): _configure_declarative_continuous_profiling( - {"splunk": {"profiling": {"exporter": exporter, "always_on": {"cpu_profiler": {}}}}}, + {"splunk": {"profiling": {"exporter": exporter_config, "always_on": {"cpu_profiler": {}}}}}, Resource({}), ) From 899682e0653760177d16fc71c877283119946096 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 14:11:02 -0500 Subject: [PATCH 15/19] Centralize declarative profiling configuration traversal --- src/splunk_otel/_declarative_profiling.py | 18 ++++++++++++------ tests/test_profiling_configuration.py | 2 ++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index 01210cd7..c5e2321d 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -41,13 +41,12 @@ def _create_profiling_validator(): def _configure_declarative_continuous_profiling(distribution_config: dict | None, resource: Resource) -> None: _validate_profiling_config(distribution_config) - splunk_config = (distribution_config or {}).get("splunk") or {} - profiling_config = splunk_config.get("profiling") or {} - always_on_config = profiling_config.get("always_on") or {} + profiling_config = _config_at(distribution_config, ("splunk", "profiling")) + always_on_config = _config_at(profiling_config, ("always_on",)) if "cpu_profiler" not in always_on_config: return - cpu_profiler_config = always_on_config["cpu_profiler"] or {} + cpu_profiler_config = _config_at(always_on_config, ("cpu_profiler",)) interval_millis = cpu_profiler_config.get("sampling_interval", _DEFAULT_PROF_CALL_STACK_INTERVAL_MILLIS) logger_provider = _create_profiling_logger_provider(profiling_config, resource) @@ -56,14 +55,21 @@ def _configure_declarative_continuous_profiling(distribution_config: dict | None def _create_profiling_logger_provider(profiling_config: dict, resource: Resource) -> LoggerProvider: - exporter_config = profiling_config.get("exporter") or {} - http_exporter_config = exporter_config.get("otlp_log_http") or {} + http_exporter_config = _config_at(profiling_config, ("exporter", "otlp_log_http")) http_exporter_config = {"endpoint": "http://localhost:4318/v1/logs", **http_exporter_config} logger_provider_data = {"processors": [{"batch": {"exporter": {"otlp_http": http_exporter_config}}}]} logger_provider_config = _dict_to_dataclass(logger_provider_data, LoggerProviderConfig) return create_logger_provider(logger_provider_config, resource) +def _config_at(config: dict | None, path: tuple[str, ...]) -> dict: + """Read mapping-valued sections from already validated configuration.""" + section = config or {} + for key in path: + section = section.get(key) or {} + return section + + def _validate_profiling_config(distribution_config: dict | None) -> None: try: _PROFILING_VALIDATOR.validate(distribution_config) diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index 8d14abbe..1e881ce0 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -107,7 +107,9 @@ def _fake_register_shutdown(callback): None, {}, {"splunk": None}, + {"splunk": {"profiling": None}}, {"splunk": {"profiling": {}}}, + {"splunk": {"profiling": {"always_on": None}}}, {"splunk": {"profiling": {"always_on": {}}}}, {"splunk": {"profiling": {"callgraphs": {}}}}, {"splunk": {"profiling": {"exporter": {"otlp_log_grpc": {}}}}}, From bc186c887a3f333c11afe36c506635592a1f660a Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 14:15:43 -0500 Subject: [PATCH 16/19] Use upstream defaults for the profiling export endpoint --- src/splunk_otel/_declarative_profiling.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index c5e2321d..6fefd54a 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -3,6 +3,7 @@ import json from importlib.resources import files +from urllib.parse import urljoin from jsonschema import Draft202012Validator, ValidationError, validators from opentelemetry.configuration import ConfigurationError @@ -10,6 +11,7 @@ from opentelemetry.configuration._logger_provider import create_logger_provider from opentelemetry.configuration.file._loader import _get_schema from opentelemetry.configuration.models import LoggerProvider as LoggerProviderConfig +from opentelemetry.exporter.otlp.proto.http import _log_exporter as otlp_http_logs from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.resources import Resource @@ -56,7 +58,9 @@ def _configure_declarative_continuous_profiling(distribution_config: dict | None def _create_profiling_logger_provider(profiling_config: dict, resource: Resource) -> LoggerProvider: http_exporter_config = _config_at(profiling_config, ("exporter", "otlp_log_http")) - http_exporter_config = {"endpoint": "http://localhost:4318/v1/logs", **http_exporter_config} + default_endpoint = urljoin(otlp_http_logs.DEFAULT_ENDPOINT, otlp_http_logs.DEFAULT_LOGS_EXPORT_PATH) + # An omitted endpoint lets the exporter read environment settings, so supply the SDK default explicitly. + http_exporter_config = {"endpoint": default_endpoint, **http_exporter_config} logger_provider_data = {"processors": [{"batch": {"exporter": {"otlp_http": http_exporter_config}}}]} logger_provider_config = _dict_to_dataclass(logger_provider_data, LoggerProviderConfig) return create_logger_provider(logger_provider_config, resource) From e9cc4a2b9433c7ee27c03f5ca5e60977c042ada1 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 14:27:38 -0500 Subject: [PATCH 17/19] Fix profiling endpoint defaults and interrupt shutdown waits --- src/splunk_otel/_declarative_profiling.py | 5 ++-- src/splunk_otel/profile.py | 5 +++- tests/test_profile.py | 14 ++++++++++ tests/test_profiling_configuration.py | 34 +++++++++++++++++++++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/splunk_otel/_declarative_profiling.py b/src/splunk_otel/_declarative_profiling.py index 6fefd54a..15abe617 100644 --- a/src/splunk_otel/_declarative_profiling.py +++ b/src/splunk_otel/_declarative_profiling.py @@ -59,8 +59,9 @@ def _configure_declarative_continuous_profiling(distribution_config: dict | None def _create_profiling_logger_provider(profiling_config: dict, resource: Resource) -> LoggerProvider: http_exporter_config = _config_at(profiling_config, ("exporter", "otlp_log_http")) default_endpoint = urljoin(otlp_http_logs.DEFAULT_ENDPOINT, otlp_http_logs.DEFAULT_LOGS_EXPORT_PATH) - # An omitted endpoint lets the exporter read environment settings, so supply the SDK default explicitly. - http_exporter_config = {"endpoint": default_endpoint, **http_exporter_config} + # Missing or null endpoints must use the SDK default rather than environment settings. + endpoint = http_exporter_config.get("endpoint") or default_endpoint + http_exporter_config = {**http_exporter_config, "endpoint": endpoint} logger_provider_data = {"processors": [{"batch": {"exporter": {"otlp_http": http_exporter_config}}}]} logger_provider_config = _dict_to_dataclass(logger_provider_data, LoggerProviderConfig) return create_logger_provider(logger_provider_config, resource) diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index a18300d2..b5b85bf2 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -276,6 +276,7 @@ def __init__(self, interval_millis, target): self.running = False self.pause_at = None self.wakeup_event = threading.Event() + self._stop_event = threading.Event() def start(self): self.pause_at = None @@ -284,6 +285,7 @@ def start(self): if self.running: return + self._stop_event.clear() self.running = True self.thread.start() @@ -299,11 +301,12 @@ def _loop(self): self.target() elapsed_seconds = time.monotonic() - start_time_seconds sleep_seconds = max(0, self.interval_seconds - elapsed_seconds) - time.sleep(sleep_seconds) + self._stop_event.wait(sleep_seconds) def stop(self): self.running = False self.pause_at = None + self._stop_event.set() self.wakeup_event.set() # unblock _loop() if waiting, so it can return and thread can exit if self.thread.is_alive(): self.thread.join() diff --git a/tests/test_profile.py b/tests/test_profile.py index 729403b1..5890d1a9 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -3,6 +3,7 @@ import json import random import time +from threading import Event, Thread from collections import OrderedDict from os.path import abspath, dirname @@ -180,6 +181,19 @@ def test_interval_timer_pause_and_resume(): timer.stop() +def test_interval_timer_stop_interrupts_long_sampling_interval(): + sampled = Event() + timer = _IntervalTimer(60000, sampled.set) + timer.start() + assert sampled.wait(timeout=2) + + stop_thread = Thread(target=timer.stop, daemon=True) + stop_thread.start() + stop_thread.join(timeout=2) + + assert not stop_thread.is_alive(), "stop must interrupt the sampling wait" + + class _FakeLogger(Logger): def __init__(self): super().__init__("fake-logger") diff --git a/tests/test_profiling_configuration.py b/tests/test_profiling_configuration.py index 1e881ce0..f710a7c7 100644 --- a/tests/test_profiling_configuration.py +++ b/tests/test_profiling_configuration.py @@ -9,7 +9,12 @@ @pytest.fixture -def profiling_contexts(monkeypatch): +def logger_provider_configs(): + return [] + + +@pytest.fixture +def profiling_contexts(monkeypatch, logger_provider_configs): contexts = [] class FakeProfilingContext: @@ -28,7 +33,8 @@ def stop(self): monkeypatch.setattr("splunk_otel.profile.ProfilingContext", FakeProfilingContext) - def _fake_create_logger_provider(_logger_provider_config, _resource): + def _fake_create_logger_provider(logger_provider_config, _resource): + logger_provider_configs.append(logger_provider_config) return NoOpLoggerProvider() monkeypatch.setattr("splunk_otel._declarative_profiling.create_logger_provider", _fake_create_logger_provider) @@ -123,6 +129,30 @@ def test_yaml_without_cpu_profiler_does_not_start_profiling(monkeypatch, profili assert not profiling_contexts +@pytest.mark.parametrize("http_exporter_config", [None, {}, {"endpoint": None}, {"endpoint": ""}]) +def test_unset_profile_endpoint_uses_sdk_default( + monkeypatch, profiling_contexts, logger_provider_configs, http_exporter_config +): + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:1/base") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "http://localhost:1/logs") + + _configure_declarative_continuous_profiling( + { + "splunk": { + "profiling": { + "always_on": {"cpu_profiler": {}}, + "exporter": {"otlp_log_http": http_exporter_config}, + } + } + }, + Resource({}), + ) + + (logger_provider_config,) = logger_provider_configs + assert logger_provider_config.processors[0].batch.exporter.otlp_http.endpoint == "http://localhost:4318/v1/logs" + assert profiling_contexts[0].started + + @pytest.mark.parametrize("interval_millis", [0, -1, True, 1.0, 1.5, "1000", None]) def test_invalid_sampling_interval_fails_before_starting(profiling_contexts, interval_millis): with pytest.raises( From 44781c046179216a12fc487236694e39b503ea7e Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 14:44:53 -0500 Subject: [PATCH 18/19] Test profiling shutdown ordering and fix lost pause wakeups --- src/splunk_otel/profile.py | 10 ++- tests/test_profile.py | 175 +++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 3 deletions(-) diff --git a/src/splunk_otel/profile.py b/src/splunk_otel/profile.py index b5b85bf2..b1b663ba 100644 --- a/src/splunk_otel/profile.py +++ b/src/splunk_otel/profile.py @@ -293,9 +293,13 @@ def _loop(self): while self.running: start_time_seconds = time.monotonic() - if self.pause_at is not None and start_time_seconds >= self.pause_at: - self.wakeup_event.clear() # clear event so next line will block - self.wakeup_event.wait() # wait for event.set() (either via start() or stop()) + pause_at = self.pause_at + if pause_at is not None and start_time_seconds >= pause_at: + self.wakeup_event.clear() + # Recheck state because clear() can erase a concurrent stop or resume signal. + if not self.running or self.pause_at is None: + continue + self.wakeup_event.wait() continue self.target() diff --git a/tests/test_profile.py b/tests/test_profile.py index 5890d1a9..a25bdb2c 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -12,6 +12,8 @@ from opentelemetry._logs import Logger from opentelemetry.context import attach, detach from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogRecordExportResult from opentelemetry.trace import ( NonRecordingSpan, SpanContext, @@ -26,6 +28,7 @@ _pb_profile_to_str, _ProfileScraper, _stacktraces_to_cpu_profile, + _start_continuous_profiling, _StringTable, ) @@ -194,6 +197,178 @@ def test_interval_timer_stop_interrupts_long_sampling_interval(): assert not stop_thread.is_alive(), "stop must interrupt the sampling wait" +@pytest.mark.parametrize("pause_state", ["paused", "entering_pause"]) +@pytest.mark.parametrize("pause_action", ["stop", "resume"]) +def test_interval_timer_stop_or_resume_during_pause(monkeypatch, pause_state, pause_action): + pause_reached = Event() + release_pause = Event() + wakeup_requested = Event() + resumed = Event() + + def _fake_sample(): + if pause_reached.is_set(): + resumed.set() + else: + timer.pause_after(0) + + timer = _IntervalTimer(0, _fake_sample) + wakeup_clear = timer.wakeup_event.clear + wakeup_wait = timer.wakeup_event.wait + wakeup_set = timer.wakeup_event.set + + def _fake_wakeup_clear(): + if pause_state == "entering_pause": + pause_reached.set() + assert release_pause.wait(timeout=5) + wakeup_clear() + + def _fake_wakeup_wait(): + if pause_state == "paused": + pause_reached.set() + return wakeup_wait() + + def _fake_wakeup_set(): + wakeup_set() + wakeup_requested.set() + + monkeypatch.setattr(timer.wakeup_event, "clear", _fake_wakeup_clear) + monkeypatch.setattr(timer.wakeup_event, "wait", _fake_wakeup_wait) + monkeypatch.setattr(timer.wakeup_event, "set", _fake_wakeup_set) + timer.start() + try: + assert pause_reached.wait(timeout=2) + wakeup_requested.clear() + if pause_action == "stop": + stop_thread = Thread(target=timer.stop, daemon=True) + stop_thread.start() + assert wakeup_requested.wait(timeout=2) + release_pause.set() + stop_thread.join(timeout=2) + assert not stop_thread.is_alive(), "stop must wake a paused sampler" + else: + timer.start() + release_pause.set() + assert resumed.wait(timeout=2), "resume must wake a paused sampler" + finally: + release_pause.set() + wakeup_set() + timer.stop() + + +def test_interval_timer_stop_waits_for_current_sample(monkeypatch): + sample_started = Event() + release_sample = Event() + stop_waiting = Event() + lifecycle = [] + + def _fake_sample(): + sample_started.set() + assert release_sample.wait(timeout=5) + lifecycle.append("sample completed") + + timer = _IntervalTimer(60000, _fake_sample) + sampler_join = timer.thread.join + + def _fake_sampler_join(timeout=None): + stop_waiting.set() + sampler_join(timeout) + + def _stop_timer(): + timer.stop() + lifecycle.append("stop completed") + + monkeypatch.setattr(timer.thread, "join", _fake_sampler_join) + timer.start() + try: + assert sample_started.wait(timeout=2) + stop_thread = Thread(target=_stop_timer, daemon=True) + stop_thread.start() + assert stop_waiting.wait(timeout=2) + assert stop_thread.is_alive() + assert not lifecycle + release_sample.set() + stop_thread.join(timeout=2) + assert not stop_thread.is_alive() + assert lifecycle == ["sample completed", "stop completed"] + finally: + release_sample.set() + timer.stop() + + +def test_exit_flushes_final_profile_after_sampling_stops(monkeypatch): + shutdown_callbacks = [] + sample_started = Event() + release_sample = Event() + stop_waiting = Event() + exported_records = [] + lifecycle = [] + timers = [] + + def _fake_register_shutdown(callback): + shutdown_callbacks.append(callback) + return callback + + def _fake_interval_timer(interval_millis, target): + timer = _IntervalTimer(interval_millis, target) + timers.append(timer) + return timer + + class FakeLogExporter: + def export(self, records): + exported_records.extend(records) + lifecycle.append("sample exported") + return LogRecordExportResult.SUCCESS + + def shutdown(self): + lifecycle.append("exporter shut down") + + monkeypatch.setattr("splunk_otel.profile._atexit.register", _fake_register_shutdown) + monkeypatch.setattr("splunk_otel.profile._IntervalTimer", _fake_interval_timer) + resource = Resource({"service.name": "final-profile-test"}) + provider = LoggerProvider(resource=resource) + provider.add_log_record_processor(BatchLogRecordProcessor(FakeLogExporter(), schedule_delay_millis=60000)) + logger = provider.get_logger("otel.profiling") + + class FakeProfilingLogger: + def emit(self, record): + sample_started.set() + assert release_sample.wait(timeout=5) + logger.emit(record) + lifecycle.append("sample emitted") + + context = _start_continuous_profiling(resource=resource, interval_millis=60000, logger=FakeProfilingLogger()) + (timer,) = timers + sampler_join = timer.thread.join + + def _fake_sampler_join(timeout=None): + stop_waiting.set() + sampler_join(timeout) + + def _release_final_sample(): + if stop_waiting.wait(timeout=5): + release_sample.set() + + monkeypatch.setattr(timer.thread, "join", _fake_sampler_join) + release_thread = Thread(target=_release_final_sample, daemon=True) + release_thread.start() + try: + assert sample_started.wait(timeout=2) + assert not exported_records + for shutdown in reversed(shutdown_callbacks): + shutdown() + release_thread.join(timeout=2) + assert not release_thread.is_alive() + assert lifecycle == ["sample emitted", "sample exported", "exporter shut down"] + (record,) = exported_records + assert record.log_record.attributes["profiling.instrumentation.source"] == "continuous" + assert _pb_profile_from_str(record.log_record.body).sample + finally: + release_sample.set() + context.stop() + provider.shutdown() + release_thread.join(timeout=2) + + class _FakeLogger(Logger): def __init__(self): super().__init__("fake-logger") From 2731b423f2b0098d82b8d685eaabe1566799cfa4 Mon Sep 17 00:00:00 2001 From: Pablo Collins Date: Fri, 6 Nov 2026 14:59:51 -0500 Subject: [PATCH 19/19] Replace sleep-based pause coverage with event assertions --- tests/test_profile.py | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/tests/test_profile.py b/tests/test_profile.py index a25bdb2c..20e70af3 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -162,28 +162,6 @@ def test_interval_timer_stop_before_start(): timer.stop() -def test_interval_timer_pause_and_resume(): - # pause_after() should halt ticking; start() should resume it. - ticks = [] - timer = _IntervalTimer(20, lambda: ticks.append(1)) - timer.start() - - time.sleep(0.1) - assert len(ticks) > 0, "timer should have ticked before pause" - - timer.pause_after(0) - time.sleep(0.2) # enough for the timer to reach pause_at and block - count_at_pause = len(ticks) - time.sleep(0.1) # confirm no new ticks while paused - assert len(ticks) == count_at_pause, "timer should not tick while paused" - - timer.start() - time.sleep(0.1) # let it resume - assert len(ticks) > count_at_pause, "timer should tick again after start()" - - timer.stop() - - def test_interval_timer_stop_interrupts_long_sampling_interval(): sampled = Event() timer = _IntervalTimer(60000, sampled.set) @@ -203,10 +181,14 @@ def test_interval_timer_stop_or_resume_during_pause(monkeypatch, pause_state, pa pause_reached = Event() release_pause = Event() wakeup_requested = Event() + resume_requested = Event() + sampled_while_paused = Event() resumed = Event() def _fake_sample(): if pause_reached.is_set(): + if not resume_requested.is_set(): + sampled_while_paused.set() resumed.set() else: timer.pause_after(0) @@ -237,6 +219,7 @@ def _fake_wakeup_set(): timer.start() try: assert pause_reached.wait(timeout=2) + assert not resumed.is_set(), "sampling must stay paused until resume is requested" wakeup_requested.clear() if pause_action == "stop": stop_thread = Thread(target=timer.stop, daemon=True) @@ -246,9 +229,11 @@ def _fake_wakeup_set(): stop_thread.join(timeout=2) assert not stop_thread.is_alive(), "stop must wake a paused sampler" else: + resume_requested.set() timer.start() release_pause.set() assert resumed.wait(timeout=2), "resume must wake a paused sampler" + assert not sampled_while_paused.is_set(), "sampling must stay paused until resume is requested" finally: release_pause.set() wakeup_set()