From 6a484d425095f4b81e65a2202754b179d27cd356 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:24:24 -0400 Subject: [PATCH 01/90] test: the station protocol tests no longer test serde One of them built RegisterRunner without the metadata field main added, so the station runner's tests stopped compiling. It, the snake_case finish status and the decoding of known and malformed event data all checked what serde writes or accepts, and they are gone. An unknown event still decodes to nothing, which the runner decides. --- .../src/station/infrastructure/protocol.rs | 79 ------------------- 1 file changed, 79 deletions(-) diff --git a/crates/tracel-runtime/src/station/infrastructure/protocol.rs b/crates/tracel-runtime/src/station/infrastructure/protocol.rs index 2e5af8e4..b602a7a9 100644 --- a/crates/tracel-runtime/src/station/infrastructure/protocol.rs +++ b/crates/tracel-runtime/src/station/infrastructure/protocol.rs @@ -112,87 +112,8 @@ impl RunnerEvent { mod tests { use super::*; - #[test] - fn given_known_events_when_decoding_then_yields_them() { - let id = Uuid::nil(); - - let registered = RunnerEvent::decode("registered", &format!("{{\"runner_id\":\"{id}\"}}")) - .unwrap() - .unwrap(); - let job = RunnerEvent::decode( - "job", - &format!("{{\"id\":\"{id}\",\"job_name\":\"train\",\"input\":{{\"epochs\":2}}}}"), - ) - .unwrap() - .unwrap(); - let cancel = RunnerEvent::decode("cancel", &format!("{{\"job_id\":\"{id}\"}}")) - .unwrap() - .unwrap(); - - assert_eq!(registered, RunnerEvent::Registered { runner_id: id }); - assert_eq!(cancel, RunnerEvent::Cancel { job_id: id }); - assert_eq!( - job, - RunnerEvent::Job(DispatchedJob { - id, - job_name: "train".to_string(), - input: serde_json::json!({"epochs": 2}), - }) - ); - } - #[test] fn given_unknown_event_when_decoding_then_yields_none() { assert_eq!(RunnerEvent::decode("heartbeat", "{}").unwrap(), None); } - - #[test] - fn given_malformed_data_when_decoding_then_errors() { - assert!(RunnerEvent::decode("job", "not json").is_err()); - assert!(RunnerEvent::decode("cancel", "{}").is_err()); - } - - #[test] - fn a_registration_gives_each_job_its_name_description_and_input() { - let definition = JobDefinition { - name: "train".to_string(), - description: Some("Train the model".to_string()), - input_schema: Some(serde_json::json!({"type": "integer"})), - input_example: Some(serde_json::json!(3)), - }; - - let register = RegisterRunner { - name: Some("runner".to_string()), - jobs: vec![RegisteredJob::from(&definition)], - }; - - assert_eq!( - serde_json::to_value(register).unwrap(), - serde_json::json!({ - "name": "runner", - "jobs": [{ - "name": "train", - "description": "Train the model", - "input_schema": {"type": "integer"}, - "input_example": 3 - }] - }) - ); - } - - #[test] - fn a_finish_gives_its_status_in_snake_case() { - let finish = |status| { - serde_json::to_value(FinishJob { - runner_id: Uuid::nil(), - status, - reason: None, - }) - .unwrap()["status"] - .clone() - }; - - assert_eq!(finish(FinishStatus::Completed), "completed"); - assert_eq!(finish(FinishStatus::Failed), "failed"); - } } From 2522fa03ecc9733a96dc51c03ba05ecb574c1920 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:02:51 -0400 Subject: [PATCH 02/90] chore: rstest and relative-path are workspace dependencies The crates' table tests move to rstest cases and fixtures, and the offline store checks its path names with relative-path instead of a hand-rolled check. --- Cargo.lock | 1 + Cargo.toml | 2 ++ crates/tracel-runtime/Cargo.toml | 1 + 3 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 6134d814..862ae549 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7400,6 +7400,7 @@ dependencies = [ "clap_complete", "json-patch", "reqwest", + "rstest", "schemars 1.2.2", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 14bbd594..6457925e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,8 @@ categories = ["development-tools"] burn = { version = "0.22", default-features = false, features = [ "train" ] } rand = "0.9.2" +relative-path = "1.9.3" +rstest = "0.27.0" tempfile = "3.27.0" url = "2.5.8" diff --git a/crates/tracel-runtime/Cargo.toml b/crates/tracel-runtime/Cargo.toml index 7f8561a3..1fde7524 100644 --- a/crates/tracel-runtime/Cargo.toml +++ b/crates/tracel-runtime/Cargo.toml @@ -31,6 +31,7 @@ url = { workspace = true, optional = true } uuid = { workspace = true, optional = true } [dev-dependencies] +rstest = { workspace = true } serde = { workspace = true, features = ["std"] } tempfile.workspace = true uuid = { workspace = true, features = ["v4"] } From c8ca770810c93576b5475bbc91778a888984e669 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:29:10 -0400 Subject: [PATCH 03/90] refactor: a job's definition is tracel-job's JobDefinition tracel-runtime declared its own JobDefinition with the same four fields as tracel_job::JobDefinition, and the process adapter only converted one into the other to build the command line and the definitions file. The crate now re-exports tracel-job's type, and the conversion is gone. --- crates/tracel-runtime/src/job.rs | 16 +--------------- crates/tracel-runtime/src/jobs.rs | 3 ++- crates/tracel-runtime/src/lib.rs | 3 ++- crates/tracel-runtime/src/process/describe.rs | 11 ++--------- crates/tracel-runtime/src/process/mod.rs | 16 ++-------------- 5 files changed, 9 insertions(+), 40 deletions(-) diff --git a/crates/tracel-runtime/src/job.rs b/crates/tracel-runtime/src/job.rs index 9c04732a..8c1b871b 100644 --- a/crates/tracel-runtime/src/job.rs +++ b/crates/tracel-runtime/src/job.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; +use tracel_job::JobDefinition; /// The error a job fails with. pub type BoxError = Box; @@ -77,21 +78,6 @@ where } } -/// What a transport reads about a registered job: its name and description, and the input it -/// takes. -#[derive(Debug, Clone, PartialEq)] -pub struct JobDefinition { - /// The name that selects the job. - pub name: String, - /// What the job does, as [`Job::with_description`] gives it. - pub description: Option, - /// The JSON Schema of the job's input, when it was given one (requires the `schema` feature). - pub input_schema: Option, - /// The job's default input, which its input is merged onto, as [`Job::with_default`] gives - /// it. - pub input_example: Option, -} - /// What [`Jobs::job`](crate::Jobs::job) registers: a [`Job`], or a function of the job's input. pub trait IntoJob { /// The job. diff --git a/crates/tracel-runtime/src/jobs.rs b/crates/tracel-runtime/src/jobs.rs index 092622d2..c0f8b9f2 100644 --- a/crates/tracel-runtime/src/jobs.rs +++ b/crates/tracel-runtime/src/jobs.rs @@ -4,9 +4,10 @@ use std::collections::BTreeMap; use serde::de::DeserializeOwned; use serde_json::Value; +use tracel_job::JobDefinition; use crate::execution::{self, PreparedJob}; -use crate::job::{IntoJob, JobDefinition, Prepared, Registered}; +use crate::job::{IntoJob, Prepared, Registered}; /// A program's jobs, by name, and how the program runs the one its command line names: `main` /// returns [`run`](Self::run). diff --git a/crates/tracel-runtime/src/lib.rs b/crates/tracel-runtime/src/lib.rs index b7a607ea..2b0d4af5 100644 --- a/crates/tracel-runtime/src/lib.rs +++ b/crates/tracel-runtime/src/lib.rs @@ -85,6 +85,7 @@ mod scope; pub mod station; pub use execution::{Execution, JobStatus, Outcome, PreparedJob}; -pub use job::{BoxError, IntoJob, Job, JobDefinition}; +pub use job::{BoxError, IntoJob, Job}; pub use jobs::{Jobs, PrepareError}; pub use scope::{JobScope, job_input, job_num, job_running}; +pub use tracel_job::JobDefinition; diff --git a/crates/tracel-runtime/src/process/describe.rs b/crates/tracel-runtime/src/process/describe.rs index d3f8dbea..906075c6 100644 --- a/crates/tracel-runtime/src/process/describe.rs +++ b/crates/tracel-runtime/src/process/describe.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use tracel_job::{DefinitionsFile, TRACEL_DESCRIBE}; -use super::protocol_definition; use crate::JobDefinition; use crate::process::file::write_atomically; @@ -22,7 +21,7 @@ pub fn describe<'a>( return Ok(false); }; let path = PathBuf::from(path); - let file = DefinitionsFile::new(definitions.into_iter().map(protocol_definition).collect()); + let file = DefinitionsFile::new(definitions.into_iter().cloned().collect()); write_atomically(&path, &file.to_json()).map_err(|source| DescribeError { path, source })?; Ok(true) } @@ -97,13 +96,7 @@ mod tests { }) ); let read = DefinitionsFile::from_json(contents.as_bytes()).unwrap(); - assert_eq!( - read.jobs, - definitions() - .iter() - .map(protocol_definition) - .collect::>() - ); + assert_eq!(read.jobs, definitions()); assert_eq!(contents, read.to_json()); } diff --git a/crates/tracel-runtime/src/process/mod.rs b/crates/tracel-runtime/src/process/mod.rs index 7975c503..f432452f 100644 --- a/crates/tracel-runtime/src/process/mod.rs +++ b/crates/tracel-runtime/src/process/mod.rs @@ -16,7 +16,7 @@ use serde_json::Value; use tracel_job::TRACEL_JOB_NUM; use tracel_job::cli::{command, completions, job_input}; -use crate::{Execution, JobDefinition, Jobs}; +use crate::{Execution, Jobs}; use describe::describe; use error::RunError; @@ -83,25 +83,13 @@ fn execute(jobs: &Jobs) -> Result<(), RunError> { /// The command line of the program `name`: the [`command`] of `jobs`, with `--version` when the /// program gives its version. fn command_line(jobs: &Jobs, name: impl Into) -> Command { - let definitions: Vec<_> = jobs.definitions().map(protocol_definition).collect(); - let command = command(name, &definitions); + let command = command(name, jobs.definitions()); match jobs.get_version() { Some(version) => command.version(version.to_string()), None => command, } } -/// The job `definition` defines, as the protocol gives it: in the definitions file, and to -/// generate its command line from. -fn protocol_definition(definition: &JobDefinition) -> tracel_job::JobDefinition { - tracel_job::JobDefinition { - name: definition.name.clone(), - description: definition.description.clone(), - input_schema: definition.input_schema.clone(), - input_example: definition.input_example.clone(), - } -} - /// The names of the jobs of `jobs`, in order. fn names(jobs: &Jobs) -> Vec { jobs.definitions() From 7205dd89d5b1a72979945da203bfc5d7f99b970c Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:31:42 -0400 Subject: [PATCH 04/90] test: the process adapter is tested through execute itself The tests ran a command line through run_launched, a copy of execute, so execute was never tested: that it writes the definitions before reading the command line, reads TRACEL_JOB_NUM from the environment, fails with exit code 1 when the definitions cannot be written, and succeeds after printing. execute now takes the command-line arguments and the environment lookup, Jobs::run gives it the process's own, and the tests call it directly. The copy is gone. --- crates/tracel-runtime/src/process/mod.rs | 128 ++++++++++++++--------- 1 file changed, 76 insertions(+), 52 deletions(-) diff --git a/crates/tracel-runtime/src/process/mod.rs b/crates/tracel-runtime/src/process/mod.rs index f432452f..95d22aa6 100644 --- a/crates/tracel-runtime/src/process/mod.rs +++ b/crates/tracel-runtime/src/process/mod.rs @@ -47,7 +47,7 @@ impl Jobs { /// [`job_num`](crate::job_num) gives the number its launcher gave it, and /// [`job_input`](crate::job_input) the input it runs with. pub fn run(self) -> ExitCode { - let exit = match execute(&self) { + let exit = match execute(&self, std::env::args_os(), |name| std::env::var_os(name)) { Ok(()) => tracel_job::ExitCode::Completed, Err(error) => { error.print(); @@ -58,15 +58,20 @@ impl Jobs { } } -/// Does what the process's environment and command line ask of `jobs`: writes their definitions, -/// prints the help, the version or a completion script, or runs the job named. -fn execute(jobs: &Jobs) -> Result<(), RunError> { - let vars = |name: &str| std::env::var_os(name); - if describe(jobs.definitions(), vars)? { +/// Does what the command-line arguments `args`, the program's name first, and the environment +/// variables `vars` looks up ask of `jobs`: writes their definitions, prints the help, the version +/// or a completion script, or runs the job named. +fn execute( + jobs: &Jobs, + args: impl IntoIterator, + vars: impl Fn(&str) -> Option, +) -> Result<(), RunError> { + if describe(jobs.definitions(), &vars)? { return Ok(()); } - let mut command = command_line(jobs, program_name()); - match read_request(jobs, &mut command, std::env::args_os())? { + let args: Vec = args.into_iter().collect(); + let mut command = command_line(jobs, program_name(args.first())); + match read_request(jobs, &mut command, args)? { Request::Run { job, input } => dispatch(jobs, &job, input, execution(vars)), Request::Completions(shell) => { let name = command.get_name().to_string(); @@ -154,11 +159,9 @@ fn dispatch(jobs: &Jobs, name: &str, input: Value, execution: Execution) -> Resu .map_err(RunError::JobFailed) } -/// The name the program was run as, without its extension, or `job` when it has none. -fn program_name() -> String { - std::env::args_os() - .next() - .as_deref() +/// The name the program was run as, `program`, without its extension, or `job` when it has none. +fn program_name(program: Option<&OsString>) -> String { + program .map(Path::new) .and_then(Path::file_stem) .map(|name| name.to_string_lossy().into_owned()) @@ -167,14 +170,14 @@ fn program_name() -> String { #[cfg(test)] mod tests { - use std::ffi::OsStr; use std::sync::{Arc, Mutex}; use serde::{Deserialize, Serialize}; use serde_json::json; + use tracel_job::{DefinitionsFile, TRACEL_DESCRIBE}; use super::*; - use crate::{BoxError, IntoJob, Job, job_input, job_num, job_running}; + use crate::{BoxError, IntoJob, Job, job_input, job_num}; #[derive(Clone, Copy)] enum Outcome { @@ -251,27 +254,59 @@ mod tests { read_request(jobs, &mut command, ["program"].iter().chain(args)) } - /// Runs the command line `program ` as [`Jobs::run`] does, printing nothing. + fn arguments(args: &[&str]) -> impl Iterator { + ["program"].iter().chain(args).map(OsString::from) + } + + /// The environment variable `name` set to `value`, and no other. + fn var(name: &'static str, value: impl Into) -> impl Fn(&str) -> Option { + let value = value.into(); + move |asked| (asked == name).then(|| value.clone()) + } + + /// Runs the command line `program ` with no environment variables set. fn run(jobs: Jobs, args: &[&str]) -> Result<(), RunError> { - run_launched(jobs, args, &[]) - } - - /// Runs the command line `program ` as [`run`] does, launched with the environment - /// variables `vars`. - fn run_launched(jobs: Jobs, args: &[&str], vars: &[(&str, &OsStr)]) -> Result<(), RunError> { - match parse(&jobs, args)? { - Request::Run { job, input } => { - let execution = execution(|name| { - vars.iter() - .find(|(variable, _)| *variable == name) - .map(|(_, value)| value.to_os_string()) - }); - dispatch(&jobs, &job, input, execution) - } - Request::Completions(_) | Request::Print(_) => Ok(()), + execute(&jobs, arguments(args), |_| None) + } + + #[test] + fn a_launcher_asking_for_the_definitions_gets_them_and_no_job_runs() { + for args in [&[][..], &["train"], &["--bogus"]] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.json"); + let jobs = ending(Outcome::Fail).0; + + execute(&jobs, arguments(args), var(TRACEL_DESCRIBE, &path)).unwrap(); + + let file = DefinitionsFile::from_json(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!( + file.jobs, + jobs.definitions().cloned().collect::>(), + "{args:?}" + ); } } + #[test] + fn definitions_that_cannot_be_written_fail_the_run_with_exit_code_1() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("missing").join("jobs.json"); + let jobs = ending(Outcome::Succeed).0; + + let failed = + execute(&jobs, arguments(&["train"]), var(TRACEL_DESCRIBE, &path)).unwrap_err(); + + assert!(matches!(failed, RunError::Describe(_)), "{failed:?}"); + assert_eq!(failed.exit_code().code(), 1); + } + + #[test] + fn printing_the_version_ends_the_run_successfully() { + let jobs = ending(Outcome::Fail).0.version("1.2.3"); + + execute(&jobs, arguments(&["--version"]), |_| None).unwrap(); + } + #[test] fn the_named_job_runs_with_its_json_input() { let (jobs, ran_with) = ending(Outcome::Succeed); @@ -521,28 +556,17 @@ mod tests { } #[test] - fn the_job_is_the_running_job_until_it_ends() { - let given = Arc::new(Mutex::new(Vec::new())); - let numbered = |num: &str| { - let jobs = Jobs::new().job("train", { - let given = given.clone(); - move |()| { - given.lock().unwrap().push((job_running(), job_num())); - Ok::<_, BoxError>(()) - } + fn the_job_is_numbered_by_tracel_job_num_when_its_launcher_sets_it() { + for (num, expected) in [(Some("12"), Some("12")), (Some(""), None), (None, None)] { + let jobs = Jobs::new().job("train", move |()| match job_num() { + seen if seen.as_deref() == expected => Ok(()), + seen => Err::<(), BoxError>(format!("numbered {seen:?}").into()), }); - run_launched(jobs, &["train"], &[(TRACEL_JOB_NUM, OsStr::new(num))]).unwrap(); - }; + let vars = move |name: &str| num.filter(|_| name == TRACEL_JOB_NUM).map(OsString::from); - numbered("12"); - numbered(""); - let after = (job_running(), job_num()); - - assert_eq!( - *given.lock().unwrap(), - [(true, Some("12".to_string())), (true, None)] - ); - assert_eq!(after, (false, None)); + execute(&jobs, arguments(&["train"]), vars) + .unwrap_or_else(|error| panic!("{num:?}: {error}")); + } } #[test] From 95bd16608ac51449885f1daaa1cba463c375d9f5 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:32:21 -0400 Subject: [PATCH 05/90] test: Jobs::prepare and job registration are tested in jobs.rs The Jobs::prepare tests sat in execution.rs and the duplicate-name test in the process adapter, away from the unit that decides them. They now live in jobs.rs, around one rstest fixture of two jobs. The prepare test no longer re-asserts the merge onto the default, which job.rs decides, only that the prepared job carries its name and the input it runs with. An unknown job and an input that does not decode are each refused with their own PrepareError variant, the unknown job with the registered names in order. --- crates/tracel-runtime/src/execution.rs | 42 +----------------- crates/tracel-runtime/src/jobs.rs | 54 ++++++++++++++++++++++++ crates/tracel-runtime/src/process/mod.rs | 7 --- 3 files changed, 55 insertions(+), 48 deletions(-) diff --git a/crates/tracel-runtime/src/execution.rs b/crates/tracel-runtime/src/execution.rs index 0239a6b1..a1a26ad1 100644 --- a/crates/tracel-runtime/src/execution.rs +++ b/crates/tracel-runtime/src/execution.rs @@ -111,7 +111,7 @@ mod tests { use serde_json::json; use super::*; - use crate::{Job, Jobs, PrepareError, job_input, job_num, job_running}; + use crate::{Job, Jobs, job_input, job_num, job_running}; /// `train`, which takes a number of epochs, three by default, and fails on zero; and `panic`, /// which panics. @@ -128,46 +128,6 @@ mod tests { .job("panic", |()| -> Result<(), BoxError> { panic!("kaboom") }) } - #[test] - fn a_job_is_prepared_with_its_input_merged_onto_its_default() { - let jobs = Jobs::new().job( - "train", - Job::new(|_: Value| Ok::<_, BoxError>(())) - .with_default(json!({"epochs": 3, "lr": 0.1})), - ); - - let prepared = jobs.prepare("train", json!({"epochs": 5})).unwrap(); - - assert_eq!(prepared.name(), "train"); - assert_eq!(prepared.input(), &json!({"epochs": 5, "lr": 0.1})); - } - - #[test] - fn an_unknown_job_or_an_input_that_does_not_decode_is_not_prepared() { - let jobs = jobs(); - - let unknown = jobs.prepare("infer", Value::Null).err().unwrap(); - let invalid = jobs.prepare("train", json!("ten")).err().unwrap(); - - assert!( - matches!( - &unknown, - PrepareError::UnknownJob { name, available } - if name == "infer" && available == &["panic", "train"] - ), - "{unknown:?}" - ); - assert_eq!(unknown.to_string(), "unknown job 'infer'"); - assert!( - matches!(invalid, PrepareError::InvalidInput(_)), - "{invalid:?}" - ); - assert!( - invalid.to_string().starts_with("invalid input: "), - "{invalid}" - ); - } - #[test] fn a_job_completes_or_fails_by_what_it_returns_and_a_panic_fails_it() { let run = diff --git a/crates/tracel-runtime/src/jobs.rs b/crates/tracel-runtime/src/jobs.rs index c0f8b9f2..62a4acc9 100644 --- a/crates/tracel-runtime/src/jobs.rs +++ b/crates/tracel-runtime/src/jobs.rs @@ -175,3 +175,57 @@ pub enum PrepareError { #[error("invalid input: {0}")] InvalidInput(#[source] serde_json::Error), } + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + use serde_json::json; + + use super::*; + use crate::BoxError; + + const TRAIN: &str = "train"; + const EVALUATE: &str = "evaluate"; + + #[fixture] + fn jobs() -> Jobs { + Jobs::new() + .job(TRAIN, |_: u32| Ok::<_, BoxError>(())) + .job(EVALUATE, |()| Ok::<_, BoxError>(())) + } + + #[rstest] + fn a_prepared_job_has_its_name_and_the_input_it_runs_with(jobs: Jobs) { + let input = json!(3); + + let prepared = jobs.prepare(TRAIN, input.clone()).unwrap(); + + assert_eq!((prepared.name(), prepared.input()), (TRAIN, &input)); + } + + #[rstest] + fn an_unknown_job_is_not_prepared_and_the_registered_names_are_given_in_order(jobs: Jobs) { + let unknown = "infer"; + + let refused = jobs.prepare(unknown, Value::Null); + + assert!(matches!( + refused, + Err(PrepareError::UnknownJob { name, available }) + if name == unknown && available == [EVALUATE, TRAIN] + )); + } + + #[rstest] + fn an_input_that_does_not_decode_is_not_prepared(jobs: Jobs) { + let refused = jobs.prepare(TRAIN, json!("ten")); + + assert!(matches!(refused, Err(PrepareError::InvalidInput(_)))); + } + + #[rstest] + #[should_panic] + fn registering_a_job_name_twice_panics(jobs: Jobs) { + let _ = jobs.job(TRAIN, |()| Ok::<_, BoxError>(())); + } +} diff --git a/crates/tracel-runtime/src/process/mod.rs b/crates/tracel-runtime/src/process/mod.rs index 95d22aa6..a81b9155 100644 --- a/crates/tracel-runtime/src/process/mod.rs +++ b/crates/tracel-runtime/src/process/mod.rs @@ -747,11 +747,4 @@ mod tests { ); assert!(ran_with.lock().unwrap().is_none()); } - - #[test] - #[should_panic(expected = "already registered")] - fn a_job_name_registers_once() { - jobs(job(Outcome::Succeed, &RanWith::default())) - .job("evaluate", |()| Ok::<_, BoxError>(())); - } } From 6d0e8f7c7145f22dd23471ed8e74c863237d177d Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:32:49 -0400 Subject: [PATCH 06/90] test: the execution tests assert only what running a job decides A panicked run's message is decided by catch_panic, and each thread seeing its own scope by the scope module, which both test them; the execution tests re-asserted them. They now check that a panic fails the run, and the two-thread test is gone. The scope test asserted the number and the input of two runs in one table; they are now two tests, the number one rstest case per execution, each reading back what the job saw. --- crates/tracel-runtime/src/execution.rs | 107 +++++++++---------------- 1 file changed, 38 insertions(+), 69 deletions(-) diff --git a/crates/tracel-runtime/src/execution.rs b/crates/tracel-runtime/src/execution.rs index a1a26ad1..b5a378dc 100644 --- a/crates/tracel-runtime/src/execution.rs +++ b/crates/tracel-runtime/src/execution.rs @@ -106,12 +106,13 @@ impl Outcome { #[cfg(test)] mod tests { - use std::sync::{Arc, Barrier, Mutex}; + use std::sync::{Arc, OnceLock}; + use rstest::rstest; use serde_json::json; use super::*; - use crate::{Job, Jobs, job_input, job_num, job_running}; + use crate::{Job, Jobs, job_input, job_num}; /// `train`, which takes a number of epochs, three by default, and fails on zero; and `panic`, /// which panics. @@ -128,6 +129,21 @@ mod tests { .job("panic", |()| -> Result<(), BoxError> { panic!("kaboom") }) } + fn reading(read: fn() -> T) -> (Jobs, Arc>) + where + T: Send + Sync + 'static, + { + let seen = Arc::new(OnceLock::new()); + let jobs = Jobs::new().job("train", { + let seen = seen.clone(); + move |_: Value| { + let _ = seen.set(read()); + Ok::<_, BoxError>(()) + } + }); + (jobs, seen) + } + #[test] fn a_job_completes_or_fails_by_what_it_returns_and_a_panic_fails_it() { let run = @@ -143,78 +159,31 @@ mod tests { assert_eq!(failed.status(), JobStatus::Failed); assert_eq!(failed.error().unwrap().to_string(), "no epochs"); assert_eq!(panicked.status(), JobStatus::Failed); - assert_eq!( - panicked.into_result().unwrap_err().to_string(), - "the job panicked: kaboom" - ); } - #[test] - fn the_job_reads_its_number_and_input_from_its_scope_while_it_runs() { - let seen = Arc::new(Mutex::new(Vec::new())); - let jobs = Jobs::new().job( - "train", - Job::new({ - let seen = seen.clone(); - move |_: Value| { - seen.lock() - .unwrap() - .push((job_running(), job_num(), job_input())); - Ok::<_, BoxError>(()) - } - }) - .with_default(json!({"epochs": 3})), - ); - - let numbered = jobs.prepare("train", json!({"lr": 0.1})).unwrap(); - numbered.run(Execution::new().with_num("42")); - jobs.prepare("train", Value::Null) - .unwrap() - .run(Execution::new()); - - assert_eq!( - *seen.lock().unwrap(), - [ - ( - true, - Some("42".to_string()), - Some(json!({"epochs": 3, "lr": 0.1})) - ), - (true, None, Some(json!({"epochs": 3}))), - ] - ); - assert!(!job_running()); + #[rstest] + #[case::numbered(Execution::new().with_num("42"), Some("42"))] + #[case::unnumbered(Execution::new(), None)] + fn the_job_reads_the_number_its_execution_gives( + #[case] execution: Execution, + #[case] num: Option<&str>, + ) { + let (jobs, seen) = reading(job_num); + + jobs.prepare("train", Value::Null).unwrap().run(execution); + + assert_eq!(seen.get(), Some(&num.map(str::to_string))); } #[test] - fn runs_at_once_on_two_threads_each_see_their_own_execution() { - let both = Arc::new(Barrier::new(2)); - let jobs = Arc::new(Jobs::new().job("train", { - let both = both.clone(); - move |epochs: u32| { - both.wait(); - match (job_num(), job_input()) { - (Some(num), Some(input)) - if num == epochs.to_string() && input == json!(epochs) => - { - Ok(()) - } - seen => Err::<(), BoxError>(format!("job {epochs} saw {seen:?}").into()), - } - } - })); - let run = |epochs: u32| { - let prepared = jobs.prepare("train", json!(epochs)).unwrap(); - let execution = Execution::new().with_num(epochs.to_string()); - std::thread::spawn(move || prepared.run(execution)) - }; - - let one = run(1); - let two = run(2); - - let (one, two) = (one.join().unwrap(), two.join().unwrap()); - assert!(one.error().is_none(), "{one:?}"); - assert!(two.error().is_none(), "{two:?}"); + fn the_job_reads_the_input_it_runs_with() { + let (jobs, seen) = reading(job_input); + let prepared = jobs.prepare("train", json!({"lr": 0.1})).unwrap(); + let input = prepared.input().clone(); + + prepared.run(Execution::new()); + + assert_eq!(seen.get(), Some(&Some(input))); } #[test] From befb76c8c46d3971b6993253b2cfe5d8973c02df Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:34:37 -0400 Subject: [PATCH 07/90] test: the process adapter tests assert only what the adapter maps The process tests re-asserted rules other units decide and test: the merge onto the default (job.rs), the panic message (panics.rs), the command line generated from the input schema (tracel-job), and the help and version text clap prints. They now assert what the adapter maps: a command line to the request it reads, the job and input to run or what to print, and each way a run does not complete to its exit code, one rstest case each. One test keeps that a job's schema reaches its flags. The jobs are one rstest fixture, a job that completes and one that fails, and record nothing: the tests compare the request a command line gives, and errors by variant and fields, never by message. Test names state the outcome. --- crates/tracel-runtime/src/process/mod.rs | 578 +++++------------------ 1 file changed, 117 insertions(+), 461 deletions(-) diff --git a/crates/tracel-runtime/src/process/mod.rs b/crates/tracel-runtime/src/process/mod.rs index a81b9155..449a4fe7 100644 --- a/crates/tracel-runtime/src/process/mod.rs +++ b/crates/tracel-runtime/src/process/mod.rs @@ -170,82 +170,24 @@ fn program_name(program: Option<&OsString>) -> String { #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; - + use rstest::{fixture, rstest}; use serde::{Deserialize, Serialize}; - use serde_json::json; use tracel_job::{DefinitionsFile, TRACEL_DESCRIBE}; use super::*; - use crate::{BoxError, IntoJob, Job, job_input, job_num}; - - #[derive(Clone, Copy)] - enum Outcome { - Succeed, - Fail, - Panic, - } - - /// The input a job ran with, once it ran. - type RanWith = Arc>>; - - /// A job that takes any JSON input, keeps it in `ran_with`, then ends with `outcome`. - fn job( - outcome: Outcome, - ran_with: &RanWith, - ) -> impl Fn(Value) -> Result<(), BoxError> + Send + Sync + use<> { - let ran_with = ran_with.clone(); - move |input| { - *ran_with.lock().unwrap() = Some(input); - match outcome { - Outcome::Succeed => Ok(()), - Outcome::Fail => Err("boom".into()), - Outcome::Panic => panic!("kernel exploded"), - } - } - } - - /// `train`, which runs `train`, and `evaluate`, which succeeds. - fn jobs(train: impl IntoJob) -> Jobs { - Jobs::new() - .job("train", train) - .job("evaluate", job(Outcome::Succeed, &RanWith::default())) - } - - /// `train`, which ends with `outcome`, and `evaluate`, and the input `train` ran with. - fn ending(outcome: Outcome) -> (Jobs, RanWith) { - let ran_with = RanWith::default(); - (jobs(job(outcome, &ran_with)), ran_with) - } - - /// `train`, which succeeds with its input merged onto `default`, and `evaluate`, and the input - /// `train` ran with. - fn with_default(default: Value) -> (Jobs, RanWith) { - let ran_with = RanWith::default(); - let train = Job::new(job(Outcome::Succeed, &ran_with)).with_default(default); - (jobs(train), ran_with) - } + use crate::{BoxError, job_num}; #[derive(Serialize, Deserialize)] + #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] struct Epochs { epochs: u32, } - /// `train`, which takes `Epochs`, ten by default, and `evaluate`, and whether `train` ran. - fn typed() -> (Jobs, Arc>) { - let ran = Arc::new(Mutex::new(false)); - let train = Job::new({ - let ran = ran.clone(); - move |_: Epochs| { - *ran.lock().unwrap() = true; - Ok::<_, BoxError>(()) - } - }) - .with_default(Epochs { epochs: 10 }); - let jobs = Jobs::new() - .job("train", train) - .job("evaluate", job(Outcome::Succeed, &RanWith::default())); - (jobs, ran) + #[fixture] + fn jobs() -> Jobs { + Jobs::new() + .job("train", |_: Epochs| Ok::<_, BoxError>(())) + .job("fail", |()| Err::<(), BoxError>("the job failed".into())) } /// What the command line `program ` asks of `jobs`. @@ -274,7 +216,7 @@ mod tests { for args in [&[][..], &["train"], &["--bogus"]] { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("jobs.json"); - let jobs = ending(Outcome::Fail).0; + let jobs = jobs(); execute(&jobs, arguments(args), var(TRACEL_DESCRIBE, &path)).unwrap(); @@ -291,7 +233,7 @@ mod tests { fn definitions_that_cannot_be_written_fail_the_run_with_exit_code_1() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("missing").join("jobs.json"); - let jobs = ending(Outcome::Succeed).0; + let jobs = jobs(); let failed = execute(&jobs, arguments(&["train"]), var(TRACEL_DESCRIBE, &path)).unwrap_err(); @@ -302,449 +244,163 @@ mod tests { #[test] fn printing_the_version_ends_the_run_successfully() { - let jobs = ending(Outcome::Fail).0.version("1.2.3"); + let jobs = jobs().version("1.2.3"); execute(&jobs, arguments(&["--version"]), |_| None).unwrap(); } #[test] - fn the_named_job_runs_with_its_json_input() { - let (jobs, ran_with) = ending(Outcome::Succeed); - - run(jobs, &["train", r#"{"epochs": 2, "tag": null}"#]).unwrap(); - - assert_eq!( - *ran_with.lock().unwrap(), - Some(json!({"epochs": 2, "tag": null})) - ); - } - - #[test] - fn a_missing_input_runs_the_job_with_its_default_or_no_input() { - let (jobs, ran_with) = ending(Outcome::Succeed); - let (defaulted, ran_with_default) = with_default(json!({"epochs": 10})); - let ran = Arc::new(Mutex::new(false)); - let unit = Jobs::new().job("evaluate", { - let ran = ran.clone(); - move |()| { - *ran.lock().unwrap() = true; - Ok::<_, BoxError>(()) - } - }); - - run(jobs, &["train"]).unwrap(); - run(defaulted, &["train"]).unwrap(); - run(unit, &["evaluate"]).unwrap(); + fn the_job_is_numbered_by_tracel_job_num_when_its_launcher_sets_it() { + for (num, expected) in [(Some("12"), Some("12")), (Some(""), None), (None, None)] { + let jobs = Jobs::new().job("train", move |()| match job_num() { + seen if seen.as_deref() == expected => Ok(()), + seen => Err::<(), BoxError>(format!("numbered {seen:?}").into()), + }); + let vars = move |name: &str| num.filter(|_| name == TRACEL_JOB_NUM).map(OsString::from); - assert_eq!(*ran_with.lock().unwrap(), Some(Value::Null)); - assert_eq!( - *ran_with_default.lock().unwrap(), - Some(json!({"epochs": 10})) - ); - assert!(*ran.lock().unwrap()); + execute(&jobs, arguments(&["train"]), vars) + .unwrap_or_else(|error| panic!("{num:?}: {error}")); + } } - #[test] - fn a_job_without_an_input_schema_takes_no_flags() { - let (jobs, ran_with) = with_default(json!({"epochs": 10})); + #[rstest] + fn the_named_job_runs_with_the_input_its_command_line_gives(jobs: Jobs) { + let given = serde_json::json!({"epochs": 2}); - let flag = run(jobs, &["train", "--epochs", "2"]).unwrap_err(); + let request = parse(&jobs, &["train", &given.to_string()]); - assert!( - matches!(&flag, RunError::Usage(error) if error.kind() == ErrorKind::UnknownArgument), - "{flag:?}" - ); - assert_eq!(flag.exit_code().code(), 2); - assert!(ran_with.lock().unwrap().is_none()); - } - - #[test] - fn the_json_input_is_merged_onto_the_default() { - let (jobs, ran_with) = with_default(json!({ - "num_epochs": 10, - "batch_size": 64, - "shuffle": true, - "tag": "baseline", - "layers": [64, 32], - "resume_from": null, - "optimizer": {"lr": 0.001, "weight_decay": 5e-5f32} - })); - - run( - jobs, - &[ - "train", - r#"{"num_epochs": 3, "batch_size": 16, "tag": null, "optimizer": {"lr": 0.3, "weight_decay": 0.5}}"#, - ], - ) - .unwrap(); - - // `null` removes the field from the default. - assert_eq!( - *ran_with.lock().unwrap(), - Some(json!({ - "num_epochs": 3, - "batch_size": 16, - "shuffle": true, - "layers": [64, 32], - "resume_from": null, - "optimizer": {"lr": 0.3, "weight_decay": 0.5} - })) - ); + assert!(matches!( + request, + Ok(Request::Run { job, input }) if job == "train" && input == given + )); } - #[test] - fn the_config_file_is_read_and_an_unreadable_one_rejects_the_input() { + #[rstest] + fn a_config_file_gives_the_named_job_its_input(jobs: Jobs) { + let given = serde_json::json!({"epochs": 2}); let dir = tempfile::tempdir().unwrap(); let config = dir.path().join("config.json"); - std::fs::write(&config, r#"{"epochs": 3}"#).unwrap(); - let missing = dir.path().join("missing.json"); - let (jobs, ran_with) = with_default(json!({"epochs": 10, "lr": 0.1})); - let (rejected_jobs, rejected) = with_default(json!({"epochs": 10})); - - run(jobs, &["train", "-c", config.to_str().unwrap()]).unwrap(); - let unreadable = run(rejected_jobs, &["train", "-c", missing.to_str().unwrap()]); + std::fs::write(&config, given.to_string()).unwrap(); - assert_eq!( - *ran_with.lock().unwrap(), - Some(json!({"epochs": 3, "lr": 0.1})) - ); - let unreadable = unreadable.unwrap_err(); - assert!(matches!(unreadable, RunError::Config(_)), "{unreadable:?}"); - assert_eq!(unreadable.exit_code().code(), 2); - assert!(rejected.lock().unwrap().is_none()); - } - - #[test] - fn the_default_job_runs_when_no_job_is_named() { - let (jobs, ran_with) = ending(Outcome::Succeed); + let request = parse(&jobs, &["train", "-c", config.to_str().unwrap()]); - run(jobs.default_job("train"), &[]).unwrap(); - - assert_eq!(*ran_with.lock().unwrap(), Some(Value::Null)); - } - - #[test] - fn an_unknown_or_missing_job_lists_the_registered_ones() { - let jobs = || ending(Outcome::Succeed).0; - - let unknown = run(jobs(), &["infer"]).unwrap_err(); - let missing = run(jobs(), &[]).unwrap_err(); - let unknown_default = run(jobs().default_job("infer"), &[]).unwrap_err(); - - for unknown in [&unknown, &unknown_default] { - assert!( - matches!( - unknown, - RunError::UnknownJob { name, available } - if name == "infer" && available == &["evaluate", "train"] - ), - "{unknown:?}" - ); - assert!(unknown.to_string().contains("evaluate, train"), "{unknown}"); - } - assert!(matches!(missing, RunError::MissingJob { .. })); - assert!(missing.to_string().contains("evaluate, train"), "{missing}"); + assert!(matches!( + request, + Ok(Request::Run { job, input }) if job == "train" && input == given + )); } - #[test] - fn an_unusable_command_line_or_an_input_that_does_not_decode_is_invalid() { - let rejected = |args: &[&str]| { - let (jobs, ran) = typed(); - let rejected = run(jobs, args); - assert!(!*ran.lock().unwrap(), "{args:?}"); - rejected - }; - - let not_json = rejected(&["train", "{"]); - let unknown_flag = rejected(&["train", "--epochs", "2"]); - let not_decoded = rejected(&["train", "[1, 2]"]); - let wrong_type = rejected(&["train", r#"{"epochs": "ten"}"#]); + #[rstest] + fn the_default_job_runs_with_no_input_when_no_job_is_named(jobs: Jobs) { + let request = parse(&jobs.default_job("train"), &[]); - for usage in [not_json, unknown_flag] { - assert!(matches!(usage, Err(RunError::Usage(_))), "{usage:?}"); - } - for invalid in [not_decoded, wrong_type] { - assert!( - matches!(invalid, Err(RunError::InvalidInput(_))), - "{invalid:?}" - ); - } + assert!(matches!( + request, + Ok(Request::Run { job, input }) if job == "train" && input.is_null() + )); } - #[test] - fn help_and_completions_are_printed_instead_of_running_a_job() { - let (jobs, _) = with_default(json!({"epochs": 10})); + #[rstest] + fn an_unknown_job_is_rejected_with_the_registered_names(jobs: Jobs) { + let rejected = parse(&jobs, &["infer"]); - for args in [&["--help"][..], &["train", "--help"], &["train", "-h"]] { - assert!( - matches!(parse(&jobs, args), Ok(Request::Print(help)) if help.exit_code() == 0), - "{args:?}" - ); - } - assert!(matches!( - parse(&jobs, &["--completions", "bash"]), - Ok(Request::Completions(Shell::Bash)) - )); assert!(matches!( - parse(&jobs, &["--completions", "bash", "train"]), - Err(RunError::Usage(_)) + rejected, + Err(RunError::UnknownJob { name, available }) + if name == "infer" && available == names(&jobs) )); } - #[test] - fn the_version_the_program_gives_is_printed_with_its_name() { - let jobs = ending(Outcome::Succeed).0.version("1.2.3"); + #[rstest] + fn an_unknown_default_job_is_rejected_with_the_registered_names(jobs: Jobs) { + let registered = names(&jobs); - let Ok(Request::Print(version)) = parse(&jobs, &["--version"]) else { - panic!("--version printed nothing"); - }; - - assert_eq!(version.kind(), ErrorKind::DisplayVersion); - assert!(!version.use_stderr()); - assert_eq!(version.exit_code(), 0); - assert_eq!(version.to_string(), "program 1.2.3\n"); - } + let rejected = run(jobs.default_job("infer"), &[]); - #[test] - fn without_a_version_there_is_no_version_flag() { - let jobs = ending(Outcome::Succeed).0; - - let unoffered = parse(&jobs, &["--version"]); - - assert!( - matches!( - &unoffered, - Err(RunError::Usage(error)) if error.kind() == ErrorKind::UnknownArgument - ), - "{:?}", - unoffered.as_ref().err() - ); - assert_eq!( - unoffered.err().map(|error| error.exit_code().code()), - Some(2) - ); + assert!(matches!( + rejected, + Err(RunError::UnknownJob { name, available }) + if name == "infer" && available == registered + )); } - #[test] - fn a_failing_or_panicking_job_fails() { - let failing = run(ending(Outcome::Fail).0, &["train"]); - let panicking = run(ending(Outcome::Panic).0, &["train"]); + #[rstest] + fn naming_no_job_without_a_default_is_rejected_with_the_registered_names(jobs: Jobs) { + let rejected = parse(&jobs, &[]); - assert!(matches!(failing, Err(RunError::JobFailed(e)) if e.to_string() == "boom")); assert!(matches!( - panicking, - Err(RunError::JobFailed(e)) if e.to_string().contains("kernel exploded") + rejected, + Err(RunError::MissingJob { available }) if available == names(&jobs) )); } - #[test] - fn each_outcome_has_its_exit_code() { - let exit_code = |args: &[&str]| run(typed().0, args).unwrap_err().exit_code().code(); - - assert_eq!(exit_code(&["infer"]), 2); - assert_eq!(exit_code(&[]), 2); - assert_eq!(exit_code(&["train", "{"]), 2); - assert_eq!(exit_code(&["train", "--epochs", "ten"]), 2); - assert_eq!(exit_code(&["train", "--epochs"]), 2); - assert_eq!(exit_code(&["--completions", "tcsh"]), 2); - assert_eq!(exit_code(&["train", "[1, 2]"]), 2); - for (outcome, code) in [(Outcome::Fail, 1), (Outcome::Panic, 1)] { - let failed = run(ending(outcome).0, &["train"]); - assert_eq!(failed.unwrap_err().exit_code().code(), code); - } - } + #[rstest] + #[case::an_unknown_job(&["infer"], tracel_job::ExitCode::Rejected)] + #[case::no_job(&[], tracel_job::ExitCode::Rejected)] + #[case::an_unusable_command_line(&["train", "--bogus"], tracel_job::ExitCode::Rejected)] + #[case::an_input_that_does_not_decode( + &["train", r#"{"epochs": "ten"}"#], + tracel_job::ExitCode::Rejected + )] + #[case::a_job_that_fails(&["fail"], tracel_job::ExitCode::Failed)] + fn a_run_that_does_not_complete_exits_with_the_code_that_says_why( + jobs: Jobs, + #[case] args: &[&str], + #[case] exit: tracel_job::ExitCode, + ) { + assert_eq!(run(jobs, args).unwrap_err().exit_code(), exit); + } + + #[rstest] + fn an_unreadable_config_file_is_rejected(jobs: Jobs) { + let dir = tempfile::tempdir().unwrap(); + let config = dir.path().join("missing.json"); - #[test] - fn the_job_is_numbered_by_tracel_job_num_when_its_launcher_sets_it() { - for (num, expected) in [(Some("12"), Some("12")), (Some(""), None), (None, None)] { - let jobs = Jobs::new().job("train", move |()| match job_num() { - seen if seen.as_deref() == expected => Ok(()), - seen => Err::<(), BoxError>(format!("numbered {seen:?}").into()), - }); - let vars = move |name: &str| num.filter(|_| name == TRACEL_JOB_NUM).map(OsString::from); + let rejected = run(jobs, &["train", "-c", config.to_str().unwrap()]).unwrap_err(); - execute(&jobs, arguments(&["train"]), vars) - .unwrap_or_else(|error| panic!("{num:?}: {error}")); - } + assert_eq!(rejected.exit_code(), tracel_job::ExitCode::Rejected); } - #[test] - fn the_running_job_is_given_the_input_it_runs_with() { - let given = Arc::new(Mutex::new(Vec::new())); - let train = Job::new({ - let given = given.clone(); - move |input: Value| { - given.lock().unwrap().push((job_input(), Some(input))); - Ok::<_, BoxError>(()) - } - }) - .with_default(json!({"epochs": 10, "optimizer": {"lr": 0.001}})); - let evaluate = { - let given = given.clone(); - move |()| { - given.lock().unwrap().push((job_input(), None)); - Ok::<_, BoxError>(()) - } - }; + #[rstest] + #[case::help(&["--help"])] + #[case::version(&["--version"])] + fn help_and_the_version_are_printed_instead_of_running_a_job( + jobs: Jobs, + #[case] args: &[&str], + ) { + let request = parse(&jobs.version("1.2.3"), args); - run( - Jobs::new().job("train", train), - &["train", r#"{"epochs": 2}"#], - ) - .unwrap(); - run(Jobs::new().job("evaluate", evaluate), &["evaluate"]).unwrap(); - - let merged = json!({"epochs": 2, "optimizer": {"lr": 0.001}}); - assert_eq!( - *given.lock().unwrap(), - [ - (Some(merged.clone()), Some(merged)), - (Some(Value::Null), None) - ] - ); + assert!(matches!(request, Ok(Request::Print(_)))); } - #[cfg(feature = "schema")] - #[test] - fn a_documented_input_type_gives_documented_flags() { - /// How to train. - #[derive(Serialize, Deserialize, schemars::JsonSchema)] - struct Config { - /// Passes over the data. - num_epochs: u32, - optimizer: Optimizer, - /// Where to train. - device: Device, - resume_from: Option, - } - - #[derive(Serialize, Deserialize, schemars::JsonSchema)] - struct Optimizer { - /// The learning rate. - lr: f64, - } + #[rstest] + fn the_completion_script_is_printed_for_the_shell_asked_for(jobs: Jobs) { + let request = parse(&jobs, &["--completions", "bash"]); - #[derive(Serialize, Deserialize, schemars::JsonSchema)] - #[serde(rename_all = "snake_case")] - enum Device { - /// The CPU. - Cpu, - /// The first GPU. - Gpu, - } + assert!(matches!(request, Ok(Request::Completions(Shell::Bash)))); + } - #[derive(Serialize, Deserialize, schemars::JsonSchema)] - struct Resume { - experiment: u32, - } + #[rstest] + fn without_a_version_there_is_no_version_flag(jobs: Jobs) { + let request = parse(&jobs, &["--version"]); - let ran_with = Arc::new(Mutex::new(None)); - let train = Job::new({ - let ran_with = ran_with.clone(); - move |config: Config| { - *ran_with.lock().unwrap() = Some(config); - Ok::<_, BoxError>(()) - } - }) - .with_default(Config { - num_epochs: 10, - optimizer: Optimizer { lr: 0.001 }, - device: Device::Cpu, - resume_from: None, - }) - .with_schema(); - let jobs = Jobs::new().job("train", train); - - let help = command_line(&jobs, "program") - .find_subcommand_mut("train") - .unwrap() - .render_long_help() - .to_string(); - for line in [ - "--device ", - "Where to train.", - "cpu: The CPU.", - "--num-epochs ", - "Passes over the data.", - "--optimizer.lr ", - "The learning rate.", - "--resume-from ", - ] { - assert!(help.contains(line), "no {line} in\n{help}"); - } - run( - jobs, - &[ - "train", - "--device", - "gpu", - "--resume-from", - r#"{"experiment": 4}"#, - ], - ) - .unwrap(); - let config = ran_with.lock().unwrap().take().unwrap(); - assert!(matches!(config.device, Device::Gpu)); - assert_eq!(config.resume_from.map(|resume| resume.experiment), Some(4)); - assert_eq!(config.num_epochs, 10); + assert!(matches!(request, Err(RunError::Usage(_)))); } #[cfg(feature = "schema")] #[test] - fn flags_from_the_input_schema_set_fields_after_the_json_input() { - #[derive(Serialize, Deserialize, schemars::JsonSchema)] - struct Training { - epochs: u32, - optimizer: Optimizer, - } - - #[derive(Serialize, Deserialize, schemars::JsonSchema)] - struct Optimizer { - lr: f64, - weight_decay: f64, - } + fn a_job_with_an_input_schema_takes_its_fields_as_flags() { + let jobs = Jobs::new().job( + "train", + crate::Job::new(|_: Epochs| Ok::<_, BoxError>(())).with_schema(), + ); + let epochs = 2; - let ran_with = RanWith::default(); - let jobs = || { - let train = Job::new({ - let ran_with = ran_with.clone(); - move |config: Training| { - *ran_with.lock().unwrap() = Some(serde_json::to_value(config)?); - Ok::<_, BoxError>(()) - } - }) - .with_default(Training { - epochs: 10, - optimizer: Optimizer { - lr: 0.001, - weight_decay: 0.0, - }, - }) - .with_schema(); - Jobs::new().job("train", train) - }; + let request = parse(&jobs, &["train", "--epochs", &epochs.to_string()]); - run( - jobs(), - &[ - "train", - "--optimizer.weight-decay", - "0.01", - r#"{"epochs": 2, "optimizer": {"weight_decay": 0.5}}"#, - ], - ) - .unwrap(); - let not_an_integer = run(jobs(), &["train", "--epochs", "ten"]).unwrap_err(); - - assert_eq!( - ran_with.lock().unwrap().take(), - Some(json!({"epochs": 2, "optimizer": {"lr": 0.001, "weight_decay": 0.01}})) - ); - assert!( - matches!(not_an_integer, RunError::Usage(_)), - "{not_an_integer:?}" - ); - assert!(ran_with.lock().unwrap().is_none()); + assert!(matches!( + request, + Ok(Request::Run { input, .. }) if input == serde_json::json!({"epochs": epochs}) + )); } } From 20e35fc95f290dff6a749a094396d079cd188c57 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:34:56 -0400 Subject: [PATCH 08/90] test: the describe tests leave the definitions file format to tracel-job The definitions test also read the file back through DefinitionsFile and compared it with its own to_json, a round trip tracel-job owns and tests. It now checks only the JSON describe writes. The failed-write test checks the path the error holds instead of looking for the file name in its message. --- crates/tracel-runtime/src/process/describe.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/tracel-runtime/src/process/describe.rs b/crates/tracel-runtime/src/process/describe.rs index 906075c6..bd3b04ef 100644 --- a/crates/tracel-runtime/src/process/describe.rs +++ b/crates/tracel-runtime/src/process/describe.rs @@ -72,8 +72,7 @@ mod tests { assert!(describe_to(&path).unwrap()); - let contents = std::fs::read_to_string(&path).unwrap(); - let file: Value = serde_json::from_str(&contents).unwrap(); + let file: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); assert_eq!( file, json!({ @@ -95,9 +94,6 @@ mod tests { ] }) ); - let read = DefinitionsFile::from_json(contents.as_bytes()).unwrap(); - assert_eq!(read.jobs, definitions()); - assert_eq!(contents, read.to_json()); } #[test] @@ -107,7 +103,7 @@ mod tests { let error = describe_to(&path).unwrap_err(); - assert!(error.to_string().contains("jobs.json"), "{error}"); + assert_eq!(error.path, path); } #[test] From 32e349d1bd483bbc548e5670cd0f36c4bd3bed4b Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:35:51 -0400 Subject: [PATCH 09/90] test: the job tests assert what preparing a job decides The test of an input without a default also checked what serde makes of null, for a struct and for (), and the prepared input test compared three prepared inputs, two of them with JSON written out by hand. The first now checks only that the input decodes as given, and the second that the prepared input is the JSON of what the job ran with. The inputs that do not decode are rstest cases that check only that the input is refused. --- crates/tracel-runtime/src/job.rs | 75 ++++++++++---------------------- 1 file changed, 24 insertions(+), 51 deletions(-) diff --git a/crates/tracel-runtime/src/job.rs b/crates/tracel-runtime/src/job.rs index 8c1b871b..615b3bb1 100644 --- a/crates/tracel-runtime/src/job.rs +++ b/crates/tracel-runtime/src/job.rs @@ -183,6 +183,7 @@ fn resolve(default: Option, input: Value) -> Value { mod tests { use std::sync::Mutex; + use rstest::rstest; use serde::Deserialize; use serde_json::json; @@ -274,23 +275,15 @@ mod tests { #[test] fn without_a_default_the_input_decodes_as_given() { - let (job, ran_with) = recording::(); - let (no_input, _) = recording::(); - let (unit, ran_with_unit) = recording::<()>(); + let optimizer = Optimizer { + lr: 0.1, + weight_decay: 0.01, + }; + let (job, ran_with) = recording(); - run(job, json!({"lr": 0.1, "weight_decay": 0.01})).unwrap(); - let missing = run(no_input, Value::Null); - run(unit, Value::Null).unwrap(); + run(job, serde_json::to_value(&optimizer).unwrap()).unwrap(); - assert_eq!( - ran_with.lock().unwrap().take(), - Some(Optimizer { - lr: 0.1, - weight_decay: 0.01 - }) - ); - assert!(missing.is_err()); - assert_eq!(ran_with_unit.lock().unwrap().take(), Some(())); + assert_eq!(ran_with.lock().unwrap().take(), Some(optimizer)); } #[test] @@ -304,34 +297,15 @@ mod tests { #[test] fn the_prepared_input_is_the_json_the_job_runs_with() { - let prepare = |input| { - let (job, _) = recording::(); - Registered::new("train".to_string(), job.with_default(default_config())) - .prepare(input) - .unwrap() - .input - }; - let (bare, _) = recording::(); + let (job, ran_with) = recording::(); + let prepared = Registered::new("train".to_string(), job.with_default(default_config())) + .prepare(json!({"epochs": 2, "optimizer": {"lr": 0.1}})) + .unwrap(); - assert_eq!( - prepare(json!({"epochs": 2, "optimizer": {"lr": 0.1}})), - json!({ - "epochs": 2, - "optimizer": {"lr": 0.1, "weight_decay": 0.0}, - "tag": "baseline" - }) - ); - assert_eq!( - prepare(Value::Null), - serde_json::to_value(default_config()).unwrap() - ); - assert_eq!( - Registered::new("evaluate".to_string(), bare) - .prepare(json!({"split": "test"})) - .unwrap() - .input, - json!({"split": "test"}) - ); + (prepared.run)().unwrap(); + + let ran = ran_with.lock().unwrap().take().unwrap(); + assert_eq!(prepared.input, serde_json::to_value(ran).unwrap()); } #[test] @@ -357,17 +331,16 @@ mod tests { ); } - #[test] - fn an_input_of_the_wrong_type_does_not_decode() { - for input in [json!({"epochs": "ten"}), json!([1, 2]), json!("ten")] { - let (job, ran_with) = recording(); + #[rstest] + #[case::a_field_of_the_wrong_type(json!({"epochs": "ten"}))] + #[case::not_an_object(json!("ten"))] + fn an_input_that_does_not_decode_is_refused(#[case] input: Value) { + let (job, _) = recording::(); - let prepared = Registered::new("train".to_string(), job.with_default(default_config())) - .prepare(input.clone()); + let prepared = + Registered::new("train".to_string(), job.with_default(default_config())).prepare(input); - assert!(prepared.is_err(), "{input}"); - assert!(ran_with.lock().unwrap().is_none()); - } + assert!(prepared.is_err()); } #[test] From 167c8792a1ecb62345796d503c82ab9c84f19f78 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:36:33 -0400 Subject: [PATCH 10/90] test: nested scopes and failed renames are covered Two branches had no test: a scope entered over another restores it when its guard drops, and write_atomically removes its temporary file when the rename fails. The last test used to fail at the write instead; it now renames onto an existing, non-empty directory. --- crates/tracel-runtime/src/process/file.rs | 20 ++++++++++++++++---- crates/tracel-runtime/src/scope.rs | 10 ++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/tracel-runtime/src/process/file.rs b/crates/tracel-runtime/src/process/file.rs index a56c6856..d6eda0c0 100644 --- a/crates/tracel-runtime/src/process/file.rs +++ b/crates/tracel-runtime/src/process/file.rs @@ -19,8 +19,17 @@ pub fn write_atomically(path: &Path, contents: &str) -> io::Result<()> { #[cfg(test)] mod tests { + use std::ffi::OsString; + use super::*; + fn entries(dir: &Path) -> Vec { + fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect() + } + #[test] fn the_file_holds_the_contents_and_no_temporary_file_is_left() { let dir = tempfile::tempdir().unwrap(); @@ -45,11 +54,14 @@ mod tests { } #[test] - fn a_file_that_cannot_be_written_leaves_no_file() { + fn a_failed_rename_leaves_no_temporary_file() { let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("missing").join("jobs.json"); + let path = dir.path().join("jobs.json"); + fs::create_dir(&path).unwrap(); + fs::write(path.join("kept"), "").unwrap(); + + write_atomically(&path, "{}").unwrap_err(); - assert!(write_atomically(&path, "{}").is_err()); - assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 0); + assert_eq!(entries(dir.path()), [path.file_name().unwrap()]); } } diff --git a/crates/tracel-runtime/src/scope.rs b/crates/tracel-runtime/src/scope.rs index 9a1638ee..82a7484f 100644 --- a/crates/tracel-runtime/src/scope.rs +++ b/crates/tracel-runtime/src/scope.rs @@ -114,6 +114,16 @@ mod tests { assert_eq!((job_running(), job_num(), job_input()), (false, None, None)); } + #[test] + fn a_nested_scope_gives_way_to_the_one_before_when_its_guard_drops() { + let outer = scope("1"); + let _outer = enter(outer.clone()); + + drop(enter(scope("2"))); + + assert_eq!(job_num().as_deref(), outer.num()); + } + #[test] fn each_thread_sees_the_scope_of_the_job_it_runs() { let run = |num: &'static str| { From 622a9f81b126a5abbef028589aa4ba94aed843d4 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:37:45 -0400 Subject: [PATCH 11/90] fix: one malformed station event no longer ends the session RunnerEventStream gave a frame that did not decode as an error, and serve_stream ended the session on any error, so one bad frame dropped the connection and failed every running job. The stream now skips such a frame with a warning and ends only when reading fails, which it logs; it yields plain events, and serve_stream has no error to handle. The stream is generic over BufRead, and its tests feed it bytes: a frame whose data does not decode is skipped, and a failed read ends it. --- .../src/station/infrastructure/mod.rs | 98 +++++++++++++++---- crates/tracel-runtime/src/station/serve.rs | 33 ++----- 2 files changed, 89 insertions(+), 42 deletions(-) diff --git a/crates/tracel-runtime/src/station/infrastructure/mod.rs b/crates/tracel-runtime/src/station/infrastructure/mod.rs index ffb8ea38..2c69aac8 100644 --- a/crates/tracel-runtime/src/station/infrastructure/mod.rs +++ b/crates/tracel-runtime/src/station/infrastructure/mod.rs @@ -12,6 +12,7 @@ use std::time::Duration; use reqwest::Url; use reqwest::blocking::{Client, Response}; +use tracing::warn; use uuid::Uuid; use protocol::{FinishJob, RegisterRunner, RunnerEvent}; @@ -26,10 +27,6 @@ pub enum ClientError { status: reqwest::StatusCode, body: String, }, - #[error("stream io error: {0}")] - Io(#[from] std::io::Error), - #[error("invalid event payload: {0}")] - Decode(#[from] serde_json::Error), } impl ClientError { @@ -72,7 +69,10 @@ impl StationRunnerClient { } /// Register with the station; the successful response IS the runner's event stream. - pub fn open_events(&self, register: &RegisterRunner) -> Result { + pub fn open_events( + &self, + register: &RegisterRunner, + ) -> Result>, ClientError> { let response = self .events_stream_client .post(self.join("runners/events")) @@ -80,7 +80,7 @@ impl StationRunnerClient { .json(register) .send()?; let response = ensure_success(response)?; - Ok(RunnerEventStream::new(response)) + Ok(RunnerEventStream::new(BufReader::new(response))) } pub fn finish_job(&self, job_id: Uuid, finish: &FinishJob) -> Result<(), ClientError> { @@ -110,40 +110,100 @@ fn ensure_success(response: Response) -> Result { Err(ClientError::Api { status, body }) } -/// Blocking iterator over the events of an open runner stream. +/// Blocking iterator over the events of an open runner stream, read from `R`. /// -/// Yields `None` when the station closes the stream; the session is over either way. -pub struct RunnerEventStream { - lines: std::io::Lines>, +/// Skips a frame that does not decode, with a warning, and ends when the station closes the stream +/// or reading it fails; the session is over either way. +pub struct RunnerEventStream { + lines: std::io::Lines, parser: SseParser, } -impl RunnerEventStream { - fn new(response: Response) -> Self { +impl RunnerEventStream { + fn new(reader: R) -> Self { Self { - lines: BufReader::new(response).lines(), + lines: reader.lines(), parser: SseParser::default(), } } } -impl Iterator for RunnerEventStream { - type Item = Result; +impl Iterator for RunnerEventStream { + type Item = RunnerEvent; fn next(&mut self) -> Option { loop { let line = match self.lines.next()? { Ok(line) => line, - Err(e) => return Some(Err(e.into())), + Err(error) => { + warn!(%error, "the event stream failed"); + return None; + } }; let Some(frame) = self.parser.push_line(&line) else { continue; }; match RunnerEvent::decode(&frame.event, &frame.data) { - Ok(Some(event)) => return Some(Ok(event)), - Ok(None) => continue, - Err(e) => return Some(Err(e.into())), + Ok(Some(event)) => return Some(event), + Ok(None) => {} + Err(error) => warn!(event = %frame.event, %error, "skipped a malformed event"), } } } } + +#[cfg(test)] +mod tests { + use std::io::{self, Read}; + + use serde_json::json; + + use super::*; + + struct FailingOnce(bool); + + impl Read for FailingOnce { + fn read(&mut self, _: &mut [u8]) -> io::Result { + if std::mem::replace(&mut self.0, true) { + return Ok(0); + } + Err(io::Error::other("connection reset")) + } + } + + fn frame(event: &str, data: &str) -> String { + format!("event: {event}\ndata: {data}\n\n") + } + + fn registered(runner_id: Uuid) -> String { + frame("registered", &json!({ "runner_id": runner_id }).to_string()) + } + + fn events(stream: impl Read) -> Vec { + RunnerEventStream::new(BufReader::new(stream)).collect() + } + + #[test] + fn a_frame_whose_data_does_not_decode_is_skipped() { + let runner_id = Uuid::new_v4(); + let stream = frame("job", "not json") + ®istered(runner_id); + + assert_eq!( + events(stream.as_bytes()), + [RunnerEvent::Registered { runner_id }] + ); + } + + #[test] + fn a_failed_read_ends_the_stream() { + let runner_id = Uuid::new_v4(); + let registered = registered(runner_id); + let cancel = frame("cancel", &json!({ "job_id": Uuid::new_v4() }).to_string()); + let stream = registered + .as_bytes() + .chain(FailingOnce(false)) + .chain(cancel.as_bytes()); + + assert_eq!(events(stream), [RunnerEvent::Registered { runner_id }]); + } +} diff --git a/crates/tracel-runtime/src/station/serve.rs b/crates/tracel-runtime/src/station/serve.rs index bf4917ee..3290aa07 100644 --- a/crates/tracel-runtime/src/station/serve.rs +++ b/crates/tracel-runtime/src/station/serve.rs @@ -8,10 +8,10 @@ use tracing::{info, warn}; use uuid::Uuid; use crate::station::error::RunnerError; +use crate::station::infrastructure::StationRunnerClient; use crate::station::infrastructure::protocol::{ DispatchedJob, FinishJob, FinishStatus, RegisterRunner, RunnerEvent, }; -use crate::station::infrastructure::{ClientError, StationRunnerClient}; use crate::{Execution, Jobs}; /// Reports job outcomes back to the station. Split out so the loop is testable with a fake. @@ -126,19 +126,15 @@ pub enum StreamOutcome { /// Drive one session: expect the leading `registered` event, then route job dispatches to the /// executor until the stream ends. pub fn serve_stream( - mut events: impl Iterator>, + mut events: impl Iterator, executor: &Executor, ) -> StreamOutcome { let runner_id = match events.next() { - Some(Ok(RunnerEvent::Registered { runner_id })) => runner_id, - Some(Ok(event)) => { + Some(RunnerEvent::Registered { runner_id }) => runner_id, + Some(event) => { warn!(?event, "expected a registered event first"); return StreamOutcome::NeverRegistered; } - Some(Err(e)) => { - warn!(error = %e, "the event stream failed before registration"); - return StreamOutcome::NeverRegistered; - } None => { warn!("the event stream closed before registration"); return StreamOutcome::NeverRegistered; @@ -148,17 +144,13 @@ pub fn serve_stream( for event in events { match event { - Ok(RunnerEvent::Job(job)) => executor.dispatch(runner_id, job), - Ok(RunnerEvent::Cancel { job_id }) => { + RunnerEvent::Job(job) => executor.dispatch(runner_id, job), + RunnerEvent::Cancel { job_id } => { info!(%job_id, "the station cancelled the job; it runs to completion"); } - Ok(RunnerEvent::Registered { .. }) => { + RunnerEvent::Registered { .. } => { warn!("unexpected registered event mid-stream"); } - Err(e) => { - warn!(error = %e, "the event stream failed"); - break; - } } } StreamOutcome::Served @@ -283,13 +275,8 @@ mod tests { } } - fn events( - runner_id: Uuid, - rest: Vec, - ) -> impl Iterator> { - std::iter::once(RunnerEvent::Registered { runner_id }) - .chain(rest) - .map(Ok) + fn events(runner_id: Uuid, rest: Vec) -> impl Iterator { + std::iter::once(RunnerEvent::Registered { runner_id }).chain(rest) } /// Serves `jobs`, dispatched in order, and returns the finishes reported. @@ -445,7 +432,7 @@ mod tests { let setup = setup(); let outcome = serve_stream( - vec![Ok(RunnerEvent::Job(dispatched("train")))].into_iter(), + vec![RunnerEvent::Job(dispatched("train"))].into_iter(), &setup.executor, ); setup.executor.wait_idle(); From 84cd3a0d62094f76bcc1fae91503c4f9e393960b Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:39:29 -0400 Subject: [PATCH 12/90] fix: a panic while a station job's input decodes fails the job The input decodes in Jobs::prepare, outside the catch_panic that PreparedJob::run applies, so a panicking Deserialize unwound the job thread before it reported its outcome or left the in-flight count: wait_idle then blocked forever and the runner never reconnected. The executor now catches a panic around prepare too and reports it as the job's failure, and a dispatched job leaves the in-flight count through a drop guard, whether its thread ends, unwinds or never starts. --- crates/tracel-runtime/src/station/serve.rs | 79 ++++++++++++++++------ 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/crates/tracel-runtime/src/station/serve.rs b/crates/tracel-runtime/src/station/serve.rs index 3290aa07..acd3e5f2 100644 --- a/crates/tracel-runtime/src/station/serve.rs +++ b/crates/tracel-runtime/src/station/serve.rs @@ -7,6 +7,7 @@ use std::time::Duration; use tracing::{info, warn}; use uuid::Uuid; +use crate::panics::catch_panic; use crate::station::error::RunnerError; use crate::station::infrastructure::StationRunnerClient; use crate::station::infrastructure::protocol::{ @@ -52,13 +53,13 @@ impl Executor { /// Runs `job` on a thread of its own, and reports how it ended to the station once it ends. pub fn dispatch(&self, runner_id: Uuid, job: DispatchedJob) { - *self.in_flight.0.lock().unwrap() += 1; - let (jobs, sink, in_flight) = - (self.jobs.clone(), self.sink.clone(), self.in_flight.clone()); + let running = Running::start(&self.in_flight); + let (jobs, sink) = (self.jobs.clone(), self.sink.clone()); let job_id = job.id; let spawned = std::thread::Builder::new() .name(format!("job-{job_id}")) .spawn(move || { + let _running = running; let (status, reason) = execute(&jobs, &job); sink.finish_job( job.id, @@ -68,7 +69,6 @@ impl Executor { reason, }, ); - finish_in_flight(&in_flight); }); if let Err(error) = spawned { let reason = format!("the job could not start: {error}"); @@ -80,7 +80,6 @@ impl Executor { reason: Some(reason), }, ); - finish_in_flight(&self.in_flight); } } @@ -94,23 +93,35 @@ impl Executor { } } -fn finish_in_flight(in_flight: &InFlight) { - let (count, finished) = &**in_flight; - *count.lock().unwrap() -= 1; - finished.notify_all(); +/// A dispatched job, counted in flight until this drops, when its thread ends or unwinds. +struct Running(InFlight); + +impl Running { + fn start(in_flight: &InFlight) -> Self { + *in_flight.0.lock().unwrap() += 1; + Self(in_flight.clone()) + } +} + +impl Drop for Running { + fn drop(&mut self) { + let (count, finished) = &*self.0; + *count.lock().unwrap() -= 1; + finished.notify_all(); + } } /// Runs `job` and returns how it ended: completed or failed by what it returned, with its error -/// when it failed. +/// when it failed. A panic fails it, even one while its input decodes. fn execute(jobs: &Jobs, job: &DispatchedJob) -> (FinishStatus, Option) { - let prepared = match jobs.prepare(&job.job_name, job.input.clone()) { - Ok(prepared) => prepared, - // An unknown job is unreachable under the station's strict policy; defend anyway. - Err(error) => return (FinishStatus::Failed, Some(error.to_string())), - }; - info!(job = %job.job_name, job_id = %job.id, "running the job"); - let outcome = prepared.run(Execution::new().with_num(job.id.to_string())); - match outcome.into_result() { + let ran = catch_panic(|| { + let prepared = jobs.prepare(&job.job_name, job.input.clone())?; + info!(job = %job.job_name, job_id = %job.id, "running the job"); + prepared + .run(Execution::new().with_num(job.id.to_string())) + .into_result() + }); + match ran { Ok(()) => (FinishStatus::Completed, None), Err(error) => (FinishStatus::Failed, Some(error.to_string())), } @@ -208,6 +219,8 @@ pub fn serve_forever( #[cfg(test)] mod tests { + use rstest::{fixture, rstest}; + use serde::{Deserialize, Deserializer}; use serde_json::{Value, json}; use super::*; @@ -232,8 +245,16 @@ mod tests { } } - /// `train` succeeds, `fail` fails, `panic` panics, `typed` takes a number, and `num` keeps the - /// job number it is given. + struct Explosive; + + impl<'de> Deserialize<'de> for Explosive { + fn deserialize>(_: D) -> Result { + panic!("the decoder exploded") + } + } + + /// `train` succeeds, `fail` fails, `panic` panics, `typed` takes a number, `explosive` panics + /// while its input decodes, and `num` keeps the job number it is given. fn jobs(nums: Arc>>>) -> Jobs { Jobs::new() .job("train", |_: Value| Ok::<_, BoxError>(())) @@ -242,6 +263,7 @@ mod tests { panic!("kaboom") }) .job("typed", |_: u32| Ok::<_, BoxError>(())) + .job("explosive", |_: Explosive| Ok::<_, BoxError>(())) .job("num", move |_: Value| { nums.lock().unwrap().push(job_num()); Ok::<_, BoxError>(()) @@ -255,6 +277,7 @@ mod tests { nums: Arc>>>, } + #[fixture] fn setup() -> Setup { let nums = Arc::default(); let sink = Arc::new(RecordingSink::default()); @@ -287,6 +310,13 @@ mod tests { setup.sink.finishes.lock().unwrap().clone() } + fn statuses(finishes: &[RecordedFinish]) -> Vec<(Uuid, FinishStatus)> { + finishes + .iter() + .map(|(job_id, _, status, _)| (*job_id, *status)) + .collect() + } + #[test] fn given_job_event_when_serving_then_completion_reported() { let setup = setup(); @@ -335,6 +365,15 @@ mod tests { assert_eq!(finishes[0].3.as_deref(), Some("unknown job 'mystery'")); } + #[rstest] + fn a_panic_while_the_input_decodes_fails_the_job_and_frees_the_runner(setup: Setup) { + let job = dispatched("explosive"); + + let finishes = serve(&setup, vec![job.clone()]); + + assert_eq!(statuses(&finishes), [(job.id, FinishStatus::Failed)]); + } + #[test] fn a_running_job_is_numbered_with_its_station_job_id() { let setup = setup(); From 525966906c442a4eeeb7342bc22608689f62093f Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:40:30 -0400 Subject: [PATCH 13/90] docs: every public item of tracel-runtime is documented The sibling crates deny missing docs; tracel-runtime did not, and the variants and fields of RunnerError had none. The crate now denies them too, and RunnerError says what each variant means. --- crates/tracel-runtime/src/lib.rs | 2 ++ crates/tracel-runtime/src/station/error.rs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/crates/tracel-runtime/src/lib.rs b/crates/tracel-runtime/src/lib.rs index 2b0d4af5..d3d0eb9c 100644 --- a/crates/tracel-runtime/src/lib.rs +++ b/crates/tracel-runtime/src/lib.rs @@ -1,3 +1,5 @@ +#![deny(missing_docs)] + //! A program's jobs, and the adapters that run them. //! //! A job is a function of its typed input, `Fn(I) -> Result<(), E>`, registered by name in diff --git a/crates/tracel-runtime/src/station/error.rs b/crates/tracel-runtime/src/station/error.rs index 631508bd..fa39e8f3 100644 --- a/crates/tracel-runtime/src/station/error.rs +++ b/crates/tracel-runtime/src/station/error.rs @@ -5,13 +5,19 @@ /// permanently rejects its registration. #[derive(Debug, thiserror::Error)] pub enum RunnerError { + /// The station URL does not parse. #[error("invalid station url '{url}': {source}")] InvalidUrl { + /// The URL given. url: String, + /// Why it does not parse. source: url::ParseError, }, + /// The runner was given no jobs to serve. #[error("no jobs registered")] NoJobs, + /// The station rejected the runner's registration for good, with the status and body it + /// answered with. #[error("station rejected runner registration: {0}")] Registration(String), } From b35e86dacb7532727ec5e309181404a3d87feb8c Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:41:28 -0400 Subject: [PATCH 14/90] fix: a station url that is not http no longer panics the runner run only checked that the URL parsed, so one with another scheme, or one that cannot be a base such as `localhost:8000`, started the runner and then panicked joining the endpoint path. run now rejects a URL that is not http or https with RunnerError::NotHttp; any such URL can be a base, so the joins cannot fail. The HTTP clients are built in run too, and a client that cannot be built is RunnerError::HttpClient instead of a panic. --- crates/tracel-runtime/src/station/error.rs | 9 +++++ .../src/station/infrastructure/mod.rs | 15 +++----- crates/tracel-runtime/src/station/mod.rs | 37 +++++++++++++++---- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/crates/tracel-runtime/src/station/error.rs b/crates/tracel-runtime/src/station/error.rs index fa39e8f3..dfb17bc4 100644 --- a/crates/tracel-runtime/src/station/error.rs +++ b/crates/tracel-runtime/src/station/error.rs @@ -13,6 +13,15 @@ pub enum RunnerError { /// Why it does not parse. source: url::ParseError, }, + /// The station URL is not an `http` or `https` URL. + #[error("station url '{url}' is not an http or https url")] + NotHttp { + /// The URL given. + url: String, + }, + /// The HTTP client that reaches the station could not be built. + #[error("failed to build the http client: {0}")] + HttpClient(#[source] reqwest::Error), /// The runner was given no jobs to serve. #[error("no jobs registered")] NoJobs, diff --git a/crates/tracel-runtime/src/station/infrastructure/mod.rs b/crates/tracel-runtime/src/station/infrastructure/mod.rs index 2c69aac8..e0c0896e 100644 --- a/crates/tracel-runtime/src/station/infrastructure/mod.rs +++ b/crates/tracel-runtime/src/station/infrastructure/mod.rs @@ -51,21 +51,18 @@ pub struct StationRunnerClient { } impl StationRunnerClient { - pub fn new(base_url: Url) -> Self { + /// The client of the station at `base_url`, an `http` or `https` URL. + pub fn new(base_url: Url) -> Result { let events_stream_client = Client::builder() .timeout(None) .tcp_keepalive(Duration::from_secs(60)) - .build() - .expect("failed to build HTTP client"); - let short_call_client = Client::builder() - .timeout(Duration::from_secs(60)) - .build() - .expect("failed to build HTTP client"); - Self { + .build()?; + let short_call_client = Client::builder().timeout(Duration::from_secs(60)).build()?; + Ok(Self { base_url, events_stream_client, short_call_client, - } + }) } /// Register with the station; the successful response IS the runner's event stream. diff --git a/crates/tracel-runtime/src/station/mod.rs b/crates/tracel-runtime/src/station/mod.rs index 704662d4..89bac534 100644 --- a/crates/tracel-runtime/src/station/mod.rs +++ b/crates/tracel-runtime/src/station/mod.rs @@ -88,6 +88,9 @@ impl StationRunner { url: self.url.clone(), source, })?; + if !matches!(url.scheme(), "http" | "https") { + return Err(RunnerError::NotHttp { url: self.url }); + } if jobs.definitions().next().is_none() { return Err(RunnerError::NoJobs); } @@ -97,7 +100,7 @@ impl StationRunner { metadata: self.metadata, jobs: jobs.definitions().map(RegisteredJob::from).collect(), }; - let client = StationRunnerClient::new(url); + let client = StationRunnerClient::new(url).map_err(RunnerError::HttpClient)?; let executor = Executor::new(Arc::new(jobs), Arc::new(client.clone())); Err(serve::serve_forever(client, register, executor)) } @@ -105,6 +108,8 @@ impl StationRunner { #[cfg(test)] mod tests { + use rstest::{fixture, rstest}; + use super::*; use crate::BoxError; @@ -112,17 +117,33 @@ mod tests { Ok(()) } - #[test] - fn given_invalid_url_when_running_then_fails_to_start() { - let result = StationRunner::new("not a url").run(Jobs::new().job("train", train)); + #[fixture] + fn jobs() -> Jobs { + Jobs::new().job("train", train) + } + + #[rstest] + fn a_url_that_does_not_parse_does_not_start_the_runner(jobs: Jobs) { + let given = "not a url"; + + let failed = StationRunner::new(given).run(jobs); + + assert!(matches!(failed, Err(RunnerError::InvalidUrl { url, .. }) if url == given)); + } + + #[rstest] + #[case::a_host_and_port("localhost:8000")] + #[case::of_another_scheme("ftp://station")] + fn a_url_that_is_not_http_does_not_start_the_runner(jobs: Jobs, #[case] given: &str) { + let failed = StationRunner::new(given).run(jobs); - assert!(matches!(result, Err(RunnerError::InvalidUrl { .. }))); + assert!(matches!(failed, Err(RunnerError::NotHttp { url }) if url == given)); } #[test] - fn given_no_jobs_when_running_then_fails_to_start() { - let result = StationRunner::new("http://localhost:9000").run(Jobs::new()); + fn a_runner_with_no_jobs_does_not_start() { + let failed = StationRunner::new("http://localhost:9000").run(Jobs::new()); - assert!(matches!(result, Err(RunnerError::NoJobs))); + assert!(matches!(failed, Err(RunnerError::NoJobs))); } } From 598e49cff605f80a11ac6ea131cfa352de23700a Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:42:40 -0400 Subject: [PATCH 15/90] test: the serve tests assert only what the executor reports The serve tests threaded a recording Vec of job numbers through every test for the sake of one, waited on a Condvar with a ten-second timeout to make two jobs meet, and re-asserted the panic message and each thread's own scope, which panics.rs and scope.rs decide. The numbered test now serves a job of its own that records the number it is given, and the two concurrent jobs meet at a Barrier. How a job ended is one rstest case per outcome, comparing the statuses reported rather than the reasons, which are other units' error messages; one test compares a failed job's whole finish, with the job's own error as its reason. Test names state the outcome. --- crates/tracel-runtime/src/station/serve.rs | 187 +++++++++------------ 1 file changed, 76 insertions(+), 111 deletions(-) diff --git a/crates/tracel-runtime/src/station/serve.rs b/crates/tracel-runtime/src/station/serve.rs index acd3e5f2..dde7f2e9 100644 --- a/crates/tracel-runtime/src/station/serve.rs +++ b/crates/tracel-runtime/src/station/serve.rs @@ -223,8 +223,10 @@ mod tests { use serde::{Deserialize, Deserializer}; use serde_json::{Value, json}; + use std::sync::{Barrier, OnceLock}; + use super::*; - use crate::{BoxError, job_input, job_num}; + use crate::{BoxError, job_num}; #[allow(clippy::type_complexity)] type RecordedFinish = (Uuid, Uuid, FinishStatus, Option); @@ -253,19 +255,17 @@ mod tests { } } - /// `train` succeeds, `fail` fails, `panic` panics, `typed` takes a number, `explosive` panics - /// while its input decodes, and `num` keeps the job number it is given. - fn jobs(nums: Arc>>>) -> Jobs { + const FAILURE: &str = "boom"; + + fn jobs() -> Jobs { + let both = Arc::new(Barrier::new(2)); Jobs::new() .job("train", |_: Value| Ok::<_, BoxError>(())) - .job("fail", |_: Value| Err::<(), BoxError>("boom".into())) - .job("panic", |_: Value| -> Result<(), BoxError> { - panic!("kaboom") - }) + .job("fail", |_: Value| Err::<(), BoxError>(FAILURE.into())) .job("typed", |_: u32| Ok::<_, BoxError>(())) .job("explosive", |_: Explosive| Ok::<_, BoxError>(())) - .job("num", move |_: Value| { - nums.lock().unwrap().push(job_num()); + .job("wait", move |_: Value| { + both.wait(); Ok::<_, BoxError>(()) }) } @@ -274,22 +274,23 @@ mod tests { executor: Executor, sink: Arc, runner_id: Uuid, - nums: Arc>>>, } - #[fixture] - fn setup() -> Setup { - let nums = Arc::default(); + fn serving(jobs: Jobs) -> Setup { let sink = Arc::new(RecordingSink::default()); - let executor = Executor::new(Arc::new(jobs(Arc::clone(&nums))), sink.clone()); + let executor = Executor::new(Arc::new(jobs), sink.clone()); Setup { executor, sink, runner_id: Uuid::new_v4(), - nums, } } + #[fixture] + fn setup() -> Setup { + serving(jobs()) + } + fn dispatched(job_name: &str) -> DispatchedJob { DispatchedJob { id: Uuid::new_v4(), @@ -302,12 +303,17 @@ mod tests { std::iter::once(RunnerEvent::Registered { runner_id }).chain(rest) } - /// Serves `jobs`, dispatched in order, and returns the finishes reported. fn serve(setup: &Setup, jobs: Vec) -> Vec { let rest = jobs.into_iter().map(RunnerEvent::Job).collect(); serve_stream(events(setup.runner_id, rest), &setup.executor); setup.executor.wait_idle(); - setup.sink.finishes.lock().unwrap().clone() + finishes(setup) + } + + fn finishes(setup: &Setup) -> Vec { + let mut finishes = setup.sink.finishes.lock().unwrap().clone(); + finishes.sort_by_key(|(job_id, ..)| *job_id); + finishes } fn statuses(finishes: &[RecordedFinish]) -> Vec<(Uuid, FinishStatus)> { @@ -317,54 +323,51 @@ mod tests { .collect() } - #[test] - fn given_job_event_when_serving_then_completion_reported() { - let setup = setup(); - let job = dispatched("train"); + fn completed<'a>( + jobs: impl IntoIterator, + ) -> Vec<(Uuid, FinishStatus)> { + let mut completed: Vec<_> = jobs + .into_iter() + .map(|job| (job.id, FinishStatus::Completed)) + .collect(); + completed.sort_by_key(|(job_id, _)| *job_id); + completed + } + + #[rstest] + #[case::completed("train", FinishStatus::Completed)] + #[case::failed("fail", FinishStatus::Failed)] + #[case::with_an_input_that_does_not_decode("typed", FinishStatus::Failed)] + #[case::unknown("mystery", FinishStatus::Failed)] + fn a_job_is_reported_as_it_ended( + setup: Setup, + #[case] name: &str, + #[case] status: FinishStatus, + ) { + let job = dispatched(name); let finishes = serve(&setup, vec![job.clone()]); - assert_eq!( - finishes, - [(job.id, setup.runner_id, FinishStatus::Completed, None)] - ); + assert_eq!(statuses(&finishes), [(job.id, status)]); } - #[test] - fn given_failing_job_then_failure_reason_reported() { - let finishes = serve(&setup(), vec![dispatched("fail")]); - - assert_eq!(finishes[0].2, FinishStatus::Failed); - assert_eq!(finishes[0].3.as_deref(), Some("boom")); - } + #[rstest] + fn a_failed_job_is_reported_with_its_error_as_the_reason(setup: Setup) { + let job = dispatched("fail"); - #[test] - fn given_input_the_job_rejects_then_failure_reported_as_invalid_input() { - let finishes = serve(&setup(), vec![dispatched("typed")]); + let finishes = serve(&setup, vec![job.clone()]); - assert_eq!(finishes[0].2, FinishStatus::Failed); assert_eq!( - finishes[0].3.as_deref(), - Some("invalid input: invalid type: map, expected u32") + finishes, + [( + job.id, + setup.runner_id, + FinishStatus::Failed, + Some(FAILURE.to_string()) + )] ); } - #[test] - fn given_panicking_job_then_failure_reported_with_panic_message() { - let finishes = serve(&setup(), vec![dispatched("panic")]); - - assert_eq!(finishes[0].2, FinishStatus::Failed); - assert_eq!(finishes[0].3.as_deref(), Some("the job panicked: kaboom")); - } - - #[test] - fn given_unknown_job_name_then_failure_reported() { - let finishes = serve(&setup(), vec![dispatched("mystery")]); - - assert_eq!(finishes[0].2, FinishStatus::Failed); - assert_eq!(finishes[0].3.as_deref(), Some("unknown job 'mystery'")); - } - #[rstest] fn a_panic_while_the_input_decodes_fails_the_job_and_frees_the_runner(setup: Setup) { let job = dispatched("explosive"); @@ -376,64 +379,28 @@ mod tests { #[test] fn a_running_job_is_numbered_with_its_station_job_id() { - let setup = setup(); - let job = dispatched("num"); + let seen = Arc::new(OnceLock::new()); + let setup = serving(Jobs::new().job("train", { + let seen = seen.clone(); + move |_: Value| { + let _ = seen.set(job_num()); + Ok::<_, BoxError>(()) + } + })); + let job = dispatched("train"); serve(&setup, vec![job.clone()]); - assert_eq!(*setup.nums.lock().unwrap(), [Some(job.id.to_string())]); + assert_eq!(seen.get(), Some(&Some(job.id.to_string()))); } - #[test] - fn jobs_dispatched_at_once_run_at_once_each_in_its_own_scope() { - let started = Arc::new((Mutex::new(0), Condvar::new())); - let seen = Arc::new(Mutex::new(Vec::new())); - let jobs = Jobs::new().job("wait", { - let (started, seen) = (started.clone(), seen.clone()); - move |_: Value| { - let (count, both) = &*started; - *count.lock().unwrap() += 1; - both.notify_all(); - let (count, _) = both - .wait_timeout_while(count.lock().unwrap(), Duration::from_secs(10), |count| { - *count < 2 - }) - .unwrap(); - seen.lock() - .unwrap() - .push((job_num(), job_input(), *count == 2)); - Ok::<_, BoxError>(()) - } - }); - let sink = Arc::new(RecordingSink::default()); - let executor = Executor::new(Arc::new(jobs), sink.clone()); - let with_input = |n: u32| DispatchedJob { - input: json!({ "n": n }), - ..dispatched("wait") - }; - let (first, second) = (with_input(1), with_input(2)); + #[rstest] + fn jobs_dispatched_at_once_run_at_once(setup: Setup) { + let waiting = [dispatched("wait"), dispatched("wait")]; - serve_stream( - events( - Uuid::new_v4(), - vec![ - RunnerEvent::Job(first.clone()), - RunnerEvent::Job(second.clone()), - ], - ), - &executor, - ); - executor.wait_idle(); + let finishes = serve(&setup, waiting.to_vec()); - let seen = seen.lock().unwrap().clone(); - assert_eq!(seen.len(), 2); - for (job, n) in [(&first, 1), (&second, 2)] { - assert!( - seen.contains(&(Some(job.id.to_string()), Some(json!({ "n": n })), true)), - "{seen:?}" - ); - } - assert_eq!(sink.finishes.lock().unwrap().len(), 2); + assert_eq!(statuses(&finishes), completed(&waiting)); } #[test] @@ -466,10 +433,8 @@ mod tests { } } - #[test] - fn given_stream_without_registered_first_then_nothing_served() { - let setup = setup(); - + #[rstest] + fn a_stream_that_does_not_open_with_its_registration_serves_nothing(setup: Setup) { let outcome = serve_stream( vec![RunnerEvent::Job(dispatched("train"))].into_iter(), &setup.executor, @@ -477,6 +442,6 @@ mod tests { setup.executor.wait_idle(); assert!(matches!(outcome, StreamOutcome::NeverRegistered)); - assert!(setup.sink.finishes.lock().unwrap().is_empty()); + assert!(finishes(&setup).is_empty()); } } From 7cd2909d55792faaeb64bc16bab203d11cfb9176 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 13:34:27 -0400 Subject: [PATCH 16/90] test: the station protocol and SSE tests are named by their outcome The protocol and SSE parser tests had given/when/then names, unlike the rest of the crate; they now state what each one shows. --- .../src/station/infrastructure/protocol.rs | 2 +- .../tracel-runtime/src/station/infrastructure/sse.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/tracel-runtime/src/station/infrastructure/protocol.rs b/crates/tracel-runtime/src/station/infrastructure/protocol.rs index b602a7a9..e94bcc75 100644 --- a/crates/tracel-runtime/src/station/infrastructure/protocol.rs +++ b/crates/tracel-runtime/src/station/infrastructure/protocol.rs @@ -113,7 +113,7 @@ mod tests { use super::*; #[test] - fn given_unknown_event_when_decoding_then_yields_none() { + fn an_unknown_event_decodes_to_nothing() { assert_eq!(RunnerEvent::decode("heartbeat", "{}").unwrap(), None); } } diff --git a/crates/tracel-runtime/src/station/infrastructure/sse.rs b/crates/tracel-runtime/src/station/infrastructure/sse.rs index b5a13c5c..421824d5 100644 --- a/crates/tracel-runtime/src/station/infrastructure/sse.rs +++ b/crates/tracel-runtime/src/station/infrastructure/sse.rs @@ -62,7 +62,7 @@ mod tests { } #[test] - fn given_event_and_data_when_blank_line_then_frame_is_complete() { + fn a_blank_line_completes_the_frame_of_its_event_and_data() { let frames = parse(&["event: job", "data: {\"id\":1}", ""]); assert_eq!( @@ -75,28 +75,28 @@ mod tests { } #[test] - fn given_no_event_name_then_defaults_to_message() { + fn a_frame_without_an_event_name_is_a_message() { let frames = parse(&["data: hello", ""]); assert_eq!(frames[0].event, "message"); } #[test] - fn given_multiple_data_lines_then_joined_with_newline() { + fn data_lines_join_with_a_newline() { let frames = parse(&["event: job", "data: a", "data: b", ""]); assert_eq!(frames[0].data, "a\nb"); } #[test] - fn given_comments_and_other_fields_then_ignored() { + fn comments_and_other_fields_are_ignored() { let frames = parse(&[": keep-alive", "id: 42", "retry: 1000", ""]); assert!(frames.is_empty()); } #[test] - fn given_value_without_leading_space_then_parsed() { + fn a_value_needs_no_leading_space() { let frames = parse(&["event:job", "data:x", ""]); assert_eq!(frames[0].event, "job"); @@ -104,7 +104,7 @@ mod tests { } #[test] - fn given_consecutive_frames_then_state_resets_between_them() { + fn each_frame_starts_afresh() { let frames = parse(&["event: a", "data: 1", "", "data: 2", ""]); assert_eq!(frames.len(), 2); From 5bda87952951d3af325a7b6a8a399cb1bb3f8bb9 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:43:26 -0400 Subject: [PATCH 17/90] refactor: the station runner loop has no line comments The crate keeps its reasoning in names and rustdoc. The two comments left in serve.rs repeated what the warning and serve_forever's own rustdoc already say, that a rejected finish is expected after a disconnect and that running jobs drain before the runner reconnects. --- crates/tracel-runtime/src/station/serve.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/tracel-runtime/src/station/serve.rs b/crates/tracel-runtime/src/station/serve.rs index dde7f2e9..70c678c1 100644 --- a/crates/tracel-runtime/src/station/serve.rs +++ b/crates/tracel-runtime/src/station/serve.rs @@ -22,8 +22,6 @@ pub trait FinishSink: Send + Sync + 'static { impl FinishSink for StationRunnerClient { fn finish_job(&self, job_id: Uuid, finish: &FinishJob) { - // A rejected finish is expected after a disconnect: the station has already failed the - // job and this session no longer owns it. if let Err(e) = StationRunnerClient::finish_job(self, job_id, finish) { warn!(%job_id, error = %e, "failed to report the outcome of the job to the station"); } @@ -204,8 +202,6 @@ pub fn serve_forever( if let StreamOutcome::Served = serve_stream(stream, &executor) { backoff.reset(); } - // Never re-register while user code still runs: the station failed those jobs when - // the stream dropped, and a fresh session would get new ones. Drain them first. executor.wait_idle(); } Err(e) if e.is_permanent() => return RunnerError::Registration(e.to_string()), From adc0aa9990f89abc61911e520f0121553dec8310 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:28:05 -0400 Subject: [PATCH 18/90] fix: one unresolvable reference no longer removes every flag The input schema is registered with the referencing crate, whose registry failed to prepare when the schema referred to another document or named a $schema it did not know, since its default retriever fetches nothing, and the job then had no flags at all. The registry now has a retriever that answers every such URI with the schema `true`, without fetching it, and the schema is registered under a hierarchical base URI, so a relative reference such as `shared.json` resolves too. Preparing then succeeds. A field whose reference still does not resolve, because it leads into a document that was not fetched, nowhere, or round in a loop, takes a JSON literal, and every other field keeps its flag. --- Cargo.lock | 1 + crates/tracel-job/Cargo.toml | 1 + crates/tracel-job/src/cli/flags.rs | 80 ++++++++++++++++++++++-------- crates/tracel-job/src/cli/mod.rs | 3 +- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 862ae549..ddb294fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7355,6 +7355,7 @@ dependencies = [ "clap", "clap_complete", "referencing", + "rstest", "schemars 1.2.2", "serde", "serde_json", diff --git a/crates/tracel-job/Cargo.toml b/crates/tracel-job/Cargo.toml index b450a567..5241a4b8 100644 --- a/crates/tracel-job/Cargo.toml +++ b/crates/tracel-job/Cargo.toml @@ -24,4 +24,5 @@ referencing = { workspace = true, optional = true } thiserror = { workspace = true, optional = true } [dev-dependencies] +rstest = { workspace = true } schemars.workspace = true diff --git a/crates/tracel-job/src/cli/flags.rs b/crates/tracel-job/src/cli/flags.rs index e15a12c4..0902c898 100644 --- a/crates/tracel-job/src/cli/flags.rs +++ b/crates/tracel-job/src/cli/flags.rs @@ -1,8 +1,9 @@ //! The flags of a job's command line, read from its input schema: one per field of its input. +use std::error::Error; use std::ptr; -use referencing::{Registry, Resolver}; +use referencing::{Registry, Resolver, Retrieve, Uri}; use serde_json::{Map, Number, Value}; use crate::JobDefinition; @@ -11,7 +12,7 @@ use crate::JobDefinition; const RESERVED: [&str; 3] = ["help", "version", "config"]; /// The URI the input schema is registered under, which its references resolve against. -const BASE: &str = "urn:tracel:input"; +const BASE: &str = "json-schema:///input"; /// A flag that sets one field of a job's input. #[derive(Debug, Clone, PartialEq)] @@ -163,6 +164,7 @@ pub fn flags(job: &JobDefinition) -> Vec { return Vec::new(); }; let Ok(registry) = Registry::new() + .retriever(Unfetched) .add(BASE, schema) .and_then(|registry| registry.prepare()) else { @@ -188,6 +190,17 @@ pub fn flags(job: &JobDefinition) -> Vec { walk.flags } +/// Answers each schema the input schema refers to outside itself with `true`, the schema of any +/// value, since none is fetched: the registry is prepared whatever the input schema refers to, and +/// a reference into such a schema does not resolve. +struct Unfetched; + +impl Retrieve for Unfetched { + fn retrieve(&self, _: &Uri) -> Result> { + Ok(Value::Bool(true)) + } +} + /// Whether `key` can be part of a flag name. fn is_flag_key(key: &str) -> bool { !key.is_empty() @@ -395,6 +408,7 @@ fn description(schema: &Map) -> Option { mod tests { use std::collections::HashMap; + use rstest::rstest; use schemars::JsonSchema; use serde::Serialize; use serde_json::json; @@ -723,29 +737,53 @@ mod tests { ("value".to_string(), Kind::Integer), ] ); - let flags = flags_of( - json!({ - "type": "object", - "properties": { - "node": {"$ref": "#"}, - "loop": {"$ref": "#/$defs/A"}, - "missing": {"$ref": "#/$defs/Missing"} - }, - "$defs": {"A": {"$ref": "#/$defs/B"}, "B": {"$ref": "#/$defs/A"}} - }), - None, - ); assert_eq!( - kinds(&flags), - [ - ("loop".to_string(), Kind::Json), - ("missing".to_string(), Kind::Json), - ("node".to_string(), Kind::Json), - ] + kinds(&flags_of( + json!({"type": "object", "properties": {"node": {"$ref": "#"}}}), + None + )), + [("node".to_string(), Kind::Json)] ); } + #[rstest] + #[case::missing_from_the_schema("#/$defs/Missing")] + #[case::that_comes_back_round("#/$defs/Loop")] + #[case::into_another_document("https://example.com/input.json#/$defs/Epochs")] + fn a_field_whose_reference_does_not_resolve_takes_json(#[case] reference: &str) { + let schema = json!({ + "type": "object", + "properties": {"field": {"$ref": reference}}, + "$defs": {"Loop": {"$ref": "#/$defs/Loop"}} + }); + + assert_eq!(flag(&flags_of(schema, None), "field").kind, Kind::Json); + } + + #[rstest] + #[case::from_a_field(json!({ + "type": "object", + "properties": { + "epochs": {"type": "integer"}, + "remote": {"$ref": "https://example.com/input.json"} + } + }))] + #[case::from_a_definition_to_a_file_beside_it(json!({ + "type": "object", + "properties": {"epochs": {"type": "integer"}}, + "$defs": {"Shared": {"$ref": "shared.json"}} + }))] + #[case::to_its_dialect(json!({ + "$schema": "https://example.com/an-unknown-dialect", + "type": "object", + "properties": {"epochs": {"type": "integer"}} + }))] + fn a_schema_that_refers_outside_itself_keeps_its_flags(#[case] schema: Value) { + assert_eq!(flag(&flags_of(schema, None), "epochs").kind, Kind::Integer); + } + #[derive(JsonSchema)] + #[expect(dead_code)] struct Escaped { odd: Odd, slash: Slash, @@ -753,12 +791,14 @@ mod tests { #[derive(JsonSchema)] #[schemars(rename = "Odd name%")] + #[expect(dead_code)] struct Odd { a: u32, } #[derive(JsonSchema)] #[schemars(rename = "a/b~c")] + #[expect(dead_code)] struct Slash { b: bool, } diff --git a/crates/tracel-job/src/cli/mod.rs b/crates/tracel-job/src/cli/mod.rs index 04e56780..24c457d0 100644 --- a/crates/tracel-job/src/cli/mod.rs +++ b/crates/tracel-job/src/cli/mod.rs @@ -18,7 +18,8 @@ //! - An object's `properties` are its fields, and a nested object's fields are flags too: the //! field `weight_decay` of `optimizer` is `--optimizer.weight-decay`, the keys joined with dots //! and `_` written as `-`. -//! - `$ref` is followed into the schema's `$defs` or `definitions`. +//! - `$ref` is followed within the schema, such as into its `$defs` or `definitions`; a field +//! whose reference leads out of the schema or nowhere takes a JSON literal. //! - `enum`, `const`, and `oneOf` or `anyOf` of constants, such as the unit variants of an enum, //! are the values a flag takes, each with its description. //! - An optional field, `type: [T, "null"]` or `anyOf` of `T` and `null`, takes the values of `T`. From 9ee58e04aac284c495ec9f9dbfcfc7b027539585 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:28:47 -0400 Subject: [PATCH 19/90] fix: a bool flag no longer swallows the json input after it A bool flag took zero or one value, so in ` --shuffle '{"epochs":2}'` the JSON document was read as the flag's value and rejected as an invalid bool. A bool flag now takes a value only as `--shuffle=false`, which is how tracel-cli forwards flags, so an argument after a bare `--shuffle` is the job's JSON input. --- crates/tracel-job/src/cli/command.rs | 28 +++++++++++++++++++++------- crates/tracel-job/src/cli/mod.rs | 5 +++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/crates/tracel-job/src/cli/command.rs b/crates/tracel-job/src/cli/command.rs index 1c25cc7d..a2de0513 100644 --- a/crates/tracel-job/src/cli/command.rs +++ b/crates/tracel-job/src/cli/command.rs @@ -214,7 +214,10 @@ fn flag_arg(flag: &Flag) -> Arg { arg = arg.required_unless_present_any([INPUT, CONFIG]); } match flag.kind { - Kind::Bool => arg.num_args(0..=1).default_missing_value("true"), + Kind::Bool => arg + .num_args(0..=1) + .require_equals(true) + .default_missing_value("true"), Kind::Integer | Kind::Number | Kind::Json => arg.allow_negative_numbers(true), Kind::String | Kind::Choice(_) => arg, } @@ -262,6 +265,7 @@ fn overlay(input: &mut Value, layer: Value) { #[allow(dead_code)] mod tests { use clap::error::ErrorKind; + use rstest::rstest; use schemars::JsonSchema; use serde::Serialize; use serde_json::json; @@ -441,7 +445,7 @@ mod tests { "[default: 10]", "--optimizer.weight-decay ", "[default: 0.00005]", - "--shuffle []", + "--shuffle[=]", "--layers ", "[default: [64,32]]", "--resume-from ", @@ -460,15 +464,11 @@ mod tests { input(&train, &["--shuffle", "--batch-size", "-1"]).unwrap(), json!({"shuffle": true, "batch_size": -1}) ); - assert_eq!( - input(&train, &["--shuffle", "false"]).unwrap(), - json!({"shuffle": false}) - ); for args in [ &["--num-epochs", "ten"][..], &["--num-epochs", "1.5"], &["--optimizer.lr", "fast"], - &["--shuffle", "yes"], + &["--shuffle=yes"], &["--layers", "[1,"], ] { let error = input(&train, args).unwrap_err(); @@ -491,6 +491,20 @@ mod tests { ); } + #[rstest] + #[case::alone(&["--shuffle"], json!({"shuffle": true}))] + #[case::after_an_equals_sign(&["--shuffle=false"], json!({"shuffle": false}))] + #[case::before_the_json_input( + &["--shuffle", r#"{"num_epochs": 2}"#], + json!({"num_epochs": 2, "shuffle": true}) + )] + fn a_bool_flag_takes_a_value_only_after_an_equals_sign( + #[case] args: &[&str], + #[case] expected: Value, + ) { + assert_eq!(input(&train(), args).unwrap(), expected); + } + #[test] fn json_literal_flags_take_any_json() { assert_eq!( diff --git a/crates/tracel-job/src/cli/mod.rs b/crates/tracel-job/src/cli/mod.rs index 24c457d0..aacd4153 100644 --- a/crates/tracel-job/src/cli/mod.rs +++ b/crates/tracel-job/src/cli/mod.rs @@ -29,8 +29,9 @@ //! A field's `description` is its flag's help, and its value in the job's default input, or else //! its schema `default`, is the flag's shown default. A field the schema requires is a required //! flag unless it has a default or the JSON document or `--config` is given. A boolean flag given -//! no value is `true`. The schema's other constraints, such as `minimum` or `pattern`, are not -//! checked here: the job's input is checked as a whole once it is resolved. +//! no value is `true`, and takes a value only as `--=`, so an argument after it is the +//! JSON document. The schema's other constraints, such as `minimum` or `pattern`, are not checked +//! here: the job's input is checked as a whole once it is resolved. //! //! `--help`, `--version` and `--config` belong to the command line, so a field named `help`, //! `version` or `config` has no flag; neither does a field whose key is not made of letters, From fc29aeaa74032d0fb90e053fe66b1d83cd51106e Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:29:06 -0400 Subject: [PATCH 20/90] fix: a field named version gets its flag Job subcommands never have a --version flag: only the program's top-level command does, when it gives a version. Reserving the name only took the flag away from an input field named `version`. Only `help` and `config` are reserved now. --- crates/tracel-job/src/cli/command.rs | 6 +++--- crates/tracel-job/src/cli/flags.rs | 3 ++- crates/tracel-job/src/cli/mod.rs | 7 +++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/tracel-job/src/cli/command.rs b/crates/tracel-job/src/cli/command.rs index a2de0513..c0cfff71 100644 --- a/crates/tracel-job/src/cli/command.rs +++ b/crates/tracel-job/src/cli/command.rs @@ -692,7 +692,7 @@ mod tests { let command = job_command(&job.job()); let longs: Vec<&str> = command.get_arguments().filter_map(Arg::get_long).collect(); - assert_eq!(longs, ["epochs", "config"]); + assert_eq!(longs, ["epochs", "version", "config"]); assert_eq!( input_from(&job, &["--config", "large.json"], &files) .unwrap() @@ -704,8 +704,8 @@ mod tests { ErrorKind::DisplayHelp ); assert_eq!( - input(&job, &["--version"]).unwrap_err().kind(), - ErrorKind::UnknownArgument + input(&job, &["--version", "3"]).unwrap(), + json!({"version": 3}) ); } diff --git a/crates/tracel-job/src/cli/flags.rs b/crates/tracel-job/src/cli/flags.rs index 0902c898..22561a03 100644 --- a/crates/tracel-job/src/cli/flags.rs +++ b/crates/tracel-job/src/cli/flags.rs @@ -9,7 +9,7 @@ use serde_json::{Map, Number, Value}; use crate::JobDefinition; /// Flag names the command line keeps: a field with one of these names has no flag. -const RESERVED: [&str; 3] = ["help", "version", "config"]; +const RESERVED: [&str; 2] = ["help", "config"]; /// The URI the input schema is registered under, which its references resolve against. const BASE: &str = "json-schema:///input"; @@ -846,6 +846,7 @@ mod tests { ("batch-size".to_string(), Kind::Integer), ("config.path".to_string(), Kind::String), ("optimizer.config".to_string(), Kind::Integer), + ("version".to_string(), Kind::String), ] ); } diff --git a/crates/tracel-job/src/cli/mod.rs b/crates/tracel-job/src/cli/mod.rs index aacd4153..853fda63 100644 --- a/crates/tracel-job/src/cli/mod.rs +++ b/crates/tracel-job/src/cli/mod.rs @@ -33,10 +33,9 @@ //! JSON document. The schema's other constraints, such as `minimum` or `pattern`, are not checked //! here: the job's input is checked as a whole once it is resolved. //! -//! `--help`, `--version` and `--config` belong to the command line, so a field named `help`, -//! `version` or `config` has no flag; neither does a field whose key is not made of letters, -//! digits, `_` and `-`, or whose flag name an earlier field took. Such a field is set through the -//! JSON document or `--config`. +//! `--help` and `--config` belong to the command line, so a field named `help` or `config` has no +//! flag; neither does a field whose key is not made of letters, digits, `_` and `-`, or whose flag +//! name an earlier field took. Such a field is set through the JSON document or `--config`. //! //! ## Input //! From e1681d43938dcd9baf4af1f55348b55fc700551f Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:30:07 -0400 Subject: [PATCH 21/90] fix: the definitions file gives its program's sdk version DefinitionsFile::new wrote tracel-job's own CARGO_PKG_VERSION as sdk_version, which is documented as the version of the Tracel SDK the program is built with. It now takes that version from its caller, and tracel-runtime passes its own. --- crates/tracel-job/src/definitions.rs | 18 +++++++++--------- crates/tracel-runtime/src/process/describe.rs | 5 ++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/tracel-job/src/definitions.rs b/crates/tracel-job/src/definitions.rs index f1887dca..51bca0c3 100644 --- a/crates/tracel-job/src/definitions.rs +++ b/crates/tracel-job/src/definitions.rs @@ -36,12 +36,12 @@ pub struct DefinitionsFile { } impl DefinitionsFile { - /// The file of a program that can run `jobs`: following [`PROTOCOL`], and built with this - /// version of the SDK. - pub fn new(jobs: Vec) -> Self { + /// The file of a program built with version `sdk_version` of the Tracel SDK that can run + /// `jobs`, following [`PROTOCOL`]. + pub fn new(sdk_version: impl Into, jobs: Vec) -> Self { Self { protocol: PROTOCOL, - sdk_version: env!("CARGO_PKG_VERSION").to_string(), + sdk_version: sdk_version.into(), jobs, } } @@ -113,7 +113,7 @@ mod tests { use super::*; fn file() -> DefinitionsFile { - DefinitionsFile::new(vec![ + let jobs = vec![ JobDefinition { name: "echo".to_string(), description: None, @@ -126,14 +126,15 @@ mod tests { input_schema: None, input_example: Some(json!({"epochs": 10, "optimizer": {"lr": 0.001}})), }, - ]) + ]; + DefinitionsFile::new("0.10.0", jobs) } #[test] fn the_file_is_pretty_printed_json_ending_with_a_newline() { let expected = r#"{ "protocol": 1, - "sdk_version": "SDK_VERSION", + "sdk_version": "0.10.0", "jobs": [ { "name": "echo", @@ -156,8 +157,7 @@ mod tests { } ] } -"# - .replace("SDK_VERSION", env!("CARGO_PKG_VERSION")); +"#; assert_eq!(file().to_json(), expected); assert_eq!( diff --git a/crates/tracel-runtime/src/process/describe.rs b/crates/tracel-runtime/src/process/describe.rs index bd3b04ef..b097271b 100644 --- a/crates/tracel-runtime/src/process/describe.rs +++ b/crates/tracel-runtime/src/process/describe.rs @@ -21,7 +21,10 @@ pub fn describe<'a>( return Ok(false); }; let path = PathBuf::from(path); - let file = DefinitionsFile::new(definitions.into_iter().cloned().collect()); + let file = DefinitionsFile::new( + env!("CARGO_PKG_VERSION"), + definitions.into_iter().cloned().collect(), + ); write_atomically(&path, &file.to_json()).map_err(|source| DescribeError { path, source })?; Ok(true) } From 4ad86093e0af38f06e2ce1c1d48e467187d291a6 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:30:37 -0400 Subject: [PATCH 22/90] fix: the definitions file orders its jobs by name The protocol states that `jobs` is ordered by name, but DefinitionsFile::new kept the order its caller gave. It now sorts them, so every file it writes keeps the stated order. --- crates/tracel-job/src/definitions.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/crates/tracel-job/src/definitions.rs b/crates/tracel-job/src/definitions.rs index 51bca0c3..9bb4327d 100644 --- a/crates/tracel-job/src/definitions.rs +++ b/crates/tracel-job/src/definitions.rs @@ -37,8 +37,9 @@ pub struct DefinitionsFile { impl DefinitionsFile { /// The file of a program built with version `sdk_version` of the Tracel SDK that can run - /// `jobs`, following [`PROTOCOL`]. - pub fn new(sdk_version: impl Into, jobs: Vec) -> Self { + /// `jobs`, following [`PROTOCOL`], with `jobs` ordered by name. + pub fn new(sdk_version: impl Into, mut jobs: Vec) -> Self { + jobs.sort_by(|a, b| a.name.cmp(&b.name)); Self { protocol: PROTOCOL, sdk_version: sdk_version.into(), @@ -166,6 +167,21 @@ mod tests { ); } + #[test] + fn the_jobs_are_ordered_by_name() { + let job = |name: &str| JobDefinition { + name: name.to_string(), + description: None, + input_schema: None, + input_example: None, + }; + + let file = + DefinitionsFile::new("0.10.0", vec![job("train"), job("evaluate"), job("infer")]); + + assert_eq!(file.jobs, [job("evaluate"), job("infer"), job("train")]); + } + #[test] fn only_a_definitions_file_of_this_protocol_parses() { let parse = |json: &str| DefinitionsFile::from_json(json.as_bytes()); From 53e005b22335eabf23df6a6555a725184ad1e41e Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:31:00 -0400 Subject: [PATCH 23/90] docs: input_example is documented as the job's default input It was documented as an example that is typically the default input, but the command line always treats it as the default input: its values are the flags' defaults and the input is merged onto it. The docs now say so, as the crate docs already did. --- crates/tracel-job/src/definitions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tracel-job/src/definitions.rs b/crates/tracel-job/src/definitions.rs index 9bb4327d..82e01f37 100644 --- a/crates/tracel-job/src/definitions.rs +++ b/crates/tracel-job/src/definitions.rs @@ -20,7 +20,7 @@ pub struct JobDefinition { pub description: Option, /// JSON Schema of the job's input, when the program gives one. pub input_schema: Option, - /// An example input, typically the job's default input, which its input is merged onto. + /// The job's default input, which its input is merged onto. pub input_example: Option, } From ee3f55fb87822d0e8435c73a4b990cd6a11895f9 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:31:40 -0400 Subject: [PATCH 24/90] fix: an optional field takes the help of the type it makes optional A field's help fell back to the description of the type its $ref points to, but not through the anyOf of that type and null that schemars writes for an Option, so an undocumented `Option` had no help. The fallback now follows the optional's value as well. --- crates/tracel-job/src/cli/flags.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/tracel-job/src/cli/flags.rs b/crates/tracel-job/src/cli/flags.rs index 22561a03..6c9651ac 100644 --- a/crates/tracel-job/src/cli/flags.rs +++ b/crates/tracel-job/src/cli/flags.rs @@ -319,9 +319,18 @@ impl<'r> Walk<'r> { } } + /// The description of `schema`, or else that of the schema it refers to, or else the help of + /// the value it makes optional. + fn help(&self, schema: &'r Value) -> Option { + schema.as_object().and_then(description).or_else(|| { + let target = self.target(schema)?.as_object()?; + description(target).or_else(|| self.help(self.nullable(target)?)) + }) + } + /// Adds the flags of the fields of `object`, the schema of the object at `path`, whose value /// in the default input is `defaults`. The object is required when `required` is. A field's - /// help is its description, or else that of the schema it refers to. + /// help is its [`help`](Self::help). fn object( &mut self, object: &'r Map, @@ -350,15 +359,10 @@ impl<'r> Walk<'r> { match self.shape(property) { Shape::Object(object) => self.object(object, path, default, required), Shape::Leaf(kind) => { - let help = [Some(property), self.target(property)] - .into_iter() - .flatten() - .filter_map(Value::as_object) - .find_map(description); self.flags.push(Flag { path: path.clone(), kind, - help, + help: self.help(property), default: default.cloned(), required: required && default.is_none(), }); @@ -571,8 +575,9 @@ mod tests { /// Where to train. device: Device, backend: Backend, - /// The fallback. fallback: Option, + /// The backend to compare with. + baseline: Option, optimizer: Optimizer, } @@ -609,7 +614,8 @@ mod tests { ("epochs", Some("Passes over the data.")), ("device", Some("Where to train.")), ("backend", Some("The tensor backend.")), - ("fallback", Some("The fallback.")), + ("fallback", Some("The tensor backend.")), + ("baseline", Some("The backend to compare with.")), ("optimizer.lr", None), ] { assert_eq!(flag(&flags, name).help.as_deref(), help, "--{name}"); From 3054fb636f3ab814cb5bb7c389b2f853ede13bf6 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:33:56 -0400 Subject: [PATCH 25/90] test: cli fixtures are job definitions with distinct descriptions The command tests built jobs through a Definition fixture that duplicated JobDefinition and gave every job the same description, so the subcommand test could not tell whose description it saw. The fixtures now return JobDefinition with distinct descriptions, and the test checks each job's own. The flags tests renamed their Optimizer and Resume fixtures, which the command tests define differently under the same names. Each test module allowed dead code as a whole; the expectation now sits on the fixture types that need it. Two command tests are renamed after what they assert. --- crates/tracel-job/src/cli/command.rs | 120 ++++++++++++--------------- crates/tracel-job/src/cli/flags.rs | 25 ++++-- 2 files changed, 72 insertions(+), 73 deletions(-) diff --git a/crates/tracel-job/src/cli/command.rs b/crates/tracel-job/src/cli/command.rs index c0cfff71..8156131a 100644 --- a/crates/tracel-job/src/cli/command.rs +++ b/crates/tracel-job/src/cli/command.rs @@ -262,7 +262,6 @@ fn overlay(input: &mut Value, layer: Value) { } #[cfg(test)] -#[allow(dead_code)] mod tests { use clap::error::ErrorKind; use rstest::rstest; @@ -272,28 +271,6 @@ mod tests { use super::*; - /// A job's input schema and default input. - struct Definition { - name: &'static str, - schema: Option, - default: Option, - } - - fn jobs(definitions: &[Definition]) -> Vec { - definitions.iter().map(Definition::job).collect() - } - - impl Definition { - fn job(&self) -> JobDefinition { - JobDefinition { - name: self.name.to_string(), - description: Some("Run the job".to_string()), - input_schema: self.schema.clone(), - input_example: self.default.clone(), - } - } - } - #[derive(Serialize, JsonSchema)] struct Training { num_epochs: u32, @@ -317,7 +294,7 @@ mod tests { } /// `train`, taking `Training`, with a default input. - fn train() -> Definition { + fn train() -> JobDefinition { let default = Training { num_epochs: 10, batch_size: 64, @@ -330,14 +307,16 @@ mod tests { weight_decay: 5e-5, }, }; - Definition { - name: "train", - schema: Some(schemars::schema_for!(Training).to_value()), - default: Some(serde_json::to_value(default).unwrap()), + JobDefinition { + name: "train".to_string(), + description: Some("Train the model".to_string()), + input_schema: Some(schemars::schema_for!(Training).to_value()), + input_example: Some(serde_json::to_value(default).unwrap()), } } #[derive(JsonSchema)] + #[expect(dead_code)] struct Prompt { /// The prompt. text: String, @@ -347,38 +326,41 @@ mod tests { #[derive(JsonSchema)] #[serde(rename_all = "snake_case")] + #[expect(dead_code)] enum Mode { Fast, Exact, } /// `prompt`, taking `Prompt`, with no default input. - fn prompt() -> Definition { - Definition { - name: "prompt", - schema: Some(schemars::schema_for!(Prompt).to_value()), - default: None, + fn prompt() -> JobDefinition { + JobDefinition { + name: "prompt".to_string(), + description: Some("Answer a prompt".to_string()), + input_schema: Some(schemars::schema_for!(Prompt).to_value()), + input_example: None, } } - /// `evaluate`, with a default input and no schema. - fn evaluate() -> Definition { - Definition { - name: "evaluate", - schema: None, - default: Some(json!({"split": "test", "limit": 10})), + /// `evaluate`, with a default input and no schema or description. + fn evaluate() -> JobDefinition { + JobDefinition { + name: "evaluate".to_string(), + description: None, + input_schema: None, + input_example: Some(json!({"split": "test", "limit": 10})), } } /// The input the job's command line ` ` gives, its `--config` file read from /// `files`, by name. fn input_from( - definition: &Definition, + job: &JobDefinition, args: &[&str], files: &[(&str, &str)], ) -> Result, clap::Error> { - let matches = job_command(&definition.job()) - .try_get_matches_from([definition.name].iter().chain(args))?; + let matches = + job_command(job).try_get_matches_from([job.name.as_str()].iter().chain(args))?; Ok(job_input(&matches, |path| { files .iter() @@ -389,28 +371,35 @@ mod tests { } /// The input the job's command line ` ` gives, with no `--config` file to read. - fn input(definition: &Definition, args: &[&str]) -> Result { - input_from(definition, args, &[]).map(|input| input.unwrap()) + fn input(job: &JobDefinition, args: &[&str]) -> Result { + input_from(job, args, &[]).map(|input| input.unwrap()) } /// The job's `--help`. - fn help(definition: &Definition) -> String { - job_command(&definition.job()) - .render_long_help() - .to_string() + fn help(job: &JobDefinition) -> String { + job_command(job).render_long_help().to_string() } #[test] fn the_command_has_a_subcommand_per_job() { - let definitions = [train(), prompt()]; - let mut command = command("program", &jobs(&definitions)); + let jobs = [train(), prompt(), evaluate()]; + let mut command = command("program", &jobs); command.build(); - let names: Vec<&str> = command.get_subcommands().map(Command::get_name).collect(); - assert_eq!(names, ["train", "prompt"]); + let abouts: Vec<(&str, Option)> = command + .get_subcommands() + .map(|job| (job.get_name(), job.get_about().map(ToString::to_string))) + .collect(); + assert_eq!( + abouts, + [ + ("train", Some("Train the model".to_string())), + ("prompt", Some("Answer a prompt".to_string())), + ("evaluate", None), + ] + ); let help = command.render_help().to_string(); assert!(help.contains("Jobs:"), "{help}"); - assert!(help.contains("Run the job"), "{help}"); assert!(help.contains("--completions "), "{help}"); } @@ -522,7 +511,7 @@ mod tests { } #[test] - fn a_schema_gives_flags_their_values_help_and_requirement() { + fn a_schema_gives_flags_their_values_and_help() { let prompt = prompt(); assert_eq!( @@ -642,7 +631,7 @@ mod tests { } #[test] - fn an_unreadable_or_invalid_config_file_gives_no_input() { + fn an_unreadable_or_invalid_config_file_is_an_error_naming_it() { let files = [("config.toml", "epochs = 3")]; let error = |path| { input_from(&train(), &["-c", path], &files) @@ -675,9 +664,10 @@ mod tests { #[test] fn reserved_names_belong_to_the_command_line() { - let job = Definition { - name: "train", - schema: Some(json!({ + let job = JobDefinition { + name: "train".to_string(), + description: None, + input_schema: Some(json!({ "type": "object", "properties": { "config": {"type": "string"}, @@ -686,11 +676,11 @@ mod tests { "epochs": {"type": "integer"} } })), - default: None, + input_example: None, }; let files = [("large.json", r#"{"config": "large"}"#)]; - let command = job_command(&job.job()); + let command = job_command(&job); let longs: Vec<&str> = command.get_arguments().filter_map(Arg::get_long).collect(); assert_eq!(longs, ["epochs", "version", "config"]); assert_eq!( @@ -711,9 +701,9 @@ mod tests { #[test] fn completions_names_the_shell_the_command_line_asks_for() { - let definitions = [train()]; + let jobs = [train()]; let matches = |args: &[&str]| { - command("program", &jobs(&definitions)) + command("program", &jobs) .try_get_matches_from(["program"].iter().chain(args)) .unwrap() }; @@ -724,7 +714,7 @@ mod tests { ); assert_eq!(completions(&matches(&["train"])), None); assert_eq!( - command("program", &jobs(&definitions)) + command("program", &jobs) .try_get_matches_from(["program", "--completions", "tcsh"]) .unwrap_err() .kind(), @@ -734,12 +724,12 @@ mod tests { #[test] fn completions_cover_the_jobs_and_their_flags() { - let definitions = [train(), prompt()]; + let jobs = [train(), prompt()]; let mut script = Vec::new(); clap_complete::generate( Shell::Bash, - &mut command("program", &jobs(&definitions)), + &mut command("program", &jobs), "program", &mut script, ); diff --git a/crates/tracel-job/src/cli/flags.rs b/crates/tracel-job/src/cli/flags.rs index 6c9651ac..192644b1 100644 --- a/crates/tracel-job/src/cli/flags.rs +++ b/crates/tracel-job/src/cli/flags.rs @@ -408,7 +408,6 @@ fn description(schema: &Map) -> Option { } #[cfg(test)] -#[allow(dead_code)] mod tests { use std::collections::HashMap; @@ -464,6 +463,7 @@ mod tests { #[derive(JsonSchema)] #[serde(rename_all = "camelCase")] + #[expect(dead_code)] struct Shapes { epochs: u32, offset: i64, @@ -475,8 +475,8 @@ mod tests { #[serde(rename = "seed_value")] seed: Option, label: Option, - optimizer: Optimizer, - resume_from: Option, + optimizer: Nested, + resume_from: Option, layers: Vec, labels: HashMap, pair: (u32, String), @@ -490,7 +490,7 @@ mod tests { } #[derive(Serialize, JsonSchema)] - struct Optimizer { + struct Nested { lr: f64, weight_decay: f64, schedule: Schedule, @@ -502,12 +502,14 @@ mod tests { } #[derive(JsonSchema)] - struct Resume { + #[expect(dead_code)] + struct Checkpoint { experiment: u32, } #[derive(JsonSchema)] #[serde(tag = "kind")] + #[expect(dead_code)] enum Tagged { Adam { beta: f64 }, Sgd, @@ -515,18 +517,21 @@ mod tests { #[derive(JsonSchema)] #[serde(untagged)] + #[expect(dead_code)] enum Untagged { Count(u32), Name(String), } #[derive(JsonSchema)] + #[expect(dead_code)] enum Mixed { Unit, Data(u32), } #[derive(JsonSchema)] + #[expect(dead_code)] struct Flattened { workers: u32, } @@ -569,6 +574,7 @@ mod tests { /// How to train. #[derive(JsonSchema)] + #[expect(dead_code)] struct Documented { /// Passes over the data. epochs: u32, @@ -578,11 +584,12 @@ mod tests { fallback: Option, /// The backend to compare with. baseline: Option, - optimizer: Optimizer, + optimizer: Nested, } #[derive(JsonSchema)] #[serde(rename_all = "snake_case")] + #[expect(dead_code)] enum Device { /// The CPU. Cpu, @@ -592,6 +599,7 @@ mod tests { /// The tensor backend. #[derive(JsonSchema)] + #[expect(dead_code)] enum Backend { Wgpu, Cuda, @@ -631,7 +639,7 @@ mod tests { seed: u64, #[serde(default)] schedule: Scheduled, - optimizer: Optimizer, + optimizer: Nested, tag: Option, } @@ -707,6 +715,7 @@ mod tests { } #[derive(JsonSchema)] + #[expect(dead_code)] struct Node { value: u32, next: Option>, @@ -719,7 +728,7 @@ mod tests { assert!(flags_for::<()>().is_empty()); assert!(flags_for::>().is_empty()); assert!(flags_for::().is_empty()); - assert!(flags_for::>().is_empty()); + assert!(flags_for::>().is_empty()); assert!(flags_for::().is_empty()); assert!(flags_of(json!({"type": "object", "properties": {}}), None).is_empty()); assert!( From 52a407f9633262f61c251f58fb4e92a6d346e305 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:35:46 -0400 Subject: [PATCH 26/90] test: the command tests check what tracel-job decides, not clap The command tests searched clap's rendered help and argument settings, matched clap's error kinds and compared ConfigError messages. They now check the input a command line gives: the field a flag sets and the value it reads, that a required flag may be left to the JSON input or the --config file, the order the layers win in, and the ConfigError variant and path. The merge rules are checked on overlay itself, and repeated cases are rstest cases. --- crates/tracel-job/src/cli/command.rs | 459 ++++++++++----------------- 1 file changed, 167 insertions(+), 292 deletions(-) diff --git a/crates/tracel-job/src/cli/command.rs b/crates/tracel-job/src/cli/command.rs index 8156131a..40ad4b55 100644 --- a/crates/tracel-job/src/cli/command.rs +++ b/crates/tracel-job/src/cli/command.rs @@ -263,7 +263,6 @@ fn overlay(input: &mut Value, layer: Value) { #[cfg(test)] mod tests { - use clap::error::ErrorKind; use rstest::rstest; use schemars::JsonSchema; use serde::Serialize; @@ -278,6 +277,7 @@ mod tests { shuffle: bool, tag: String, layers: Vec, + mode: Mode, resume_from: Option, optimizer: Optimizer, } @@ -293,7 +293,6 @@ mod tests { epoch: u32, } - /// `train`, taking `Training`, with a default input. fn train() -> JobDefinition { let default = Training { num_epochs: 10, @@ -301,6 +300,7 @@ mod tests { shuffle: true, tag: "baseline".to_string(), layers: vec![64, 32], + mode: Mode::Fast, resume_from: None, optimizer: Optimizer { lr: 0.001, @@ -318,13 +318,11 @@ mod tests { #[derive(JsonSchema)] #[expect(dead_code)] struct Prompt { - /// The prompt. text: String, mode: Mode, - limit: Option, } - #[derive(JsonSchema)] + #[derive(Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] #[expect(dead_code)] enum Mode { @@ -332,7 +330,6 @@ mod tests { Exact, } - /// `prompt`, taking `Prompt`, with no default input. fn prompt() -> JobDefinition { JobDefinition { name: "prompt".to_string(), @@ -342,7 +339,6 @@ mod tests { } } - /// `evaluate`, with a default input and no schema or description. fn evaluate() -> JobDefinition { JobDefinition { name: "evaluate".to_string(), @@ -352,8 +348,21 @@ mod tests { } } - /// The input the job's command line ` ` gives, its `--config` file read from - /// `files`, by name. + fn named_like_flags() -> JobDefinition { + JobDefinition { + name: "train".to_string(), + description: None, + input_schema: Some(json!({ + "type": "object", + "properties": { + "config": {"type": "string"}, + "version": {"type": "integer"} + } + })), + input_example: None, + } + } + fn input_from( job: &JobDefinition, args: &[&str], @@ -370,114 +379,74 @@ mod tests { })) } - /// The input the job's command line ` ` gives, with no `--config` file to read. fn input(job: &JobDefinition, args: &[&str]) -> Result { input_from(job, args, &[]).map(|input| input.unwrap()) } - /// The job's `--help`. - fn help(job: &JobDefinition) -> String { - job_command(job).render_long_help().to_string() - } - #[test] - fn the_command_has_a_subcommand_per_job() { + fn each_job_is_a_subcommand_about_its_description() { let jobs = [train(), prompt(), evaluate()]; - let mut command = command("program", &jobs); - command.build(); - let abouts: Vec<(&str, Option)> = command + let command = command("program", &jobs); + + let subcommands: Vec<(&str, Option)> = command .get_subcommands() .map(|job| (job.get_name(), job.get_about().map(ToString::to_string))) .collect(); - assert_eq!( - abouts, - [ - ("train", Some("Train the model".to_string())), - ("prompt", Some("Answer a prompt".to_string())), - ("evaluate", None), - ] - ); - let help = command.render_help().to_string(); - assert!(help.contains("Jobs:"), "{help}"); - assert!(help.contains("--completions "), "{help}"); + let expected: Vec<(&str, Option)> = jobs + .iter() + .map(|job| (job.name.as_str(), job.description.clone())) + .collect(); + assert_eq!(subcommands, expected); } - #[test] - fn flags_set_fields_under_their_original_keys() { - let input = input( - &train(), - &[ - "--num-epochs", - "2", - "--optimizer.weight-decay", - "3e-4", - "--tag", - "42", - ], - ) - .unwrap(); + #[rstest] + #[case::at_the_top(&["--num-epochs", "2"], json!({"num_epochs": 2}))] + #[case::in_its_object( + &["--optimizer.weight-decay", "0.5"], + json!({"optimizer": {"weight_decay": 0.5}}) + )] + #[case::beside_another_in_its_object( + &["--optimizer.lr", "0.1", "--optimizer.weight-decay", "0.5"], + json!({"optimizer": {"lr": 0.1, "weight_decay": 0.5}}) + )] + fn a_flag_sets_its_field_under_its_original_keys( + #[case] args: &[&str], + #[case] expected: Value, + ) { + assert_eq!(input(&train(), args).unwrap(), expected); + } + #[test] + fn a_choice_flag_sets_its_field_to_the_value_chosen() { assert_eq!( - input, - json!({"num_epochs": 2, "optimizer": {"weight_decay": 3e-4}, "tag": "42"}) + input(&train(), &["--mode", "exact"]).unwrap(), + json!({"mode": "exact"}) ); } #[test] - fn no_arguments_give_no_input_and_defaults_are_only_shown() { + fn the_defaults_alone_give_no_input() { assert_eq!(input(&train(), &[]).unwrap(), Value::Null); - - let help = help(&train()); - for line in [ - "--num-epochs ", - "[default: 10]", - "--optimizer.weight-decay ", - "[default: 0.00005]", - "--shuffle[=]", - "--layers ", - "[default: [64,32]]", - "--resume-from ", - "[default: null]", - "-c, --config ", - ] { - assert!(help.contains(line), "no {line} in\n{help}"); - } } - #[test] - fn a_flag_parses_as_its_type() { - let train = train(); + #[rstest] + #[case::integer(&["--batch-size", "-1"], json!({"batch_size": -1}))] + #[case::number(&["--optimizer.lr", "-0.5"], json!({"optimizer": {"lr": -0.5}}))] + #[case::json(&["--layers", "-1"], json!({"layers": -1}))] + fn a_numeric_or_json_flag_takes_a_negative_number( + #[case] args: &[&str], + #[case] expected: Value, + ) { + assert_eq!(input(&train(), args).unwrap(), expected); + } - assert_eq!( - input(&train, &["--shuffle", "--batch-size", "-1"]).unwrap(), - json!({"shuffle": true, "batch_size": -1}) - ); - for args in [ - &["--num-epochs", "ten"][..], - &["--num-epochs", "1.5"], - &["--optimizer.lr", "fast"], - &["--shuffle=yes"], - &["--layers", "[1,"], - ] { - let error = input(&train, args).unwrap_err(); - assert!( - matches!( - error.kind(), - ErrorKind::ValueValidation | ErrorKind::InvalidValue - ), - "{args:?}: {error}" - ); - assert_eq!(error.exit_code(), 2); - } - assert_eq!( - input(&train, &["--num-epochs"]).unwrap_err().kind(), - ErrorKind::InvalidValue - ); - assert_eq!( - input(&train, &["--epochs", "2"]).unwrap_err().kind(), - ErrorKind::UnknownArgument - ); + #[rstest] + #[case::integer(&["--num-epochs", "ten"])] + #[case::bool(&["--shuffle=yes"])] + #[case::choice(&["--mode", "slow"])] + fn a_flag_given_a_value_its_field_does_not_take_is_rejected(#[case] args: &[&str]) { + assert!(input(&train(), args).is_err()); } #[rstest] @@ -494,134 +463,97 @@ mod tests { assert_eq!(input(&train(), args).unwrap(), expected); } - #[test] - fn json_literal_flags_take_any_json() { - assert_eq!( - input( - &train(), - &["--layers", "[8]", "--resume-from", r#"{"epoch": 3}"#] - ) - .unwrap(), - json!({"layers": [8], "resume_from": {"epoch": 3}}) - ); - assert_eq!( - input(&train(), &["--resume-from", "null"]).unwrap(), - json!({"resume_from": null}) - ); + #[rstest] + #[case::an_object(json!({"epoch": 3}))] + #[case::null(Value::Null)] + fn a_json_flag_sets_its_field_to_the_json_it_is_given(#[case] value: Value) { + let input = input(&train(), &["--resume-from", &value.to_string()]).unwrap(); + + assert_eq!(input, json!({"resume_from": value})); } #[test] - fn a_schema_gives_flags_their_values_and_help() { - let prompt = prompt(); - - assert_eq!( - input( - &prompt, - &["--text", "hi", "--mode", "exact", "--limit", "3"] - ) - .unwrap(), - json!({"text": "hi", "mode": "exact", "limit": 3}) - ); - assert_eq!( - input(&prompt, &["--text", "hi", "--mode", "slow"]) - .unwrap_err() - .kind(), - ErrorKind::InvalidValue - ); - let help = help(&prompt); - assert!(help.contains("The prompt."), "{help}"); - assert!(help.contains("[possible values: fast, exact]"), "{help}"); + fn a_required_flag_left_out_is_rejected() { + assert!(input(&prompt(), &["--mode", "fast"]).is_err()); } - #[test] - fn a_required_flag_can_be_left_to_the_json_input_or_the_config_file() { - let prompt = prompt(); + #[rstest] + #[case::the_json_input(&[r#"{"text": "hi"}"#])] + #[case::the_config_file(&["--config", "config.json"])] + fn a_required_flag_can_be_left_to(#[case] args: &[&str]) { let files = [("config.json", r#"{"text": "from a file"}"#)]; - let missing = input(&prompt, &["--mode", "fast"]).unwrap_err(); + assert!(input_from(&prompt(), args, &files).is_ok()); + } - assert_eq!(missing.kind(), ErrorKind::MissingRequiredArgument); - assert!(missing.to_string().contains("--text "), "{missing}"); - assert_eq!( - input(&prompt, &[r#"{"text": "hi"}"#]).unwrap(), - json!({"text": "hi"}) - ); - assert_eq!( - input_from(&prompt, &["-c", "config.json"], &files) - .unwrap() - .unwrap(), - json!({"text": "from a file"}) - ); + #[rstest] + #[case::an_object(json!({"tag": null, "layers": [], "optimizer": {"lr": 0.1}}))] + #[case::a_number(json!(7))] + #[case::a_negative_number(json!(-7))] + fn the_json_input_alone_is_the_input_as_given(#[case] given: Value) { + assert_eq!(input(&train(), &[&given.to_string()]).unwrap(), given); } #[test] - fn a_job_without_a_schema_takes_its_json_input_and_config_file_only() { - let evaluate = evaluate(); - let files = [("config.json", r#"{"split": "valid"}"#)]; + fn an_input_that_is_not_json_is_rejected() { + assert!(input(&train(), &["{"]).is_err()); + } - let flag = input(&evaluate, &["--split", "valid"]).unwrap_err(); + #[test] + fn the_config_file_then_the_json_input_then_the_flags_win() { + let files = [( + "config.json", + r#"{"num_epochs": 1, "batch_size": 8, "tag": "file"}"#, + )]; + let args = [ + "--num-epochs", + "3", + "--config", + "config.json", + r#"{"num_epochs": 2, "batch_size": 16}"#, + ]; + + let input = input_from(&train(), &args, &files).unwrap().unwrap(); - assert_eq!(flag.kind(), ErrorKind::UnknownArgument); - assert_eq!(flag.exit_code(), 2); - assert_eq!( - input(&evaluate, &[r#"{"split": "valid"}"#]).unwrap(), - json!({"split": "valid"}) - ); assert_eq!( - input_from(&evaluate, &["--config", "config.json"], &files) - .unwrap() - .unwrap(), - json!({"split": "valid"}) + input, + json!({"num_epochs": 3, "batch_size": 16, "tag": "file"}) ); - let help = help(&evaluate); - assert!(help.contains("[INPUT]"), "{help}"); - assert!(help.contains("-c, --config "), "{help}"); - assert!(!help.contains("--split"), "{help}"); } - #[test] - fn the_json_input_alone_is_the_input_as_given() { - let given = json!({"tag": null, "layers": [], "optimizer": {"lr": 0.1}}); + #[rstest] + #[case::a_value_onto_an_object(json!({"num_epochs": 1}), json!(7))] + #[case::an_object_onto_a_value(json!(7), json!({"num_epochs": 3}))] + fn a_layer_replaces_what_is_below_unless_both_are_objects( + #[case] mut below: Value, + #[case] layer: Value, + ) { + overlay(&mut below, layer.clone()); - assert_eq!(input(&train(), &[&given.to_string()]).unwrap(), given); - assert_eq!(input(&train(), &["7"]).unwrap(), json!(7)); - assert_eq!(input(&train(), &["-7"]).unwrap(), json!(-7)); + assert_eq!(below, layer); } #[test] - fn the_config_file_then_the_json_input_then_the_flags_win() { - let files = [( - "config.json", - r#"{"num_epochs": 1, "batch_size": 8, "tag": "file", "optimizer": {"lr": 0.1, "weight_decay": 0.5}}"#, - )]; + fn an_object_layer_is_overlaid_field_by_field() { + let mut below = json!({"num_epochs": 1, "optimizer": {"lr": 0.1, "weight_decay": 0.5}}); - let input = input_from( - &train(), - &[ - "--num-epochs", - "3", - "--config", - "config.json", - r#"{"num_epochs": 2, "batch_size": 16, "tag": null, "optimizer": {"lr": 0.2}}"#, - "--optimizer.lr", - "0.3", - ], - &files, - ) - .unwrap() - .unwrap(); + overlay(&mut below, json!({"optimizer": {"lr": 0.2}})); assert_eq!( - input, - json!({ - "num_epochs": 3, - "batch_size": 16, - "tag": null, - "optimizer": {"lr": 0.3, "weight_decay": 0.5} - }) + below, + json!({"num_epochs": 1, "optimizer": {"lr": 0.2, "weight_decay": 0.5}}) ); } + #[test] + fn a_null_field_is_kept_rather_than_applied() { + let mut below = json!({"num_epochs": 1, "tag": "file"}); + + overlay(&mut below, json!({"tag": null})); + + assert_eq!(below, json!({"num_epochs": 1, "tag": null})); + } + #[test] fn a_repeated_flag_takes_its_last_value() { assert_eq!( @@ -631,118 +563,61 @@ mod tests { } #[test] - fn an_unreadable_or_invalid_config_file_is_an_error_naming_it() { - let files = [("config.toml", "epochs = 3")]; - let error = |path| { - input_from(&train(), &["-c", path], &files) - .unwrap() - .unwrap_err() - }; + fn an_unreadable_config_file_is_an_error_naming_it() { + let missing = "missing.json"; - let missing = error("missing.json"); - let not_json = error("config.toml"); + let error = input_from(&train(), &["--config", missing], &[]) + .unwrap() + .unwrap_err(); assert!( - matches!(&missing, ConfigError::Unreadable { path, .. } if path == Path::new("missing.json")), - "{missing:?}" - ); - assert!( - missing - .to_string() - .starts_with("cannot read the --config file missing.json: ") - ); - assert!( - matches!(¬_json, ConfigError::NotJson { .. }), - "{not_json:?}" - ); - assert!( - not_json - .to_string() - .starts_with("the --config file config.toml is not JSON: ") + matches!(error, ConfigError::Unreadable { path, .. } if path == Path::new(missing)) ); } #[test] - fn reserved_names_belong_to_the_command_line() { - let job = JobDefinition { - name: "train".to_string(), - description: None, - input_schema: Some(json!({ - "type": "object", - "properties": { - "config": {"type": "string"}, - "help": {"type": "boolean"}, - "version": {"type": "integer"}, - "epochs": {"type": "integer"} - } - })), - input_example: None, - }; - let files = [("large.json", r#"{"config": "large"}"#)]; + fn a_config_file_that_is_not_json_is_an_error_naming_it() { + let toml = "config.toml"; - let command = job_command(&job); - let longs: Vec<&str> = command.get_arguments().filter_map(Arg::get_long).collect(); - assert_eq!(longs, ["epochs", "version", "config"]); - assert_eq!( - input_from(&job, &["--config", "large.json"], &files) - .unwrap() - .unwrap(), - json!({"config": "large"}) - ); - assert_eq!( - input(&job, &["--help"]).unwrap_err().kind(), - ErrorKind::DisplayHelp - ); - assert_eq!( - input(&job, &["--version", "3"]).unwrap(), - json!({"version": 3}) - ); + let error = input_from(&train(), &["--config", toml], &[(toml, "epochs = 3")]) + .unwrap() + .unwrap_err(); + + assert!(matches!(error, ConfigError::NotJson { path, .. } if path == Path::new(toml))); } #[test] - fn completions_names_the_shell_the_command_line_asks_for() { - let jobs = [train()]; - let matches = |args: &[&str]| { - command("program", &jobs) - .try_get_matches_from(["program"].iter().chain(args)) - .unwrap() - }; + fn the_config_flag_names_the_config_file_beside_a_field_named_config() { + let files = [("large.json", r#"{"config": "large"}"#)]; - assert_eq!( - completions(&matches(&["--completions", "zsh"])), - Some(Shell::Zsh) - ); - assert_eq!(completions(&matches(&["train"])), None); - assert_eq!( - command("program", &jobs) - .try_get_matches_from(["program", "--completions", "tcsh"]) - .unwrap_err() - .kind(), - ErrorKind::InvalidValue - ); + let input = input_from(&named_like_flags(), &["--config", "large.json"], &files) + .unwrap() + .unwrap(); + + assert_eq!(input, json!({"config": "large"})); } #[test] - fn completions_cover_the_jobs_and_their_flags() { - let jobs = [train(), prompt()]; - let mut script = Vec::new(); - - clap_complete::generate( - Shell::Bash, - &mut command("program", &jobs), - "program", - &mut script, + fn a_field_named_version_is_set_by_its_flag() { + assert_eq!( + input(&named_like_flags(), &["--version", "3"]).unwrap(), + json!({"version": 3}) ); + } - let script = String::from_utf8(script).unwrap(); - for word in [ - "train", - "prompt", - "--optimizer.weight-decay", - "--config", - "--completions", - ] { - assert!(script.contains(word), "no {word} in the script"); - } + #[rstest] + #[case::when_asked(&["--completions", "zsh"], Some(Shell::Zsh))] + #[case::when_a_job_is_named(&["train"], None)] + fn completions_names_the_shell_the_command_line_asks_for( + #[case] args: &[&str], + #[case] shell: Option, + ) { + let jobs = [train()]; + + let matches = command("program", &jobs) + .try_get_matches_from(["program"].iter().chain(args)) + .unwrap(); + + assert_eq!(completions(&matches), shell); } } From c851e56b9f13323b9312fa7c9c14c604048d2e55 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 13:10:36 -0400 Subject: [PATCH 27/90] test: cover the schema branches nothing tested Tests now show that an object that may be null takes a JSON literal, and that listed values leave out null, take JSON when one is not a scalar, and keep numbers as numbers. --- crates/tracel-job/src/cli/flags.rs | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/tracel-job/src/cli/flags.rs b/crates/tracel-job/src/cli/flags.rs index 192644b1..820e2d88 100644 --- a/crates/tracel-job/src/cli/flags.rs +++ b/crates/tracel-job/src/cli/flags.rs @@ -461,6 +461,15 @@ mod tests { ) } + fn one_of(values: &[Value]) -> Kind { + Kind::Choice( + values + .iter() + .map(|value| Choice::new(value, None).unwrap()) + .collect(), + ) + } + #[derive(JsonSchema)] #[serde(rename_all = "camelCase")] #[expect(dead_code)] @@ -572,6 +581,21 @@ mod tests { assert_eq!(weight_decay.id(), "optimizer.weight_decay"); } + #[test] + fn an_object_that_may_be_null_takes_json() { + let schema = json!({ + "type": "object", + "properties": { + "resume": { + "type": ["object", "null"], + "properties": {"epoch": {"type": "integer"}} + } + } + }); + + assert_eq!(flag(&flags_of(schema, None), "resume").kind, Kind::Json); + } + /// How to train. #[derive(JsonSchema)] #[expect(dead_code)] @@ -630,6 +654,25 @@ mod tests { } } + #[rstest] + #[case::leaving_out_null(json!(["adam", null]), &[json!("adam")])] + #[case::keeping_numbers_as_numbers(json!([1, 2.5]), &[json!(1), json!(2.5)])] + fn listed_values_are_the_choices_of_their_flag( + #[case] listed: Value, + #[case] values: &[Value], + ) { + let schema = json!({"type": "object", "properties": {"field": {"enum": listed}}}); + + assert_eq!(flag(&flags_of(schema, None), "field").kind, one_of(values)); + } + + #[test] + fn listed_values_of_which_one_is_not_a_scalar_take_json() { + let schema = json!({"type": "object", "properties": {"field": {"enum": ["adam", [1]]}}}); + + assert_eq!(flag(&flags_of(schema, None), "field").kind, Kind::Json); + } + #[derive(Serialize, JsonSchema)] struct Defaulted { epochs: u32, From fa8138009593c8335bf86f3b9a166573f7d3c214 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:30:42 -0400 Subject: [PATCH 28/90] refactor: split the local store into one module per job local.rs ran to about a thousand lines doing six jobs. It is now local/ with a module each for run directories and numbering, the status file, the events line format, and artifacts, the session and provider staying in mod.rs. The events line format is the file contract other tools read, so it gets its own module and tests. A pure move: no behaviour changes, and each test moves to the module that owns what it checks. --- crates/tracel-experiment/src/local.rs | 1013 ----------------- .../tracel-experiment/src/local/artifacts.rs | 184 +++ crates/tracel-experiment/src/local/events.rs | 330 ++++++ crates/tracel-experiment/src/local/mod.rs | 293 +++++ crates/tracel-experiment/src/local/runs.rs | 106 ++ crates/tracel-experiment/src/local/status.rs | 154 +++ .../src/local/test_support.rs | 26 + 7 files changed, 1093 insertions(+), 1013 deletions(-) delete mode 100644 crates/tracel-experiment/src/local.rs create mode 100644 crates/tracel-experiment/src/local/artifacts.rs create mode 100644 crates/tracel-experiment/src/local/events.rs create mode 100644 crates/tracel-experiment/src/local/mod.rs create mode 100644 crates/tracel-experiment/src/local/runs.rs create mode 100644 crates/tracel-experiment/src/local/status.rs create mode 100644 crates/tracel-experiment/src/local/test_support.rs diff --git a/crates/tracel-experiment/src/local.rs b/crates/tracel-experiment/src/local.rs deleted file mode 100644 index fae5e008..00000000 --- a/crates/tracel-experiment/src/local.rs +++ /dev/null @@ -1,1013 +0,0 @@ -//! Recording experiments on this machine, without reaching any server. - -use std::collections::HashMap; -use std::fs::{self, File, OpenOptions}; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use std::sync::mpsc::{Receiver, Sender, channel}; -use std::thread::{self, JoinHandle}; - -use chrono::{SecondsFormat, Utc}; -use serde::Serialize; -use serde_json::{Map, Value}; -use tracel_artifact::bundle::FsBundle; - -use crate::error::{ExperimentError, ExperimentErrorKind}; -use crate::reader::{ArtifactRef, ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact}; -use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; -use crate::{ - ActivityEvent, ActivitySpec, ActivityStatus, ArtifactKind, CancelToken, ExperimentId, - ExperimentLocation, ExperimentProvider, ExperimentRun, MetricSpec, MetricValue, -}; - -/// The file a run appends its events to, one JSON object per line. -const EVENTS_FILE: &str = "events.jsonl"; -/// The file a run keeps its [`Status`] in. -const STATUS_FILE: &str = "status.json"; -/// The `.gitignore` written in a runs directory this backend creates: everything in it stays out -/// of git and of code packages. -const RUNS_GITIGNORE: &str = "# Created by tracel: experiment runs recorded on this machine.\n*\n"; - -/// Records experiments under a directory on this machine. -/// -/// The runs of the experiment named `name` are numbered from 1 under `/`. Each run -/// keeps its status in `status.json`, written when the run starts and again when it ends, appends -/// its events to `events.jsonl`, and saves its artifacts under `artifacts/`. Creating a -/// `LocalExperiments` performs no I/O; the first run creates the directories. When it creates -/// `dir` itself, it writes a `.gitignore` there that ignores everything under it, so the records -/// stay out of git and of code packages; a `dir` that already exists is left as it is. -/// -/// `status.json` names the experiment and numbers the run, says whether the run is `running`, -/// `completed` or `failed`, when it started and ended, in RFC 3339 UTC to the second, and why it -/// failed: -/// -/// ```json -/// { -/// "name": "mnist", -/// "num": 3, -/// "status": "failed", -/// "started_at": "2026-10-06T14:02:11Z", -/// "finished_at": "2026-10-06T14:31:40Z", -/// "error": "loss is NaN" -/// } -/// ``` -/// -/// Each line of `events.jsonl` is one event, a JSON object whose `type` names it, with the -/// fields the console's experiment API gives it: -/// -/// | `type` | Fields | -/// | --- | --- | -/// | `attribute` | `key`, `value` | -/// | `arguments` | `value`, the run's input, which the console lists as the experiment's `config`; the last one is the run's | -/// | `config` | `name`, `value` | -/// | `log` | `timestamp`, `level`, `message`, `metadata`, `activity` | -/// | `metric` | `name`, `epoch`, `iteration`, `value`, `group`, `activity` | -/// | `metric_definition` | `name`, `description`, `unit`, `higher_is_better` | -/// | `epoch_summary` | `name`, `epoch`, `group`, `value`, `activity` | -/// | `summary` | `name`, `value`, `activity` | -/// | `input_used` | `experiment`, `name`, `artifact_id` | -/// | `activity_started` | `activity`: `id`, `parent`, `name`, `cancellable`, `meter`, `attributes` | -/// | `activity_updated` | `id`, `current` | -/// | `activity_message` | `id`, `message` | -/// | `activity_finished` | `id`, `status`, `message` | -/// -/// `activity` is the ID of the activity an event was recorded in, or `null`. A metric logged with -/// several values is one `metric` line per value. -/// -/// ``` -/// use std::sync::Arc; -/// -/// use tracel_experiment::Experiments; -/// use tracel_experiment::local::LocalExperiments; -/// -/// let experiments = Experiments::new(Arc::new(LocalExperiments::new("./runs"))); -/// ``` -#[derive(Debug, Clone)] -pub struct LocalExperiments { - dir: PathBuf, -} - -impl LocalExperiments { - /// Records experiments under `dir`. - pub fn new(dir: impl Into) -> Self { - Self { dir: dir.into() } - } -} - -impl ExperimentProvider for LocalExperiments { - fn create_experiment( - &self, - name: String, - attributes: HashMap, - ) -> Result { - let internal = |message: &'static str| { - move |error| ExperimentError::with_source(ExperimentErrorKind::Internal, message, error) - }; - create_runs_dir(&self.dir) - .map_err(internal("Failed to create local experiment directory"))?; - let root = self.dir.join(&name); - let root = root - .canonicalize() - .or_else(|_| { - fs::create_dir_all(&root)?; - root.canonicalize() - }) - .map_err(internal("Failed to create local experiment directory"))?; - let (num, run_root) = create_local_run_dir(&root) - .map_err(internal("Failed to create local experiment run directory"))?; - let session = LocalExperimentSession::start(run_root.clone(), name, num, attributes) - .map_err(internal("Failed to initialize local experiment session"))?; - let reader = LocalExperimentReader { root }; - - let run = ExperimentRun::new(num.to_string(), session, reader, CancelToken::new()); - Ok(run.with_location(ExperimentLocation::Dir(run_root))) - } -} - -/// Creates `dir`, the directory runs are recorded under, with a [`RUNS_GITIGNORE`] in it, unless -/// it already exists. -fn create_runs_dir(dir: &Path) -> io::Result<()> { - if let Some(parent) = dir.parent().filter(|parent| !parent.as_os_str().is_empty()) { - fs::create_dir_all(parent)?; - } - match fs::create_dir(dir) { - Ok(()) => fs::write(dir.join(".gitignore"), RUNS_GITIGNORE), - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => Ok(()), - Err(error) => Err(error), - } -} - -struct LocalExperimentSession { - root: PathBuf, - status: Status, - active: Mutex>, -} - -impl LocalExperimentSession { - /// Starts recording the run `num` of the experiment `name` in `root`: writes its status as - /// running and its attributes as its first events. - fn start( - root: PathBuf, - name: String, - num: u64, - attributes: HashMap, - ) -> io::Result { - fs::create_dir_all(root.join("artifacts"))?; - let mut attributes: Vec<(String, Value)> = attributes.into_iter().collect(); - attributes.sort_by(|(a, _), (b, _)| a.cmp(b)); - let attributes = attributes - .into_iter() - .map(|(key, value)| Line::Attribute { key, value }); - let mut events = OpenOptions::new() - .create(true) - .append(true) - .open(root.join(EVENTS_FILE))?; - events.write_all(to_jsonl(attributes)?.as_bytes())?; - let status = Status::running(name, num); - status.write(&root.join(STATUS_FILE))?; - let (sender, receiver) = channel(); - let join = thread::spawn(move || local_worker(receiver, events)); - - Ok(Self { - root, - status, - active: Mutex::new(Some(LocalWorker { sender, join })), - }) - } - - fn sender(&self) -> Result, ExperimentError> { - let guard = self.active.lock().unwrap(); - guard - .as_ref() - .map(|worker| worker.sender.clone()) - .ok_or_else(|| { - ExperimentError::new( - ExperimentErrorKind::AlreadyFinished, - "Local experiment session has already finished", - ) - }) - } - - /// Stops the writer once it has written every event, then writes the run's status. - fn finish_worker(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { - let worker = self.active.lock().unwrap().take().ok_or_else(|| { - ExperimentError::new( - ExperimentErrorKind::AlreadyFinished, - "Local experiment session has already finished", - ) - })?; - - let send_result = worker.sender.send(LocalWrite::Finish); - match worker.join.join() { - Ok(Ok(())) if send_result.is_ok() => {} - Ok(Ok(())) => { - return Err(ExperimentError::new( - ExperimentErrorKind::Internal, - "Failed to send local experiment completion", - )); - } - Ok(Err(err)) => { - return Err(ExperimentError::with_source( - ExperimentErrorKind::Internal, - "Local experiment writer failed", - err, - )); - } - Err(_) => { - return Err(ExperimentError::new( - ExperimentErrorKind::Internal, - "Local experiment writer thread panicked", - )); - } - } - - self.status - .ended(completion) - .write(&self.root.join(STATUS_FILE)) - .map_err(|err| { - ExperimentError::with_source( - ExperimentErrorKind::Internal, - "Failed to write the local experiment status", - err, - ) - }) - } -} - -impl ExperimentSession for LocalExperimentSession { - fn record_event(&self, event: Event) -> Result<(), ExperimentError> { - let lines = to_jsonl(lines(event)).map_err(|err| { - ExperimentError::with_source( - ExperimentErrorKind::Internal, - "Failed to serialize local experiment event", - err, - ) - })?; - self.sender()?.send(LocalWrite::Lines(lines)).map_err(|_| { - ExperimentError::new( - ExperimentErrorKind::Internal, - "Failed to queue local experiment event", - ) - }) - } - - fn save_artifact( - &self, - name: &str, - _kind: ArtifactKind, - artifact: Box, - ) -> Result<(), ExperimentError> { - let artifact_root = self.root.join("artifacts").join(name); - if artifact_root.exists() { - fs::remove_dir_all(&artifact_root).map_err(|err| { - ExperimentError::with_source( - ExperimentErrorKind::Artifact, - "Failed to replace existing local artifact", - err, - ) - })?; - } - - let mut bundle = FsBundle::create(artifact_root.clone()).map_err(|err| { - ExperimentError::with_source( - ExperimentErrorKind::Artifact, - "Failed to create local artifact bundle", - err, - ) - })?; - - let res = artifact(&mut bundle); - - if res.is_err() { - _ = bundle.delete(); - } - - res - } - - fn finish(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { - self.finish_worker(completion) - } -} - -/// How a run is going, as its `status.json` gives it. -#[derive(Clone, Serialize)] -struct Status { - name: String, - num: u64, - status: StatusKind, - started_at: String, - finished_at: Option, - error: Option, -} - -#[derive(Clone, Copy, Serialize)] -#[serde(rename_all = "snake_case")] -enum StatusKind { - Running, - Completed, - Failed, -} - -impl Status { - /// The status of the run `num` of the experiment `name`, started now. - fn running(name: String, num: u64) -> Self { - Self { - name, - num, - status: StatusKind::Running, - started_at: now(), - finished_at: None, - error: None, - } - } - - /// The status of the run once it ended now with `completion`. - fn ended(&self, completion: ExperimentCompletion) -> Self { - let (status, error) = match completion { - ExperimentCompletion::Success => (StatusKind::Completed, None), - ExperimentCompletion::Failed(reason) => (StatusKind::Failed, Some(reason)), - }; - Self { - status, - finished_at: Some(now()), - error, - ..self.clone() - } - } - - /// Writes the status to `path` as pretty-printed JSON: to `.tmp` first, then renamed to - /// `path`, so it is never read half written. - fn write(&self, path: &Path) -> io::Result<()> { - let mut contents = serde_json::to_vec_pretty(self)?; - contents.push(b'\n'); - let tmp = path.with_extension("json.tmp"); - let written = fs::write(&tmp, contents).and_then(|()| fs::rename(&tmp, path)); - if written.is_err() { - let _ = fs::remove_file(&tmp); - } - written - } -} - -/// The current time, in RFC 3339 UTC to the second. -fn now() -> String { - Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) -} - -struct LocalWorker { - sender: Sender, - join: JoinHandle>, -} - -enum LocalWrite { - /// Lines to append to `events.jsonl`, each ending with a newline. - Lines(String), - /// Stop once every line sent before is written. - Finish, -} - -/// One line of `events.jsonl`, with the fields the console's experiment API gives the event. -#[derive(Serialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum Line { - Attribute { - key: String, - value: Value, - }, - Arguments { - value: Value, - }, - Config { - name: String, - value: Value, - }, - Log { - timestamp: String, - level: &'static str, - message: String, - metadata: Map, - activity: Option, - }, - Metric { - name: String, - epoch: usize, - iteration: usize, - value: f64, - group: String, - activity: Option, - }, - MetricDefinition { - name: String, - description: Option, - unit: Option, - higher_is_better: bool, - }, - EpochSummary { - name: String, - epoch: usize, - group: String, - value: f64, - activity: Option, - }, - Summary { - name: String, - value: f64, - activity: Option, - }, - InputUsed { - experiment: String, - name: String, - artifact_id: String, - }, - ActivityStarted { - activity: ActivitySpec, - }, - ActivityUpdated { - id: u64, - current: u64, - }, - ActivityMessage { - id: u64, - message: String, - }, - ActivityFinished { - id: u64, - status: ActivityStatus, - message: Option, - }, -} - -/// The lines `event` is recorded as: one, or one per value of a metric event. -fn lines(event: Event) -> Vec { - match event { - Event::Args(value) => vec![Line::Arguments { value }], - Event::Config { name, value } => vec![Line::Config { name, value }], - Event::Log { record, activity } => vec![Line::Log { - timestamp: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - level: record.level.as_str(), - message: record.message, - metadata: record.attributes, - activity: activity.map(|id| id.as_u64()), - }], - Event::Metrics { - epoch, - split, - iteration, - items, - activity, - } => items - .into_iter() - .map(|MetricValue { name, value }| Line::Metric { - name, - epoch, - iteration, - value, - group: split.clone(), - activity: activity.map(|id| id.as_u64()), - }) - .collect(), - Event::MetricDefinition(MetricSpec { - name, - description, - unit, - higher_is_better, - }) => vec![Line::MetricDefinition { - name, - description, - unit, - higher_is_better, - }], - Event::EpochSummary { - epoch, - split, - items, - activity, - } => items - .into_iter() - .map(|MetricValue { name, value }| Line::EpochSummary { - name, - epoch, - group: split.clone(), - value, - activity: activity.map(|id| id.as_u64()), - }) - .collect(), - Event::Summary { items, activity } => items - .into_iter() - .map(|MetricValue { name, value }| Line::Summary { - name, - value, - activity: activity.map(|id| id.as_u64()), - }) - .collect(), - Event::ArtifactUsed { - experiment_id, - reference, - } => vec![Line::InputUsed { - experiment: experiment_id.to_string(), - name: reference.name, - artifact_id: reference.id, - }], - Event::Activity(event) => vec![match event { - ActivityEvent::Started { activity } => Line::ActivityStarted { activity }, - ActivityEvent::Updated { id, current } => Line::ActivityUpdated { - id: id.as_u64(), - current, - }, - ActivityEvent::Message { id, message } => Line::ActivityMessage { - id: id.as_u64(), - message, - }, - ActivityEvent::Finished { - id, - status, - message, - } => Line::ActivityFinished { - id: id.as_u64(), - status, - message, - }, - }], - } -} - -/// `lines` as JSON Lines: each one JSON object followed by a newline. -fn to_jsonl(lines: impl IntoIterator) -> serde_json::Result { - let mut jsonl = String::new(); - for line in lines { - jsonl.push_str(&serde_json::to_string(&line)?); - jsonl.push('\n'); - } - Ok(jsonl) -} - -struct LocalExperimentReader { - root: PathBuf, -} - -impl ExperimentArtifactReader for LocalExperimentReader { - fn load_artifact_raw( - &self, - experiment_id: ExperimentId, - name: &str, - ) -> Result { - let experiment_root = parse_local_experiment_root(&self.root, &experiment_id)?; - let artifact_root = experiment_root.join("artifacts").join(name); - - if !artifact_root.is_dir() { - return Err(ExperimentReaderError::new(format!( - "Local artifact not found: {}", - artifact_root.display() - ))); - } - - let files = collect_bundle_files(&artifact_root, &artifact_root).map_err(|err| { - ExperimentReaderError::with_source("Failed to inspect local artifact files", err) - })?; - let bundle = FsBundle::with_files(artifact_root.clone(), files).map_err(|err| { - ExperimentReaderError::with_source("Failed to create local artifact bundle", err) - })?; - - Ok(LoadedArtifact::new( - ArtifactRef { - id: artifact_root.to_string_lossy().to_string(), - name: name.to_string(), - }, - bundle, - )) - } -} - -fn parse_local_experiment_root( - root: &Path, - experiment_id: &ExperimentId, -) -> Result { - let experiment_dir = experiment_id.as_str(); - if experiment_dir.is_empty() || experiment_dir.contains('/') || experiment_dir.contains('\\') { - return Err(ExperimentReaderError::new( - "Invalid local experiment ID format", - )); - } - - Ok(root.join(experiment_dir)) -} - -/// Appends each batch of lines to `events` as it arrives, until told to finish. -fn local_worker(receiver: Receiver, mut events: File) -> Result<(), io::Error> { - while let Ok(message) = receiver.recv() { - match message { - LocalWrite::Lines(lines) => events.write_all(lines.as_bytes())?, - LocalWrite::Finish => break, - } - } - Ok(()) -} - -fn collect_bundle_files(root: &Path, current: &Path) -> Result, std::io::Error> { - let mut files = Vec::new(); - for entry in fs::read_dir(current)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() { - files.extend(collect_bundle_files(root, &path)?); - continue; - } - - let rel = path - .strip_prefix(root) - .map_err(std::io::Error::other)? - .to_string_lossy() - .to_string(); - files.push(rel); - } - Ok(files) -} - -/// Creates the directory of the next run under `root`, numbered after the highest one there, and -/// returns its number and path. -fn create_local_run_dir(root: &Path) -> Result<(u64, PathBuf), std::io::Error> { - let mut next_id = 1u64; - - for entry in fs::read_dir(root)? { - let entry = entry?; - if !entry.file_type()?.is_dir() { - continue; - } - - if let Some(id) = entry - .file_name() - .to_str() - .and_then(|name| name.parse::().ok()) - { - next_id = next_id.max(id + 1); - } - } - - loop { - let run_root = root.join(next_id.to_string()); - match fs::create_dir(&run_root) { - Ok(()) => return Ok((next_id, run_root)), - Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { - next_id += 1; - } - Err(err) => return Err(err), - } - } -} - -#[cfg(test)] -mod tests { - use std::io::Read; - - use serde_json::json; - use tracel_artifact::bundle::{BundleDecode, BundleEncode, BundleSink, BundleSource}; - - use super::*; - use crate::LogRecord; - - /// A one-file artifact. - #[derive(Debug, PartialEq)] - struct Note(String); - - impl BundleEncode for Note { - type Settings = (); - type Error = String; - - fn encode(self, sink: &mut O, _settings: &()) -> Result<(), String> { - sink.put_bytes("note.txt", self.0.as_bytes()) - } - } - - impl BundleDecode for Note { - type Settings = (); - type Error = String; - - fn decode(source: &I, _settings: &()) -> Result { - let mut text = String::new(); - source - .open("note.txt")? - .read_to_string(&mut text) - .map_err(|error| error.to_string())?; - Ok(Self(text)) - } - } - - fn run(dir: &Path, name: &str) -> ExperimentRun { - LocalExperiments::new(dir) - .create_experiment(name.to_string(), HashMap::new()) - .unwrap() - } - - #[test] - fn a_runs_directory_it_creates_ignores_everything_in_it() { - let dir = tempfile::tempdir().unwrap(); - let runs = dir.path().join("project/runs"); - - let _run = run(&runs, "mnist"); - - assert_eq!( - fs::read_to_string(runs.join(".gitignore")).unwrap(), - RUNS_GITIGNORE - ); - } - - #[test] - fn a_runs_directory_that_exists_is_left_as_it_is() { - let dir = tempfile::tempdir().unwrap(); - - let _run = run(dir.path(), "mnist"); - - assert!(!dir.path().join(".gitignore").exists()); - } - - #[test] - fn runs_are_numbered_from_one_under_the_experiment_name() { - let dir = tempfile::tempdir().unwrap(); - - let first = run(dir.path(), "mnist"); - let second = run(dir.path(), "mnist"); - let other = run(dir.path(), "other"); - - assert_eq!(first.id().as_str(), "1"); - assert_eq!(second.id().as_str(), "2"); - assert_eq!(other.id().as_str(), "1"); - assert!(dir.path().join("mnist/1").is_dir()); - assert!(dir.path().join("mnist/2").is_dir()); - assert!(dir.path().join("other/1").is_dir()); - } - - #[test] - fn numbering_continues_after_the_highest_existing_run() { - let dir = tempfile::tempdir().unwrap(); - fs::create_dir_all(dir.path().join("mnist/7")).unwrap(); - fs::create_dir_all(dir.path().join("mnist/notes")).unwrap(); - - assert_eq!(run(dir.path(), "mnist").id().as_str(), "8"); - } - - fn read_lines(path: &Path) -> Vec { - fs::read_to_string(path) - .unwrap() - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .collect() - } - - fn read_json(path: &Path) -> Value { - serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap() - } - - #[test] - fn a_run_records_each_event_as_a_line_of_json() { - let dir = tempfile::tempdir().unwrap(); - let experiment = LocalExperiments::new(dir.path()) - .create_experiment( - "mnist".to_string(), - HashMap::from([ - ("tracel.job_num".to_string(), json!(12)), - ("kind".to_string(), json!("example")), - ]), - ) - .unwrap(); - - experiment.log_args(&json!({"epochs": 2})).unwrap(); - experiment - .log_config("optimizer", &json!({"lr": 0.01})) - .unwrap(); - experiment.log_metric_definition(MetricSpec { - name: "loss".to_string(), - description: Some("training loss".to_string()), - unit: None, - higher_is_better: false, - }); - let epoch = experiment.activity("epoch 1").meter(4, "batch").start(); - epoch.log(LogRecord::warn("slow batch").with("batch", 3)); - epoch.log_metric( - 1, - "train", - 4, - vec![ - MetricValue { - name: "loss".to_string(), - value: 0.5, - }, - MetricValue { - name: "accuracy".to_string(), - value: 0.9, - }, - ], - ); - epoch.inc(1); - epoch.finish_with_message("done"); - experiment.log_epoch_summary( - 1, - "train", - vec![MetricValue { - name: "loss".to_string(), - value: 0.4, - }], - ); - experiment.log_summary(vec![MetricValue { - name: "accuracy".to_string(), - value: 0.95, - }]); - experiment.finish().unwrap(); - - let mut lines = read_lines(&dir.path().join("mnist/1/events.jsonl")); - let timestamp = lines[6]["timestamp"].take(); - assert!( - timestamp - .as_str() - .is_some_and(|t| chrono::DateTime::parse_from_rfc3339(t).is_ok()), - "{timestamp}" - ); - assert_eq!( - lines, - [ - json!({"type": "attribute", "key": "kind", "value": "example"}), - json!({"type": "attribute", "key": "tracel.job_num", "value": 12}), - json!({"type": "arguments", "value": {"epochs": 2}}), - json!({"type": "config", "name": "optimizer", "value": {"lr": 0.01}}), - json!({ - "type": "metric_definition", - "name": "loss", - "description": "training loss", - "unit": null, - "higher_is_better": false - }), - json!({ - "type": "activity_started", - "activity": { - "id": 1, - "parent": null, - "name": "epoch 1", - "cancellable": false, - "meter": {"unit": "batch", "total": 4}, - "attributes": {} - } - }), - json!({ - "type": "log", - "timestamp": null, - "level": "warn", - "message": "slow batch", - "metadata": {"batch": 3}, - "activity": 1 - }), - json!({ - "type": "metric", - "name": "loss", - "epoch": 1, - "iteration": 4, - "value": 0.5, - "group": "train", - "activity": 1 - }), - json!({ - "type": "metric", - "name": "accuracy", - "epoch": 1, - "iteration": 4, - "value": 0.9, - "group": "train", - "activity": 1 - }), - json!({"type": "activity_updated", "id": 1, "current": 1}), - json!({ - "type": "activity_finished", - "id": 1, - "status": "Success", - "message": "done" - }), - json!({ - "type": "epoch_summary", - "name": "loss", - "epoch": 1, - "group": "train", - "value": 0.4, - "activity": null - }), - json!({"type": "summary", "name": "accuracy", "value": 0.95, "activity": null}), - ] - ); - } - - fn is_utc_seconds(timestamp: &Value) -> bool { - timestamp.as_str().is_some_and(|timestamp| { - timestamp.ends_with('Z') - && !timestamp.contains('.') - && chrono::DateTime::parse_from_rfc3339(timestamp).is_ok() - }) - } - - #[test] - fn the_status_follows_the_run_from_the_start_to_the_end() { - let dir = tempfile::tempdir().unwrap(); - let experiment = run(dir.path(), "mnist"); - let run_dir = dir.path().join("mnist/1"); - - let mut running = read_json(&run_dir.join("status.json")); - experiment.finish().unwrap(); - let mut completed = read_json(&run_dir.join("status.json")); - - assert!(is_utc_seconds(&running["started_at"]), "{running}"); - assert!(is_utc_seconds(&completed["finished_at"]), "{completed}"); - assert_eq!(completed["started_at"], running["started_at"]); - running["started_at"] = Value::Null; - completed["started_at"] = Value::Null; - completed["finished_at"] = Value::Null; - assert_eq!( - running, - json!({ - "name": "mnist", - "num": 1, - "status": "running", - "started_at": null, - "finished_at": null, - "error": null - }) - ); - assert_eq!( - completed, - json!({ - "name": "mnist", - "num": 1, - "status": "completed", - "started_at": null, - "finished_at": null, - "error": null - }) - ); - assert!(!run_dir.join("status.json.tmp").exists()); - } - - #[test] - fn a_run_says_how_it_ended_in_its_status() { - let dir = tempfile::tempdir().unwrap(); - run(dir.path(), "mnist").fail("diverged").unwrap(); - let cancelled = run(dir.path(), "mnist"); - cancelled.cancel().unwrap(); - cancelled.finish().unwrap(); - drop(run(dir.path(), "mnist")); - - let status = |num: u32| read_json(&dir.path().join(format!("mnist/{num}/status.json"))); - let (failed, cancelled, dropped) = (status(1), status(2), status(3)); - assert_eq!(failed["status"], "failed"); - assert_eq!(failed["error"], "diverged"); - assert_eq!(cancelled["status"], "completed"); - assert_eq!(cancelled["error"], Value::Null); - assert_eq!(dropped["status"], "failed"); - assert_eq!( - dropped["error"], - "the run was dropped without finish() or fail()" - ); - } - - #[test] - fn a_run_is_located_in_its_directory() { - let dir = tempfile::tempdir().unwrap(); - - let experiment = run(dir.path(), "mnist"); - - assert_eq!( - experiment.location(), - Some(&ExperimentLocation::Dir( - dir.path().canonicalize().unwrap().join("mnist/1") - )) - ); - } - - #[test] - fn a_saved_artifact_loads_back_from_a_later_run() { - let dir = tempfile::tempdir().unwrap(); - let first = run(dir.path(), "mnist"); - first - .save_artifact("note", ArtifactKind::Other, Note("kept".to_string()), &()) - .unwrap(); - let first_id = first.id().clone(); - first.finish().unwrap(); - - let second = run(dir.path(), "mnist"); - let note: Note = second.use_artifact(first_id, "note", &()).unwrap(); - - assert_eq!(note, Note("kept".to_string())); - } - - #[test] - fn an_experiment_id_naming_another_directory_is_refused() { - let root = Path::new("/runs/mnist"); - - for id in ["", "../other", "a/b", "a\\b"] { - assert!( - parse_local_experiment_root(root, &ExperimentId::new(id)).is_err(), - "{id:?}" - ); - } - assert_eq!( - parse_local_experiment_root(root, &ExperimentId::new("3")).unwrap(), - root.join("3") - ); - } -} diff --git a/crates/tracel-experiment/src/local/artifacts.rs b/crates/tracel-experiment/src/local/artifacts.rs new file mode 100644 index 00000000..1833e4a6 --- /dev/null +++ b/crates/tracel-experiment/src/local/artifacts.rs @@ -0,0 +1,184 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use tracel_artifact::bundle::FsBundle; + +use crate::ExperimentId; +use crate::error::{ExperimentError, ExperimentErrorKind}; +use crate::reader::{ArtifactRef, ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact}; +use crate::session::BundleFn; + +/// Saves the artifact `name` of the run recorded in `root`, replacing the one saved before. +pub fn save_artifact( + root: &Path, + name: &str, + artifact: Box, +) -> Result<(), ExperimentError> { + let artifact_root = root.join("artifacts").join(name); + if artifact_root.exists() { + fs::remove_dir_all(&artifact_root).map_err(|err| { + ExperimentError::with_source( + ExperimentErrorKind::Artifact, + "Failed to replace existing local artifact", + err, + ) + })?; + } + + let mut bundle = FsBundle::create(artifact_root.clone()).map_err(|err| { + ExperimentError::with_source( + ExperimentErrorKind::Artifact, + "Failed to create local artifact bundle", + err, + ) + })?; + + let res = artifact(&mut bundle); + + if res.is_err() { + _ = bundle.delete(); + } + + res +} + +pub struct LocalExperimentReader { + pub root: PathBuf, +} + +impl ExperimentArtifactReader for LocalExperimentReader { + fn load_artifact_raw( + &self, + experiment_id: ExperimentId, + name: &str, + ) -> Result { + let experiment_root = parse_local_experiment_root(&self.root, &experiment_id)?; + let artifact_root = experiment_root.join("artifacts").join(name); + + if !artifact_root.is_dir() { + return Err(ExperimentReaderError::new(format!( + "Local artifact not found: {}", + artifact_root.display() + ))); + } + + let files = collect_bundle_files(&artifact_root, &artifact_root).map_err(|err| { + ExperimentReaderError::with_source("Failed to inspect local artifact files", err) + })?; + let bundle = FsBundle::with_files(artifact_root.clone(), files).map_err(|err| { + ExperimentReaderError::with_source("Failed to create local artifact bundle", err) + })?; + + Ok(LoadedArtifact::new( + ArtifactRef { + id: artifact_root.to_string_lossy().to_string(), + name: name.to_string(), + }, + bundle, + )) + } +} + +fn parse_local_experiment_root( + root: &Path, + experiment_id: &ExperimentId, +) -> Result { + let experiment_dir = experiment_id.as_str(); + if experiment_dir.is_empty() || experiment_dir.contains('/') || experiment_dir.contains('\\') { + return Err(ExperimentReaderError::new( + "Invalid local experiment ID format", + )); + } + + Ok(root.join(experiment_dir)) +} + +fn collect_bundle_files(root: &Path, current: &Path) -> Result, std::io::Error> { + let mut files = Vec::new(); + for entry in fs::read_dir(current)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + files.extend(collect_bundle_files(root, &path)?); + continue; + } + + let rel = path + .strip_prefix(root) + .map_err(std::io::Error::other)? + .to_string_lossy() + .to_string(); + files.push(rel); + } + Ok(files) +} + +#[cfg(test)] +mod tests { + use std::io::Read; + + use tracel_artifact::bundle::{BundleDecode, BundleEncode, BundleSink, BundleSource}; + + use super::*; + use crate::ArtifactKind; + use crate::local::test_support::run; + + /// A one-file artifact. + #[derive(Debug, PartialEq)] + struct Note(String); + + impl BundleEncode for Note { + type Settings = (); + type Error = String; + + fn encode(self, sink: &mut O, _settings: &()) -> Result<(), String> { + sink.put_bytes("note.txt", self.0.as_bytes()) + } + } + + impl BundleDecode for Note { + type Settings = (); + type Error = String; + + fn decode(source: &I, _settings: &()) -> Result { + let mut text = String::new(); + source + .open("note.txt")? + .read_to_string(&mut text) + .map_err(|error| error.to_string())?; + Ok(Self(text)) + } + } + + #[test] + fn a_saved_artifact_loads_back_from_a_later_run() { + let dir = tempfile::tempdir().unwrap(); + let first = run(dir.path(), "mnist"); + first + .save_artifact("note", ArtifactKind::Other, Note("kept".to_string()), &()) + .unwrap(); + let first_id = first.id().clone(); + first.finish().unwrap(); + + let second = run(dir.path(), "mnist"); + let note: Note = second.use_artifact(first_id, "note", &()).unwrap(); + + assert_eq!(note, Note("kept".to_string())); + } + + #[test] + fn an_experiment_id_naming_another_directory_is_refused() { + let root = Path::new("/runs/mnist"); + + for id in ["", "../other", "a/b", "a\\b"] { + assert!( + parse_local_experiment_root(root, &ExperimentId::new(id)).is_err(), + "{id:?}" + ); + } + assert_eq!( + parse_local_experiment_root(root, &ExperimentId::new("3")).unwrap(), + root.join("3") + ); + } +} diff --git a/crates/tracel-experiment/src/local/events.rs b/crates/tracel-experiment/src/local/events.rs new file mode 100644 index 00000000..d9e48e5f --- /dev/null +++ b/crates/tracel-experiment/src/local/events.rs @@ -0,0 +1,330 @@ +use chrono::{SecondsFormat, Utc}; +use serde::Serialize; +use serde_json::{Map, Value}; + +use crate::session::Event; +use crate::{ActivityEvent, ActivitySpec, ActivityStatus, MetricSpec, MetricValue}; + +/// The file a run appends its events to, one JSON object per line. +pub const EVENTS_FILE: &str = "events.jsonl"; + +/// One line of `events.jsonl`, with the fields the console's experiment API gives the event. +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Line { + Attribute { + key: String, + value: Value, + }, + Arguments { + value: Value, + }, + Config { + name: String, + value: Value, + }, + Log { + timestamp: String, + level: &'static str, + message: String, + metadata: Map, + activity: Option, + }, + Metric { + name: String, + epoch: usize, + iteration: usize, + value: f64, + group: String, + activity: Option, + }, + MetricDefinition { + name: String, + description: Option, + unit: Option, + higher_is_better: bool, + }, + EpochSummary { + name: String, + epoch: usize, + group: String, + value: f64, + activity: Option, + }, + Summary { + name: String, + value: f64, + activity: Option, + }, + InputUsed { + experiment: String, + name: String, + artifact_id: String, + }, + ActivityStarted { + activity: ActivitySpec, + }, + ActivityUpdated { + id: u64, + current: u64, + }, + ActivityMessage { + id: u64, + message: String, + }, + ActivityFinished { + id: u64, + status: ActivityStatus, + message: Option, + }, +} + +/// The lines `event` is recorded as: one, or one per value of a metric event. +pub fn lines(event: Event) -> Vec { + match event { + Event::Args(value) => vec![Line::Arguments { value }], + Event::Config { name, value } => vec![Line::Config { name, value }], + Event::Log { record, activity } => vec![Line::Log { + timestamp: Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), + level: record.level.as_str(), + message: record.message, + metadata: record.attributes, + activity: activity.map(|id| id.as_u64()), + }], + Event::Metrics { + epoch, + split, + iteration, + items, + activity, + } => items + .into_iter() + .map(|MetricValue { name, value }| Line::Metric { + name, + epoch, + iteration, + value, + group: split.clone(), + activity: activity.map(|id| id.as_u64()), + }) + .collect(), + Event::MetricDefinition(MetricSpec { + name, + description, + unit, + higher_is_better, + }) => vec![Line::MetricDefinition { + name, + description, + unit, + higher_is_better, + }], + Event::EpochSummary { + epoch, + split, + items, + activity, + } => items + .into_iter() + .map(|MetricValue { name, value }| Line::EpochSummary { + name, + epoch, + group: split.clone(), + value, + activity: activity.map(|id| id.as_u64()), + }) + .collect(), + Event::Summary { items, activity } => items + .into_iter() + .map(|MetricValue { name, value }| Line::Summary { + name, + value, + activity: activity.map(|id| id.as_u64()), + }) + .collect(), + Event::ArtifactUsed { + experiment_id, + reference, + } => vec![Line::InputUsed { + experiment: experiment_id.to_string(), + name: reference.name, + artifact_id: reference.id, + }], + Event::Activity(event) => vec![match event { + ActivityEvent::Started { activity } => Line::ActivityStarted { activity }, + ActivityEvent::Updated { id, current } => Line::ActivityUpdated { + id: id.as_u64(), + current, + }, + ActivityEvent::Message { id, message } => Line::ActivityMessage { + id: id.as_u64(), + message, + }, + ActivityEvent::Finished { + id, + status, + message, + } => Line::ActivityFinished { + id: id.as_u64(), + status, + message, + }, + }], + } +} + +/// `lines` as JSON Lines: each one JSON object followed by a newline. +pub fn to_jsonl(lines: impl IntoIterator) -> serde_json::Result { + let mut jsonl = String::new(); + for line in lines { + jsonl.push_str(&serde_json::to_string(&line)?); + jsonl.push('\n'); + } + Ok(jsonl) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use serde_json::json; + + use crate::local::LocalExperiments; + use crate::local::test_support::read_lines; + use crate::{ExperimentProvider, LogRecord, MetricSpec, MetricValue}; + + #[test] + fn a_run_records_each_event_as_a_line_of_json() { + let dir = tempfile::tempdir().unwrap(); + let experiment = LocalExperiments::new(dir.path()) + .create_experiment( + "mnist".to_string(), + HashMap::from([ + ("tracel.job_num".to_string(), json!(12)), + ("kind".to_string(), json!("example")), + ]), + ) + .unwrap(); + + experiment.log_args(&json!({"epochs": 2})).unwrap(); + experiment + .log_config("optimizer", &json!({"lr": 0.01})) + .unwrap(); + experiment.log_metric_definition(MetricSpec { + name: "loss".to_string(), + description: Some("training loss".to_string()), + unit: None, + higher_is_better: false, + }); + let epoch = experiment.activity("epoch 1").meter(4, "batch").start(); + epoch.log(LogRecord::warn("slow batch").with("batch", 3)); + epoch.log_metric( + 1, + "train", + 4, + vec![ + MetricValue { + name: "loss".to_string(), + value: 0.5, + }, + MetricValue { + name: "accuracy".to_string(), + value: 0.9, + }, + ], + ); + epoch.inc(1); + epoch.finish_with_message("done"); + experiment.log_epoch_summary( + 1, + "train", + vec![MetricValue { + name: "loss".to_string(), + value: 0.4, + }], + ); + experiment.log_summary(vec![MetricValue { + name: "accuracy".to_string(), + value: 0.95, + }]); + experiment.finish().unwrap(); + + let mut lines = read_lines(&dir.path().join("mnist/1/events.jsonl")); + let timestamp = lines[6]["timestamp"].take(); + assert!( + timestamp + .as_str() + .is_some_and(|t| chrono::DateTime::parse_from_rfc3339(t).is_ok()), + "{timestamp}" + ); + assert_eq!( + lines, + [ + json!({"type": "attribute", "key": "kind", "value": "example"}), + json!({"type": "attribute", "key": "tracel.job_num", "value": 12}), + json!({"type": "arguments", "value": {"epochs": 2}}), + json!({"type": "config", "name": "optimizer", "value": {"lr": 0.01}}), + json!({ + "type": "metric_definition", + "name": "loss", + "description": "training loss", + "unit": null, + "higher_is_better": false + }), + json!({ + "type": "activity_started", + "activity": { + "id": 1, + "parent": null, + "name": "epoch 1", + "cancellable": false, + "meter": {"unit": "batch", "total": 4}, + "attributes": {} + } + }), + json!({ + "type": "log", + "timestamp": null, + "level": "warn", + "message": "slow batch", + "metadata": {"batch": 3}, + "activity": 1 + }), + json!({ + "type": "metric", + "name": "loss", + "epoch": 1, + "iteration": 4, + "value": 0.5, + "group": "train", + "activity": 1 + }), + json!({ + "type": "metric", + "name": "accuracy", + "epoch": 1, + "iteration": 4, + "value": 0.9, + "group": "train", + "activity": 1 + }), + json!({"type": "activity_updated", "id": 1, "current": 1}), + json!({ + "type": "activity_finished", + "id": 1, + "status": "Success", + "message": "done" + }), + json!({ + "type": "epoch_summary", + "name": "loss", + "epoch": 1, + "group": "train", + "value": 0.4, + "activity": null + }), + json!({"type": "summary", "name": "accuracy", "value": 0.95, "activity": null}), + ] + ); + } +} diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs new file mode 100644 index 00000000..dcf5c5aa --- /dev/null +++ b/crates/tracel-experiment/src/local/mod.rs @@ -0,0 +1,293 @@ +//! Recording experiments on this machine, without reaching any server. + +mod artifacts; +mod events; +mod runs; +mod status; +#[cfg(test)] +mod test_support; + +use std::collections::HashMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::path::PathBuf; +use std::sync::Mutex; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::thread::{self, JoinHandle}; + +use serde_json::Value; + +use crate::error::{ExperimentError, ExperimentErrorKind}; +use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; +use crate::{ArtifactKind, CancelToken, ExperimentLocation, ExperimentProvider, ExperimentRun}; +use artifacts::{LocalExperimentReader, save_artifact}; +use events::{EVENTS_FILE, Line, lines, to_jsonl}; +use runs::{create_local_run_dir, create_runs_dir}; +use status::{STATUS_FILE, Status}; + +/// Records experiments under a directory on this machine. +/// +/// The runs of the experiment named `name` are numbered from 1 under `/`. Each run +/// keeps its status in `status.json`, written when the run starts and again when it ends, appends +/// its events to `events.jsonl`, and saves its artifacts under `artifacts/`. Creating a +/// `LocalExperiments` performs no I/O; the first run creates the directories. When it creates +/// `dir` itself, it writes a `.gitignore` there that ignores everything under it, so the records +/// stay out of git and of code packages; a `dir` that already exists is left as it is. +/// +/// `status.json` names the experiment and numbers the run, says whether the run is `running`, +/// `completed` or `failed`, when it started and ended, in RFC 3339 UTC to the second, and why it +/// failed: +/// +/// ```json +/// { +/// "name": "mnist", +/// "num": 3, +/// "status": "failed", +/// "started_at": "2026-10-06T14:02:11Z", +/// "finished_at": "2026-10-06T14:31:40Z", +/// "error": "loss is NaN" +/// } +/// ``` +/// +/// Each line of `events.jsonl` is one event, a JSON object whose `type` names it, with the +/// fields the console's experiment API gives it: +/// +/// | `type` | Fields | +/// | --- | --- | +/// | `attribute` | `key`, `value` | +/// | `arguments` | `value`, the run's input, which the console lists as the experiment's `config`; the last one is the run's | +/// | `config` | `name`, `value` | +/// | `log` | `timestamp`, `level`, `message`, `metadata`, `activity` | +/// | `metric` | `name`, `epoch`, `iteration`, `value`, `group`, `activity` | +/// | `metric_definition` | `name`, `description`, `unit`, `higher_is_better` | +/// | `epoch_summary` | `name`, `epoch`, `group`, `value`, `activity` | +/// | `summary` | `name`, `value`, `activity` | +/// | `input_used` | `experiment`, `name`, `artifact_id` | +/// | `activity_started` | `activity`: `id`, `parent`, `name`, `cancellable`, `meter`, `attributes` | +/// | `activity_updated` | `id`, `current` | +/// | `activity_message` | `id`, `message` | +/// | `activity_finished` | `id`, `status`, `message` | +/// +/// `activity` is the ID of the activity an event was recorded in, or `null`. A metric logged with +/// several values is one `metric` line per value. +/// +/// ``` +/// use std::sync::Arc; +/// +/// use tracel_experiment::Experiments; +/// use tracel_experiment::local::LocalExperiments; +/// +/// let experiments = Experiments::new(Arc::new(LocalExperiments::new("./runs"))); +/// ``` +#[derive(Debug, Clone)] +pub struct LocalExperiments { + dir: PathBuf, +} + +impl LocalExperiments { + /// Records experiments under `dir`. + pub fn new(dir: impl Into) -> Self { + Self { dir: dir.into() } + } +} + +impl ExperimentProvider for LocalExperiments { + fn create_experiment( + &self, + name: String, + attributes: HashMap, + ) -> Result { + let internal = |message: &'static str| { + move |error| ExperimentError::with_source(ExperimentErrorKind::Internal, message, error) + }; + create_runs_dir(&self.dir) + .map_err(internal("Failed to create local experiment directory"))?; + let root = self.dir.join(&name); + let root = root + .canonicalize() + .or_else(|_| { + fs::create_dir_all(&root)?; + root.canonicalize() + }) + .map_err(internal("Failed to create local experiment directory"))?; + let (num, run_root) = create_local_run_dir(&root) + .map_err(internal("Failed to create local experiment run directory"))?; + let session = LocalExperimentSession::start(run_root.clone(), name, num, attributes) + .map_err(internal("Failed to initialize local experiment session"))?; + let reader = LocalExperimentReader { root }; + + let run = ExperimentRun::new(num.to_string(), session, reader, CancelToken::new()); + Ok(run.with_location(ExperimentLocation::Dir(run_root))) + } +} + +struct LocalExperimentSession { + root: PathBuf, + status: Status, + active: Mutex>, +} + +impl LocalExperimentSession { + /// Starts recording the run `num` of the experiment `name` in `root`: writes its status as + /// running and its attributes as its first events. + fn start( + root: PathBuf, + name: String, + num: u64, + attributes: HashMap, + ) -> io::Result { + fs::create_dir_all(root.join("artifacts"))?; + let mut attributes: Vec<(String, Value)> = attributes.into_iter().collect(); + attributes.sort_by(|(a, _), (b, _)| a.cmp(b)); + let attributes = attributes + .into_iter() + .map(|(key, value)| Line::Attribute { key, value }); + let mut events = OpenOptions::new() + .create(true) + .append(true) + .open(root.join(EVENTS_FILE))?; + events.write_all(to_jsonl(attributes)?.as_bytes())?; + let status = Status::running(name, num); + status.write(&root.join(STATUS_FILE))?; + let (sender, receiver) = channel(); + let join = thread::spawn(move || local_worker(receiver, events)); + + Ok(Self { + root, + status, + active: Mutex::new(Some(LocalWorker { sender, join })), + }) + } + + fn sender(&self) -> Result, ExperimentError> { + let guard = self.active.lock().unwrap(); + guard + .as_ref() + .map(|worker| worker.sender.clone()) + .ok_or_else(|| { + ExperimentError::new( + ExperimentErrorKind::AlreadyFinished, + "Local experiment session has already finished", + ) + }) + } + + /// Stops the writer once it has written every event, then writes the run's status. + fn finish_worker(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { + let worker = self.active.lock().unwrap().take().ok_or_else(|| { + ExperimentError::new( + ExperimentErrorKind::AlreadyFinished, + "Local experiment session has already finished", + ) + })?; + + let send_result = worker.sender.send(LocalWrite::Finish); + match worker.join.join() { + Ok(Ok(())) if send_result.is_ok() => {} + Ok(Ok(())) => { + return Err(ExperimentError::new( + ExperimentErrorKind::Internal, + "Failed to send local experiment completion", + )); + } + Ok(Err(err)) => { + return Err(ExperimentError::with_source( + ExperimentErrorKind::Internal, + "Local experiment writer failed", + err, + )); + } + Err(_) => { + return Err(ExperimentError::new( + ExperimentErrorKind::Internal, + "Local experiment writer thread panicked", + )); + } + } + + self.status + .ended(completion) + .write(&self.root.join(STATUS_FILE)) + .map_err(|err| { + ExperimentError::with_source( + ExperimentErrorKind::Internal, + "Failed to write the local experiment status", + err, + ) + }) + } +} + +impl ExperimentSession for LocalExperimentSession { + fn record_event(&self, event: Event) -> Result<(), ExperimentError> { + let lines = to_jsonl(lines(event)).map_err(|err| { + ExperimentError::with_source( + ExperimentErrorKind::Internal, + "Failed to serialize local experiment event", + err, + ) + })?; + self.sender()?.send(LocalWrite::Lines(lines)).map_err(|_| { + ExperimentError::new( + ExperimentErrorKind::Internal, + "Failed to queue local experiment event", + ) + }) + } + + fn save_artifact( + &self, + name: &str, + _kind: ArtifactKind, + artifact: Box, + ) -> Result<(), ExperimentError> { + save_artifact(&self.root, name, artifact) + } + + fn finish(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { + self.finish_worker(completion) + } +} + +struct LocalWorker { + sender: Sender, + join: JoinHandle>, +} + +enum LocalWrite { + /// Lines to append to `events.jsonl`, each ending with a newline. + Lines(String), + /// Stop once every line sent before is written. + Finish, +} + +/// Appends each batch of lines to `events` as it arrives, until told to finish. +fn local_worker(receiver: Receiver, mut events: File) -> Result<(), io::Error> { + while let Ok(message) = receiver.recv() { + match message { + LocalWrite::Lines(lines) => events.write_all(lines.as_bytes())?, + LocalWrite::Finish => break, + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_support::run; + + #[test] + fn a_run_is_located_in_its_directory() { + let dir = tempfile::tempdir().unwrap(); + + let experiment = run(dir.path(), "mnist"); + + assert_eq!( + experiment.location(), + Some(&ExperimentLocation::Dir( + dir.path().canonicalize().unwrap().join("mnist/1") + )) + ); + } +} diff --git a/crates/tracel-experiment/src/local/runs.rs b/crates/tracel-experiment/src/local/runs.rs new file mode 100644 index 00000000..f883c606 --- /dev/null +++ b/crates/tracel-experiment/src/local/runs.rs @@ -0,0 +1,106 @@ +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +/// The `.gitignore` written in a runs directory this backend creates: everything in it stays out +/// of git and of code packages. +pub const RUNS_GITIGNORE: &str = + "# Created by tracel: experiment runs recorded on this machine.\n*\n"; + +/// Creates `dir`, the directory runs are recorded under, with a [`RUNS_GITIGNORE`] in it, unless +/// it already exists. +pub fn create_runs_dir(dir: &Path) -> io::Result<()> { + if let Some(parent) = dir.parent().filter(|parent| !parent.as_os_str().is_empty()) { + fs::create_dir_all(parent)?; + } + match fs::create_dir(dir) { + Ok(()) => fs::write(dir.join(".gitignore"), RUNS_GITIGNORE), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => Ok(()), + Err(error) => Err(error), + } +} + +/// Creates the directory of the next run under `root`, numbered after the highest one there, and +/// returns its number and path. +pub fn create_local_run_dir(root: &Path) -> Result<(u64, PathBuf), std::io::Error> { + let mut next_id = 1u64; + + for entry in fs::read_dir(root)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + + if let Some(id) = entry + .file_name() + .to_str() + .and_then(|name| name.parse::().ok()) + { + next_id = next_id.max(id + 1); + } + } + + loop { + let run_root = root.join(next_id.to_string()); + match fs::create_dir(&run_root) { + Ok(()) => return Ok((next_id, run_root)), + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + next_id += 1; + } + Err(err) => return Err(err), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::local::test_support::run; + + #[test] + fn a_runs_directory_it_creates_ignores_everything_in_it() { + let dir = tempfile::tempdir().unwrap(); + let runs = dir.path().join("project/runs"); + + let _run = run(&runs, "mnist"); + + assert_eq!( + fs::read_to_string(runs.join(".gitignore")).unwrap(), + RUNS_GITIGNORE + ); + } + + #[test] + fn a_runs_directory_that_exists_is_left_as_it_is() { + let dir = tempfile::tempdir().unwrap(); + + let _run = run(dir.path(), "mnist"); + + assert!(!dir.path().join(".gitignore").exists()); + } + + #[test] + fn runs_are_numbered_from_one_under_the_experiment_name() { + let dir = tempfile::tempdir().unwrap(); + + let first = run(dir.path(), "mnist"); + let second = run(dir.path(), "mnist"); + let other = run(dir.path(), "other"); + + assert_eq!(first.id().as_str(), "1"); + assert_eq!(second.id().as_str(), "2"); + assert_eq!(other.id().as_str(), "1"); + assert!(dir.path().join("mnist/1").is_dir()); + assert!(dir.path().join("mnist/2").is_dir()); + assert!(dir.path().join("other/1").is_dir()); + } + + #[test] + fn numbering_continues_after_the_highest_existing_run() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir_all(dir.path().join("mnist/7")).unwrap(); + fs::create_dir_all(dir.path().join("mnist/notes")).unwrap(); + + assert_eq!(run(dir.path(), "mnist").id().as_str(), "8"); + } +} diff --git a/crates/tracel-experiment/src/local/status.rs b/crates/tracel-experiment/src/local/status.rs new file mode 100644 index 00000000..a1dc3114 --- /dev/null +++ b/crates/tracel-experiment/src/local/status.rs @@ -0,0 +1,154 @@ +use std::fs; +use std::io; +use std::path::Path; + +use chrono::{SecondsFormat, Utc}; +use serde::Serialize; + +use crate::session::ExperimentCompletion; + +/// The file a run keeps its [`Status`] in. +pub const STATUS_FILE: &str = "status.json"; + +/// How a run is going, as its `status.json` gives it. +#[derive(Clone, Serialize)] +pub struct Status { + name: String, + num: u64, + status: StatusKind, + started_at: String, + finished_at: Option, + error: Option, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +enum StatusKind { + Running, + Completed, + Failed, +} + +impl Status { + /// The status of the run `num` of the experiment `name`, started now. + pub fn running(name: String, num: u64) -> Self { + Self { + name, + num, + status: StatusKind::Running, + started_at: now(), + finished_at: None, + error: None, + } + } + + /// The status of the run once it ended now with `completion`. + pub fn ended(&self, completion: ExperimentCompletion) -> Self { + let (status, error) = match completion { + ExperimentCompletion::Success => (StatusKind::Completed, None), + ExperimentCompletion::Failed(reason) => (StatusKind::Failed, Some(reason)), + }; + Self { + status, + finished_at: Some(now()), + error, + ..self.clone() + } + } + + /// Writes the status to `path` as pretty-printed JSON: to `.tmp` first, then renamed to + /// `path`, so it is never read half written. + pub fn write(&self, path: &Path) -> io::Result<()> { + let mut contents = serde_json::to_vec_pretty(self)?; + contents.push(b'\n'); + let tmp = path.with_extension("json.tmp"); + let written = fs::write(&tmp, contents).and_then(|()| fs::rename(&tmp, path)); + if written.is_err() { + let _ = fs::remove_file(&tmp); + } + written + } +} + +/// The current time, in RFC 3339 UTC to the second. +fn now() -> String { + Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true) +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + + use crate::local::test_support::{read_json, run}; + + fn is_utc_seconds(timestamp: &Value) -> bool { + timestamp.as_str().is_some_and(|timestamp| { + timestamp.ends_with('Z') + && !timestamp.contains('.') + && chrono::DateTime::parse_from_rfc3339(timestamp).is_ok() + }) + } + + #[test] + fn the_status_follows_the_run_from_the_start_to_the_end() { + let dir = tempfile::tempdir().unwrap(); + let experiment = run(dir.path(), "mnist"); + let run_dir = dir.path().join("mnist/1"); + + let mut running = read_json(&run_dir.join("status.json")); + experiment.finish().unwrap(); + let mut completed = read_json(&run_dir.join("status.json")); + + assert!(is_utc_seconds(&running["started_at"]), "{running}"); + assert!(is_utc_seconds(&completed["finished_at"]), "{completed}"); + assert_eq!(completed["started_at"], running["started_at"]); + running["started_at"] = Value::Null; + completed["started_at"] = Value::Null; + completed["finished_at"] = Value::Null; + assert_eq!( + running, + json!({ + "name": "mnist", + "num": 1, + "status": "running", + "started_at": null, + "finished_at": null, + "error": null + }) + ); + assert_eq!( + completed, + json!({ + "name": "mnist", + "num": 1, + "status": "completed", + "started_at": null, + "finished_at": null, + "error": null + }) + ); + assert!(!run_dir.join("status.json.tmp").exists()); + } + + #[test] + fn a_run_says_how_it_ended_in_its_status() { + let dir = tempfile::tempdir().unwrap(); + run(dir.path(), "mnist").fail("diverged").unwrap(); + let cancelled = run(dir.path(), "mnist"); + cancelled.cancel().unwrap(); + cancelled.finish().unwrap(); + drop(run(dir.path(), "mnist")); + + let status = |num: u32| read_json(&dir.path().join(format!("mnist/{num}/status.json"))); + let (failed, cancelled, dropped) = (status(1), status(2), status(3)); + assert_eq!(failed["status"], "failed"); + assert_eq!(failed["error"], "diverged"); + assert_eq!(cancelled["status"], "completed"); + assert_eq!(cancelled["error"], Value::Null); + assert_eq!(dropped["status"], "failed"); + assert_eq!( + dropped["error"], + "the run was dropped without finish() or fail()" + ); + } +} diff --git a/crates/tracel-experiment/src/local/test_support.rs b/crates/tracel-experiment/src/local/test_support.rs new file mode 100644 index 00000000..12882043 --- /dev/null +++ b/crates/tracel-experiment/src/local/test_support.rs @@ -0,0 +1,26 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use serde_json::Value; + +use super::LocalExperiments; +use crate::{ExperimentProvider, ExperimentRun}; + +pub fn run(dir: &Path, name: &str) -> ExperimentRun { + LocalExperiments::new(dir) + .create_experiment(name.to_string(), HashMap::new()) + .unwrap() +} + +pub fn read_lines(path: &Path) -> Vec { + fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +pub fn read_json(path: &Path) -> Value { + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap() +} From f87527877f5c35fe04c9843c62ad79b6bb399ab5 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:32:33 -0400 Subject: [PATCH 29/90] fix: a local artifact name cannot reach outside its run save_artifact joined the name onto the run's artifacts directory unchecked and removed whatever was there, so `..` deleted the run and an absolute name deleted that path, since joining an absolute path replaces the base. The reader had the same gap. Both now read the name as a relative-path RelativePath and normalize it, refusing a name that resolves to the artifacts directory itself or leaves it, as `..` does. The directory is built with to_path, which ignores a leading `/`, so an absolute name stays inside the run. Nested names such as `checkpoints/best` stay allowed, as tracel-artifact's own bundle paths allow them. --- Cargo.lock | 2 + crates/tracel-experiment/Cargo.toml | 2 + .../tracel-experiment/src/local/artifacts.rs | 50 +++++++++++++++++-- crates/tracel-experiment/src/local/mod.rs | 10 ++-- .../src/local/test_support.rs | 16 ++++++ 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ddb294fe..0c2683d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7312,6 +7312,8 @@ version = "0.12.0" dependencies = [ "burn", "chrono", + "relative-path", + "rstest", "serde", "serde_json", "tempfile", diff --git a/crates/tracel-experiment/Cargo.toml b/crates/tracel-experiment/Cargo.toml index 15a21bfc..9b9947cb 100644 --- a/crates/tracel-experiment/Cargo.toml +++ b/crates/tracel-experiment/Cargo.toml @@ -18,6 +18,7 @@ local = [] [dependencies] chrono.workspace = true +relative-path = { workspace = true } thiserror.workspace = true tracel-artifact.workspace = true serde_json.workspace = true @@ -27,5 +28,6 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } burn = { workspace = true, optional = true } [dev-dependencies] +rstest = { workspace = true } tempfile.workspace = true tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/crates/tracel-experiment/src/local/artifacts.rs b/crates/tracel-experiment/src/local/artifacts.rs index 1833e4a6..ec498073 100644 --- a/crates/tracel-experiment/src/local/artifacts.rs +++ b/crates/tracel-experiment/src/local/artifacts.rs @@ -1,6 +1,7 @@ use std::fs; use std::path::{Path, PathBuf}; +use relative_path::RelativePath; use tracel_artifact::bundle::FsBundle; use crate::ExperimentId; @@ -14,7 +15,8 @@ pub fn save_artifact( name: &str, artifact: Box, ) -> Result<(), ExperimentError> { - let artifact_root = root.join("artifacts").join(name); + let artifact_root = artifact_dir(root, name) + .map_err(|message| ExperimentError::new(ExperimentErrorKind::Artifact, message))?; if artifact_root.exists() { fs::remove_dir_all(&artifact_root).map_err(|err| { ExperimentError::with_source( @@ -53,7 +55,8 @@ impl ExperimentArtifactReader for LocalExperimentReader { name: &str, ) -> Result { let experiment_root = parse_local_experiment_root(&self.root, &experiment_id)?; - let artifact_root = experiment_root.join("artifacts").join(name); + let artifact_root = + artifact_dir(&experiment_root, name).map_err(ExperimentReaderError::new)?; if !artifact_root.is_dir() { return Err(ExperimentReaderError::new(format!( @@ -93,6 +96,19 @@ fn parse_local_experiment_root( Ok(root.join(experiment_dir)) } +/// The directory the artifact `name` of the run recorded in `root` is saved in. +/// +/// `name` is a relative path inside the run's `artifacts` directory, such as `checkpoints/best`, +/// a leading `/` included. Fails when it resolves to that directory itself or to one outside it, +/// as `..` does, or when it holds a `\`, which separates paths on Windows only. +fn artifact_dir(root: &Path, name: &str) -> Result { + let path = RelativePath::new(name).normalize(); + if name.contains('\\') || path.as_str().is_empty() || path.starts_with("..") { + return Err(format!("Invalid local artifact name: {name:?}")); + } + Ok(path.to_path(root.join("artifacts"))) +} + fn collect_bundle_files(root: &Path, current: &Path) -> Result, std::io::Error> { let mut files = Vec::new(); for entry in fs::read_dir(current)? { @@ -117,11 +133,12 @@ fn collect_bundle_files(root: &Path, current: &Path) -> Result, std: mod tests { use std::io::Read; + use rstest::rstest; use tracel_artifact::bundle::{BundleDecode, BundleEncode, BundleSink, BundleSource}; use super::*; use crate::ArtifactKind; - use crate::local::test_support::run; + use crate::local::test_support::{LocalRun, local_run, run}; /// A one-file artifact. #[derive(Debug, PartialEq)] @@ -166,6 +183,33 @@ mod tests { assert_eq!(note, Note("kept".to_string())); } + #[rstest] + #[case::empty("")] + #[case::current(".")] + #[case::parent("..")] + #[case::parent_through_a_nested_name("checkpoints/../..")] + #[case::windows_separator("..\\outside")] + fn an_artifact_name_not_inside_the_run_artifacts_is_refused(#[case] name: &str) { + assert!(artifact_dir(Path::new("run"), name).is_err()); + } + + #[rstest] + fn an_absolute_artifact_name_is_saved_inside_its_run(local_run: LocalRun) { + let outside = local_run.runs.path().join("outside"); + + local_run + .run + .save_artifact( + outside.to_str().unwrap(), + ArtifactKind::Other, + Note("saved".to_string()), + &(), + ) + .unwrap(); + + assert!(!outside.exists()); + } + #[test] fn an_experiment_id_naming_another_directory_is_refused() { let root = Path::new("/runs/mnist"); diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs index dcf5c5aa..666eb395 100644 --- a/crates/tracel-experiment/src/local/mod.rs +++ b/crates/tracel-experiment/src/local/mod.rs @@ -29,10 +29,12 @@ use status::{STATUS_FILE, Status}; /// /// The runs of the experiment named `name` are numbered from 1 under `/`. Each run /// keeps its status in `status.json`, written when the run starts and again when it ends, appends -/// its events to `events.jsonl`, and saves its artifacts under `artifacts/`. Creating a -/// `LocalExperiments` performs no I/O; the first run creates the directories. When it creates -/// `dir` itself, it writes a `.gitignore` there that ignores everything under it, so the records -/// stay out of git and of code packages; a `dir` that already exists is left as it is. +/// its events to `events.jsonl`, and saves its artifacts under `artifacts/`, each in the directory +/// its name gives, such as `checkpoints/best`; a name that is not a relative path of plain +/// components, such as `..` or an absolute path, is refused. Creating a `LocalExperiments` +/// performs no I/O; the first run creates the directories. When it creates `dir` itself, it writes +/// a `.gitignore` there that ignores everything under it, so the records stay out of git and of +/// code packages; a `dir` that already exists is left as it is. /// /// `status.json` names the experiment and numbers the run, says whether the run is `running`, /// `completed` or `failed`, when it started and ended, in RFC 3339 UTC to the second, and why it diff --git a/crates/tracel-experiment/src/local/test_support.rs b/crates/tracel-experiment/src/local/test_support.rs index 12882043..511e0d5b 100644 --- a/crates/tracel-experiment/src/local/test_support.rs +++ b/crates/tracel-experiment/src/local/test_support.rs @@ -2,11 +2,27 @@ use std::collections::HashMap; use std::fs; use std::path::Path; +use rstest::fixture; use serde_json::Value; +use tempfile::TempDir; use super::LocalExperiments; use crate::{ExperimentProvider, ExperimentRun}; +pub const EXPERIMENT: &str = "mnist"; + +pub struct LocalRun { + pub run: ExperimentRun, + pub runs: TempDir, +} + +#[fixture] +pub fn local_run() -> LocalRun { + let runs = tempfile::tempdir().unwrap(); + let run = run(runs.path(), EXPERIMENT); + LocalRun { run, runs } +} + pub fn run(dir: &Path, name: &str) -> ExperimentRun { LocalExperiments::new(dir) .create_experiment(name.to_string(), HashMap::new()) From 9445fb5993ec7f8aac005cadaeb4c3f62c5ac590 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:34:03 -0400 Subject: [PATCH 30/90] fix: a local experiment name or id is exactly one directory The reader refused only empty ids and ids holding a slash or a backslash, so `.` and `..` named the experiment's own directory or the runs directory above it. Experiment names were not checked at all, so `..` or an absolute name put runs outside the runs directory and its .gitignore. Both are now read as a relative-path RelativePath that must be one plain component, and the directory is joined from that component, so a leading `/` cannot leave the parent. The store's docs say how both experiment and artifact names are read. The rule is tested on names, then once through each caller: create_experiment creates nothing for a name it refuses, and the reader loads nothing through an id that climbs out and back into a run. --- .../tracel-experiment/src/local/artifacts.rs | 40 ++++++++----------- crates/tracel-experiment/src/local/mod.rs | 39 +++++++++++++----- crates/tracel-experiment/src/local/runs.rs | 36 +++++++++++++++++ 3 files changed, 82 insertions(+), 33 deletions(-) diff --git a/crates/tracel-experiment/src/local/artifacts.rs b/crates/tracel-experiment/src/local/artifacts.rs index ec498073..c21b27a6 100644 --- a/crates/tracel-experiment/src/local/artifacts.rs +++ b/crates/tracel-experiment/src/local/artifacts.rs @@ -4,6 +4,7 @@ use std::path::{Path, PathBuf}; use relative_path::RelativePath; use tracel_artifact::bundle::FsBundle; +use super::runs::child_dir; use crate::ExperimentId; use crate::error::{ExperimentError, ExperimentErrorKind}; use crate::reader::{ArtifactRef, ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact}; @@ -86,14 +87,8 @@ fn parse_local_experiment_root( root: &Path, experiment_id: &ExperimentId, ) -> Result { - let experiment_dir = experiment_id.as_str(); - if experiment_dir.is_empty() || experiment_dir.contains('/') || experiment_dir.contains('\\') { - return Err(ExperimentReaderError::new( - "Invalid local experiment ID format", - )); - } - - Ok(root.join(experiment_dir)) + child_dir(root, experiment_id.as_str()) + .ok_or_else(|| ExperimentReaderError::new("Invalid local experiment ID format")) } /// The directory the artifact `name` of the run recorded in `root` is saved in. @@ -138,7 +133,7 @@ mod tests { use super::*; use crate::ArtifactKind; - use crate::local::test_support::{LocalRun, local_run, run}; + use crate::local::test_support::{EXPERIMENT, LocalRun, local_run, run}; /// A one-file artifact. #[derive(Debug, PartialEq)] @@ -210,19 +205,18 @@ mod tests { assert!(!outside.exists()); } - #[test] - fn an_experiment_id_naming_another_directory_is_refused() { - let root = Path::new("/runs/mnist"); - - for id in ["", "../other", "a/b", "a\\b"] { - assert!( - parse_local_experiment_root(root, &ExperimentId::new(id)).is_err(), - "{id:?}" - ); - } - assert_eq!( - parse_local_experiment_root(root, &ExperimentId::new("3")).unwrap(), - root.join("3") - ); + #[rstest] + fn an_experiment_id_reaching_through_its_parent_is_not_loaded(local_run: LocalRun) { + local_run + .run + .save_artifact("note", ArtifactKind::Other, Note("kept".to_string()), &()) + .unwrap(); + let through_parent = format!("../{EXPERIMENT}/{}", local_run.run.id()); + + let loaded = local_run + .run + .use_artifact::(through_parent, "note", &()); + + assert!(loaded.is_err()); } } diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs index 666eb395..1e62be8a 100644 --- a/crates/tracel-experiment/src/local/mod.rs +++ b/crates/tracel-experiment/src/local/mod.rs @@ -22,19 +22,21 @@ use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; use crate::{ArtifactKind, CancelToken, ExperimentLocation, ExperimentProvider, ExperimentRun}; use artifacts::{LocalExperimentReader, save_artifact}; use events::{EVENTS_FILE, Line, lines, to_jsonl}; -use runs::{create_local_run_dir, create_runs_dir}; +use runs::{child_dir, create_local_run_dir, create_runs_dir}; use status::{STATUS_FILE, Status}; /// Records experiments under a directory on this machine. /// -/// The runs of the experiment named `name` are numbered from 1 under `/`. Each run -/// keeps its status in `status.json`, written when the run starts and again when it ends, appends -/// its events to `events.jsonl`, and saves its artifacts under `artifacts/`, each in the directory -/// its name gives, such as `checkpoints/best`; a name that is not a relative path of plain -/// components, such as `..` or an absolute path, is refused. Creating a `LocalExperiments` -/// performs no I/O; the first run creates the directories. When it creates `dir` itself, it writes -/// a `.gitignore` there that ignores everything under it, so the records stay out of git and of -/// code packages; a `dir` that already exists is left as it is. +/// The runs of the experiment named `name` are numbered from 1 under `/`, so `name`, +/// read as a relative path, is one directory name: one that is empty, `.`, `..` or a path of +/// several is refused. Each run keeps its status in `status.json`, written when the run starts and +/// again when it ends, appends its events to `events.jsonl`, and saves its artifacts under +/// `artifacts/`, each in the directory its name gives there, read as a relative path, such as +/// `checkpoints/best`; a name that resolves to `artifacts/` itself or outside it, such as `..`, is +/// refused. Creating a `LocalExperiments` performs no I/O; the first run creates the directories. +/// When it creates `dir` itself, it writes a `.gitignore` there that ignores everything under it, +/// so the records stay out of git and of code packages; a `dir` that already exists is left as it +/// is. /// /// `status.json` names the experiment and numbers the run, says whether the run is `running`, /// `completed` or `failed`, when it started and ended, in RFC 3339 UTC to the second, and why it @@ -99,12 +101,17 @@ impl ExperimentProvider for LocalExperiments { name: String, attributes: HashMap, ) -> Result { + let root = child_dir(&self.dir, &name).ok_or_else(|| { + ExperimentError::new( + ExperimentErrorKind::Internal, + format!("Invalid local experiment name: {name:?}"), + ) + })?; let internal = |message: &'static str| { move |error| ExperimentError::with_source(ExperimentErrorKind::Internal, message, error) }; create_runs_dir(&self.dir) .map_err(internal("Failed to create local experiment directory"))?; - let root = self.dir.join(&name); let root = root .canonicalize() .or_else(|_| { @@ -292,4 +299,16 @@ mod tests { )) ); } + + #[test] + fn an_experiment_name_that_is_not_one_directory_creates_nothing() { + let dir = tempfile::tempdir().unwrap(); + + LocalExperiments::new(dir.path().join("runs")) + .create_experiment("..".to_string(), HashMap::new()) + .err() + .expect("the name is refused"); + + assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 0); + } } diff --git a/crates/tracel-experiment/src/local/runs.rs b/crates/tracel-experiment/src/local/runs.rs index f883c606..c6dbf971 100644 --- a/crates/tracel-experiment/src/local/runs.rs +++ b/crates/tracel-experiment/src/local/runs.rs @@ -2,6 +2,8 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; +use relative_path::{Component, RelativePath}; + /// The `.gitignore` written in a runs directory this backend creates: everything in it stays out /// of git and of code packages. pub const RUNS_GITIGNORE: &str = @@ -52,10 +54,25 @@ pub fn create_local_run_dir(root: &Path) -> Result<(u64, PathBuf), std::io::Erro } } +/// The directory `name` names directly inside `parent`, when `name`, read as a relative path, is +/// one plain component: `None` when it is empty, `.`, `..`, a path of several, or holds a `\`, +/// which separates paths on Windows only. +pub fn child_dir(parent: &Path, name: &str) -> Option { + if name.contains('\\') { + return None; + } + let mut components = RelativePath::new(name).components(); + match (components.next(), components.next()) { + (Some(Component::Normal(dir)), None) => Some(parent.join(dir)), + _ => None, + } +} + #[cfg(test)] mod tests { use super::*; use crate::local::test_support::run; + use rstest::rstest; #[test] fn a_runs_directory_it_creates_ignores_everything_in_it() { @@ -79,6 +96,25 @@ mod tests { assert!(!dir.path().join(".gitignore").exists()); } + #[rstest] + #[case::plain("mnist")] + #[case::absolute("/mnist")] + fn a_name_of_one_directory_names_that_directory_of_its_parent(#[case] name: &str) { + let parent = Path::new("runs"); + + assert_eq!(child_dir(parent, name), Some(parent.join("mnist"))); + } + + #[rstest] + #[case::empty("")] + #[case::current(".")] + #[case::parent("..")] + #[case::nested("sweep/lr-3")] + #[case::windows_separator("..\\outside")] + fn a_name_that_is_not_exactly_one_directory_is_refused(#[case] name: &str) { + assert_eq!(child_dir(Path::new("runs"), name), None); + } + #[test] fn runs_are_numbered_from_one_under_the_experiment_name() { let dir = tempfile::tempdir().unwrap(); From 13573975393b0f481500bdddbe7c32b2cc7dfc76 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:35:03 -0400 Subject: [PATCH 31/90] fix: flushing a local run writes the events it recorded The local session handed each event to a writer thread over a channel and left flush to the trait's default, which does nothing, so the panic watcher's flush wrote nothing offline: events could still be waiting in the channel. Events are now appended synchronously through a buffered writer behind a mutex, flushed by flush and by finish. Their order and line format are unchanged. The thread, its channel and the match arm for a send that could not fail are gone. --- crates/tracel-experiment/src/local/mod.rs | 156 ++++++++---------- .../src/local/test_support.rs | 18 +- 2 files changed, 80 insertions(+), 94 deletions(-) diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs index 1e62be8a..3a709360 100644 --- a/crates/tracel-experiment/src/local/mod.rs +++ b/crates/tracel-experiment/src/local/mod.rs @@ -9,11 +9,9 @@ mod test_support; use std::collections::HashMap; use std::fs::{self, File, OpenOptions}; -use std::io::{self, Write}; +use std::io::{self, BufWriter, Write}; use std::path::PathBuf; use std::sync::Mutex; -use std::sync::mpsc::{Receiver, Sender, channel}; -use std::thread::{self, JoinHandle}; use serde_json::Value; @@ -133,7 +131,7 @@ impl ExperimentProvider for LocalExperiments { struct LocalExperimentSession { root: PathBuf, status: Status, - active: Mutex>, + events: Mutex>>, } impl LocalExperimentSession { @@ -151,79 +149,30 @@ impl LocalExperimentSession { let attributes = attributes .into_iter() .map(|(key, value)| Line::Attribute { key, value }); - let mut events = OpenOptions::new() + let file = OpenOptions::new() .create(true) .append(true) .open(root.join(EVENTS_FILE))?; + let mut events = BufWriter::new(file); events.write_all(to_jsonl(attributes)?.as_bytes())?; let status = Status::running(name, num); status.write(&root.join(STATUS_FILE))?; - let (sender, receiver) = channel(); - let join = thread::spawn(move || local_worker(receiver, events)); Ok(Self { root, status, - active: Mutex::new(Some(LocalWorker { sender, join })), + events: Mutex::new(Some(events)), }) } - fn sender(&self) -> Result, ExperimentError> { - let guard = self.active.lock().unwrap(); - guard - .as_ref() - .map(|worker| worker.sender.clone()) - .ok_or_else(|| { - ExperimentError::new( - ExperimentErrorKind::AlreadyFinished, - "Local experiment session has already finished", - ) - }) - } - - /// Stops the writer once it has written every event, then writes the run's status. - fn finish_worker(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { - let worker = self.active.lock().unwrap().take().ok_or_else(|| { - ExperimentError::new( - ExperimentErrorKind::AlreadyFinished, - "Local experiment session has already finished", - ) - })?; - - let send_result = worker.sender.send(LocalWrite::Finish); - match worker.join.join() { - Ok(Ok(())) if send_result.is_ok() => {} - Ok(Ok(())) => { - return Err(ExperimentError::new( - ExperimentErrorKind::Internal, - "Failed to send local experiment completion", - )); - } - Ok(Err(err)) => { - return Err(ExperimentError::with_source( - ExperimentErrorKind::Internal, - "Local experiment writer failed", - err, - )); - } - Err(_) => { - return Err(ExperimentError::new( - ExperimentErrorKind::Internal, - "Local experiment writer thread panicked", - )); - } - } - - self.status - .ended(completion) - .write(&self.root.join(STATUS_FILE)) - .map_err(|err| { - ExperimentError::with_source( - ExperimentErrorKind::Internal, - "Failed to write the local experiment status", - err, - ) - }) + /// Runs `write` on the run's events file, unless the run has finished. + fn write_events( + &self, + write: impl FnOnce(&mut BufWriter) -> io::Result<()>, + ) -> Result<(), ExperimentError> { + let mut events = self.events.lock().unwrap(); + let events = events.as_mut().ok_or_else(already_finished)?; + write(events).map_err(writer_failed) } } @@ -236,12 +185,11 @@ impl ExperimentSession for LocalExperimentSession { err, ) })?; - self.sender()?.send(LocalWrite::Lines(lines)).map_err(|_| { - ExperimentError::new( - ExperimentErrorKind::Internal, - "Failed to queue local experiment event", - ) - }) + self.write_events(|events| events.write_all(lines.as_bytes())) + } + + fn flush(&self) -> Result<(), ExperimentError> { + self.write_events(|events| events.flush()) } fn save_artifact( @@ -253,38 +201,52 @@ impl ExperimentSession for LocalExperimentSession { save_artifact(&self.root, name, artifact) } + /// Writes every event recorded, then the run's status. fn finish(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { - self.finish_worker(completion) - } -} + let mut events = self + .events + .lock() + .unwrap() + .take() + .ok_or_else(already_finished)?; + events.flush().map_err(writer_failed)?; -struct LocalWorker { - sender: Sender, - join: JoinHandle>, + self.status + .ended(completion) + .write(&self.root.join(STATUS_FILE)) + .map_err(|err| { + ExperimentError::with_source( + ExperimentErrorKind::Internal, + "Failed to write the local experiment status", + err, + ) + }) + } } -enum LocalWrite { - /// Lines to append to `events.jsonl`, each ending with a newline. - Lines(String), - /// Stop once every line sent before is written. - Finish, +fn already_finished() -> ExperimentError { + ExperimentError::new( + ExperimentErrorKind::AlreadyFinished, + "Local experiment session has already finished", + ) } -/// Appends each batch of lines to `events` as it arrives, until told to finish. -fn local_worker(receiver: Receiver, mut events: File) -> Result<(), io::Error> { - while let Ok(message) = receiver.recv() { - match message { - LocalWrite::Lines(lines) => events.write_all(lines.as_bytes())?, - LocalWrite::Finish => break, - } - } - Ok(()) +fn writer_failed(error: io::Error) -> ExperimentError { + ExperimentError::with_source( + ExperimentErrorKind::Internal, + "Local experiment writer failed", + error, + ) } #[cfg(test)] mod tests { + use rstest::rstest; + + use super::test_support::{LocalRun, local_run}; use super::*; - use test_support::run; + use crate::MetricValue; + use test_support::{read_lines, run}; #[test] fn a_run_is_located_in_its_directory() { @@ -311,4 +273,16 @@ mod tests { assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 0); } + + #[rstest] + fn flush_writes_the_events_recorded_before_it_returns(local_run: LocalRun) { + local_run.run.log_summary(vec![MetricValue { + name: "accuracy".to_string(), + value: 0.95, + }]); + + local_run.run.handle().flush(); + + assert_eq!(read_lines(&local_run.dir.join(EVENTS_FILE)).len(), 1); + } } diff --git a/crates/tracel-experiment/src/local/test_support.rs b/crates/tracel-experiment/src/local/test_support.rs index 511e0d5b..442fa5c4 100644 --- a/crates/tracel-experiment/src/local/test_support.rs +++ b/crates/tracel-experiment/src/local/test_support.rs @@ -1,18 +1,19 @@ use std::collections::HashMap; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use rstest::fixture; use serde_json::Value; use tempfile::TempDir; use super::LocalExperiments; -use crate::{ExperimentProvider, ExperimentRun}; +use crate::{ExperimentLocation, ExperimentProvider, ExperimentRun}; pub const EXPERIMENT: &str = "mnist"; pub struct LocalRun { pub run: ExperimentRun, + pub dir: PathBuf, pub runs: TempDir, } @@ -20,7 +21,11 @@ pub struct LocalRun { pub fn local_run() -> LocalRun { let runs = tempfile::tempdir().unwrap(); let run = run(runs.path(), EXPERIMENT); - LocalRun { run, runs } + LocalRun { + dir: run_dir(&run), + run, + runs, + } } pub fn run(dir: &Path, name: &str) -> ExperimentRun { @@ -29,6 +34,13 @@ pub fn run(dir: &Path, name: &str) -> ExperimentRun { .unwrap() } +pub fn run_dir(run: &ExperimentRun) -> PathBuf { + match run.location() { + Some(ExperimentLocation::Dir(dir)) => dir.clone(), + location => panic!("a local run is located in a directory, not {location:?}"), + } +} + pub fn read_lines(path: &Path) -> Vec { fs::read_to_string(path) .unwrap() From ae00e150e13cc436755cc9b25df36334003fe1bf Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:37:24 -0400 Subject: [PATCH 32/90] fix: a local run whose events fail to write ends as failed Every error arm of finish returned before the status write, so a run whose event writer failed stayed "running" in status.json forever. The first write that fails now ends the recording, and finish always writes the final status, failed with the writer's error when an event could not be written, before returning that error. The session takes its events writer from a seam so a test can hand it a failing one. --- crates/tracel-experiment/src/local/mod.rs | 148 +++++++++++++++---- crates/tracel-experiment/src/local/status.rs | 2 +- 2 files changed, 121 insertions(+), 29 deletions(-) diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs index 3a709360..8b2c6d7d 100644 --- a/crates/tracel-experiment/src/local/mod.rs +++ b/crates/tracel-experiment/src/local/mod.rs @@ -8,8 +8,9 @@ mod status; mod test_support; use std::collections::HashMap; -use std::fs::{self, File, OpenOptions}; +use std::fs::{self, OpenOptions}; use std::io::{self, BufWriter, Write}; +use std::mem; use std::path::PathBuf; use std::sync::Mutex; @@ -131,7 +132,14 @@ impl ExperimentProvider for LocalExperiments { struct LocalExperimentSession { root: PathBuf, status: Status, - events: Mutex>>, + events: Mutex, +} + +/// Where a run's events go: appended to its events file, until a write fails or the run finishes. +enum Events { + Recording(BufWriter>), + Failed(io::Error), + Finished, } impl LocalExperimentSession { @@ -144,16 +152,28 @@ impl LocalExperimentSession { attributes: HashMap, ) -> io::Result { fs::create_dir_all(root.join("artifacts"))?; + let file = OpenOptions::new() + .create(true) + .append(true) + .open(root.join(EVENTS_FILE))?; + Self::start_writing(root, Box::new(file), name, num, attributes) + } + + /// Starts recording the run as [`start`](Self::start) does, with its events written to + /// `events`. + fn start_writing( + root: PathBuf, + events: Box, + name: String, + num: u64, + attributes: HashMap, + ) -> io::Result { let mut attributes: Vec<(String, Value)> = attributes.into_iter().collect(); attributes.sort_by(|(a, _), (b, _)| a.cmp(b)); let attributes = attributes .into_iter() .map(|(key, value)| Line::Attribute { key, value }); - let file = OpenOptions::new() - .create(true) - .append(true) - .open(root.join(EVENTS_FILE))?; - let mut events = BufWriter::new(file); + let mut events = BufWriter::new(events); events.write_all(to_jsonl(attributes)?.as_bytes())?; let status = Status::running(name, num); status.write(&root.join(STATUS_FILE))?; @@ -161,18 +181,27 @@ impl LocalExperimentSession { Ok(Self { root, status, - events: Mutex::new(Some(events)), + events: Mutex::new(Events::Recording(events)), }) } - /// Runs `write` on the run's events file, unless the run has finished. + /// Runs `write` on the run's events file, unless the run has finished or a write has failed: + /// the first write that fails ends the recording. fn write_events( &self, - write: impl FnOnce(&mut BufWriter) -> io::Result<()>, + write: impl FnOnce(&mut BufWriter>) -> io::Result<()>, ) -> Result<(), ExperimentError> { let mut events = self.events.lock().unwrap(); - let events = events.as_mut().ok_or_else(already_finished)?; - write(events).map_err(writer_failed) + let file = match &mut *events { + Events::Recording(file) => file, + Events::Failed(error) => return Err(writer_failed(error)), + Events::Finished => return Err(already_finished()), + }; + write(file).map_err(|error| { + let failure = writer_failed(&error); + *events = Events::Failed(error); + failure + }) } } @@ -201,15 +230,19 @@ impl ExperimentSession for LocalExperimentSession { save_artifact(&self.root, name, artifact) } - /// Writes every event recorded, then the run's status. + /// Writes every event recorded, then the run's status: failed, with the writer's error, when + /// an event could not be written. fn finish(&self, completion: ExperimentCompletion) -> Result<(), ExperimentError> { - let mut events = self - .events - .lock() - .unwrap() - .take() - .ok_or_else(already_finished)?; - events.flush().map_err(writer_failed)?; + let events = mem::replace(&mut *self.events.lock().unwrap(), Events::Finished); + let written = match events { + Events::Recording(mut file) => file.flush().map_err(|error| writer_failed(&error)), + Events::Failed(error) => Err(writer_failed(&error)), + Events::Finished => return Err(already_finished()), + }; + let completion = match &written { + Ok(()) => completion, + Err(error) => ExperimentCompletion::Failed(error.message.clone()), + }; self.status .ended(completion) @@ -220,7 +253,8 @@ impl ExperimentSession for LocalExperimentSession { "Failed to write the local experiment status", err, ) - }) + })?; + written } } @@ -231,22 +265,23 @@ fn already_finished() -> ExperimentError { ) } -fn writer_failed(error: io::Error) -> ExperimentError { - ExperimentError::with_source( +fn writer_failed(error: &io::Error) -> ExperimentError { + ExperimentError::new( ExperimentErrorKind::Internal, - "Local experiment writer failed", - error, + format!("Local experiment writer failed: {error}"), ) } #[cfg(test)] mod tests { - use rstest::rstest; + use rstest::{fixture, rstest}; + use serde_json::json; + use tempfile::TempDir; - use super::test_support::{LocalRun, local_run}; use super::*; use crate::MetricValue; - use test_support::{read_lines, run}; + use status::StatusKind; + use test_support::{EXPERIMENT, LocalRun, local_run, read_json, read_lines, run}; #[test] fn a_run_is_located_in_its_directory() { @@ -285,4 +320,61 @@ mod tests { assert_eq!(read_lines(&local_run.dir.join(EVENTS_FILE)).len(), 1); } + + #[derive(Default)] + struct FailsFirstWrite { + failed: bool, + } + + impl Write for FailsFirstWrite { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.failed { + return Ok(buf.len()); + } + self.failed = true; + Err(io::Error::other("no space left on device")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + struct FailedWrite { + session: LocalExperimentSession, + root: TempDir, + } + + #[fixture] + fn failed_write() -> FailedWrite { + let root = tempfile::tempdir().unwrap(); + let session = LocalExperimentSession::start_writing( + root.path().to_path_buf(), + Box::new(FailsFirstWrite::default()), + EXPERIMENT.to_string(), + 1, + HashMap::new(), + ) + .unwrap(); + session + .record_event(Event::Args(json!({"epochs": 2}))) + .unwrap(); + session.flush().unwrap_err(); + FailedWrite { session, root } + } + + #[rstest] + fn a_run_whose_events_failed_to_write_ends_as_failed(failed_write: FailedWrite) { + let _ = failed_write.session.finish(ExperimentCompletion::Success); + + let status = read_json(&failed_write.root.path().join(STATUS_FILE)); + assert_eq!(status["status"], json!(StatusKind::Failed)); + } + + #[rstest] + fn finishing_a_run_whose_events_failed_to_write_reports_the_failure(failed_write: FailedWrite) { + let finished = failed_write.session.finish(ExperimentCompletion::Success); + + assert_eq!(finished.unwrap_err().kind, ExperimentErrorKind::Internal); + } } diff --git a/crates/tracel-experiment/src/local/status.rs b/crates/tracel-experiment/src/local/status.rs index a1dc3114..9ae8b10a 100644 --- a/crates/tracel-experiment/src/local/status.rs +++ b/crates/tracel-experiment/src/local/status.rs @@ -23,7 +23,7 @@ pub struct Status { #[derive(Clone, Copy, Serialize)] #[serde(rename_all = "snake_case")] -enum StatusKind { +pub enum StatusKind { Running, Completed, Failed, From af8e0108d79405bda29b07c2148434dededbaf51 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:40:15 -0400 Subject: [PATCH 33/90] fix: a local artifact is replaced only once the new one is written Saving an artifact again removed the one saved before, then encoded the new one in its place, so a failed encode or a crash lost the previous checkpoint. The artifact is now encoded into a temporary directory in the run's artifacts directory, removed when the encode fails. Only once it is complete does it take the old one's place: since a directory cannot be renamed over one that is not empty, the old one is moved aside first, put back if the new one cannot take its place, and removed after. --- .../tracel-experiment/src/local/artifacts.rs | 197 ++++++++++++++++-- 1 file changed, 179 insertions(+), 18 deletions(-) diff --git a/crates/tracel-experiment/src/local/artifacts.rs b/crates/tracel-experiment/src/local/artifacts.rs index c21b27a6..ff1d70ba 100644 --- a/crates/tracel-experiment/src/local/artifacts.rs +++ b/crates/tracel-experiment/src/local/artifacts.rs @@ -1,4 +1,5 @@ use std::fs; +use std::io; use std::path::{Path, PathBuf}; use relative_path::RelativePath; @@ -11,6 +12,9 @@ use crate::reader::{ArtifactRef, ExperimentArtifactReader, ExperimentReaderError use crate::session::BundleFn; /// Saves the artifact `name` of the run recorded in `root`, replacing the one saved before. +/// +/// The artifact is written into a temporary directory, which takes the place of the one saved +/// before only once it is complete, so a failed encode leaves that one as it was. pub fn save_artifact( root: &Path, name: &str, @@ -18,17 +22,7 @@ pub fn save_artifact( ) -> Result<(), ExperimentError> { let artifact_root = artifact_dir(root, name) .map_err(|message| ExperimentError::new(ExperimentErrorKind::Artifact, message))?; - if artifact_root.exists() { - fs::remove_dir_all(&artifact_root).map_err(|err| { - ExperimentError::with_source( - ExperimentErrorKind::Artifact, - "Failed to replace existing local artifact", - err, - ) - })?; - } - - let mut bundle = FsBundle::create(artifact_root.clone()).map_err(|err| { + let mut staged = FsBundle::temp_in(root.join("artifacts"), ".saving-").map_err(|err| { ExperimentError::with_source( ExperimentErrorKind::Artifact, "Failed to create local artifact bundle", @@ -36,13 +30,15 @@ pub fn save_artifact( ) })?; - let res = artifact(&mut bundle); + artifact(&mut staged)?; - if res.is_err() { - _ = bundle.delete(); - } - - res + replace_dir(staged.root(), &artifact_root).map_err(|err| { + ExperimentError::with_source( + ExperimentErrorKind::Artifact, + "Failed to replace existing local artifact", + err, + ) + }) } pub struct LocalExperimentReader { @@ -104,6 +100,33 @@ fn artifact_dir(root: &Path, name: &str) -> Result { Ok(path.to_path(root.join("artifacts"))) } +/// Moves the directory `staged` to `target`, in place of the one there. +/// +/// A directory cannot be renamed over one that is not empty, so the one there is moved aside +/// first, put back if `staged` cannot take its place, and removed once it has. +fn replace_dir(staged: &Path, target: &Path) -> io::Result<()> { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + let mut replaced = staged.as_os_str().to_owned(); + replaced.push(".replaced"); + let replacing = match fs::rename(target, &replaced) { + Ok(()) => true, + Err(error) if error.kind() == io::ErrorKind::NotFound => false, + Err(error) => return Err(error), + }; + if let Err(error) = fs::rename(staged, target) { + if replacing { + let _ = fs::rename(&replaced, target); + } + return Err(error); + } + if replacing { + let _ = fs::remove_dir_all(&replaced); + } + Ok(()) +} + fn collect_bundle_files(root: &Path, current: &Path) -> Result, std::io::Error> { let mut files = Vec::new(); for entry in fs::read_dir(current)? { @@ -126,14 +149,15 @@ fn collect_bundle_files(root: &Path, current: &Path) -> Result, std: #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::io::Read; use rstest::rstest; use tracel_artifact::bundle::{BundleDecode, BundleEncode, BundleSink, BundleSource}; use super::*; - use crate::ArtifactKind; use crate::local::test_support::{EXPERIMENT, LocalRun, local_run, run}; + use crate::{ArtifactKind, ExperimentRun}; /// A one-file artifact. #[derive(Debug, PartialEq)] @@ -162,6 +186,71 @@ mod tests { } } + #[derive(Debug, PartialEq)] + struct Files(BTreeMap); + + impl Files { + fn of(files: &[(&str, &str)]) -> Self { + Self( + files + .iter() + .map(|(path, contents)| (path.to_string(), contents.to_string())) + .collect(), + ) + } + } + + impl BundleEncode for Files { + type Settings = (); + type Error = String; + + fn encode(self, sink: &mut O, _settings: &()) -> Result<(), String> { + for (path, contents) in self.0 { + sink.put_bytes(&path, contents.as_bytes())?; + } + Ok(()) + } + } + + impl BundleDecode for Files { + type Settings = (); + type Error = String; + + fn decode(source: &I, _settings: &()) -> Result { + let mut files = BTreeMap::new(); + for path in source.list()? { + let mut contents = String::new(); + source + .open(&path)? + .read_to_string(&mut contents) + .map_err(|error| error.to_string())?; + files.insert(path, contents); + } + Ok(Self(files)) + } + } + + struct HalfWritten; + + impl BundleEncode for HalfWritten { + type Settings = (); + type Error = String; + + fn encode(self, sink: &mut O, _settings: &()) -> Result<(), String> { + sink.put_bytes("note.txt", b"partial")?; + Err("disk full".to_string()) + } + } + + fn entries(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names + } + #[test] fn a_saved_artifact_loads_back_from_a_later_run() { let dir = tempfile::tempdir().unwrap(); @@ -219,4 +308,76 @@ mod tests { assert!(loaded.is_err()); } + + fn save_checkpoint(run: &ExperimentRun, files: &[(&str, &str)]) { + run.save_artifact("checkpoint", ArtifactKind::Model, Files::of(files), &()) + .unwrap(); + } + + #[rstest] + fn saving_an_artifact_again_replaces_every_file_of_the_one_saved_before(local_run: LocalRun) { + save_checkpoint( + &local_run.run, + &[("model.bin", "v1"), ("optimizer.bin", "v1")], + ); + + save_checkpoint(&local_run.run, &[("model.bin", "v2")]); + + assert_eq!( + local_run + .run + .use_artifact::(local_run.run.id().clone(), "checkpoint", &()) + .unwrap(), + Files::of(&[("model.bin", "v2")]) + ); + } + + #[rstest] + fn replacing_an_artifact_leaves_nothing_else_in_the_run_artifacts(local_run: LocalRun) { + save_checkpoint(&local_run.run, &[("model.bin", "v1")]); + + save_checkpoint(&local_run.run, &[("model.bin", "v2")]); + + assert_eq!(entries(&local_run.dir.join("artifacts")), ["checkpoint"]); + } + + #[rstest] + fn an_artifact_that_fails_to_encode_leaves_the_one_saved_before(local_run: LocalRun) { + let kept = || Note("kept".to_string()); + local_run + .run + .save_artifact("note", ArtifactKind::Other, kept(), &()) + .unwrap(); + + local_run + .run + .save_artifact("note", ArtifactKind::Other, HalfWritten, &()) + .unwrap_err(); + + assert_eq!( + local_run + .run + .use_artifact::(local_run.run.id().clone(), "note", &()) + .unwrap(), + kept() + ); + } + + #[test] + fn an_artifact_with_a_nested_name_loads_back_with_its_nested_files() { + let dir = tempfile::tempdir().unwrap(); + let experiment = run(dir.path(), EXPERIMENT); + let files = || Files::of(&[("config.json", "{}"), ("weights/layer1.bin", "w1")]); + + experiment + .save_artifact("checkpoints/best", ArtifactKind::Model, files(), &()) + .unwrap(); + + assert_eq!( + experiment + .use_artifact::(experiment.id().clone(), "checkpoints/best", &()) + .unwrap(), + files() + ); + } } From 009fc4d5e7b0a047349c0172a333889835dcd283 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:42:06 -0400 Subject: [PATCH 34/90] test: pin the runs gitignore and numbering under contention The .gitignore test compared the file with the constant the code wrote, so it would pass were the constant empty: it now checks that the file ignores everything, with a `*` pattern. Numbering gets the two cases it had no test for: a regular file in the way of the next run is numbered past, and runs started at once from several threads, released together by a barrier, are numbered from one without gaps. --- crates/tracel-experiment/src/local/runs.rs | 44 ++++++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/crates/tracel-experiment/src/local/runs.rs b/crates/tracel-experiment/src/local/runs.rs index c6dbf971..f5f9160c 100644 --- a/crates/tracel-experiment/src/local/runs.rs +++ b/crates/tracel-experiment/src/local/runs.rs @@ -70,9 +70,13 @@ pub fn child_dir(parent: &Path, name: &str) -> Option { #[cfg(test)] mod tests { + use std::sync::Barrier; + use std::thread; + + use rstest::rstest; + use super::*; use crate::local::test_support::run; - use rstest::rstest; #[test] fn a_runs_directory_it_creates_ignores_everything_in_it() { @@ -81,9 +85,10 @@ mod tests { let _run = run(&runs, "mnist"); - assert_eq!( - fs::read_to_string(runs.join(".gitignore")).unwrap(), - RUNS_GITIGNORE + let gitignore = fs::read_to_string(runs.join(".gitignore")).unwrap(); + assert!( + gitignore.lines().any(|pattern| pattern == "*"), + "{gitignore}" ); } @@ -139,4 +144,35 @@ mod tests { assert_eq!(run(dir.path(), "mnist").id().as_str(), "8"); } + + #[test] + fn a_file_named_like_the_next_run_is_numbered_past() { + let dir = tempfile::tempdir().unwrap(); + fs::create_dir(dir.path().join("mnist")).unwrap(); + fs::write(dir.path().join("mnist/1"), "").unwrap(); + + assert_eq!(run(dir.path(), "mnist").id().as_str(), "2"); + } + + #[test] + fn runs_started_at_once_are_numbered_from_one_without_gaps() { + const RUNS: u64 = 8; + let dir = tempfile::tempdir().unwrap(); + let barrier = Barrier::new(RUNS as usize); + + let mut ids: Vec = thread::scope(|scope| { + let runs: Vec<_> = (0..RUNS) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + run(dir.path(), "mnist").id().parse().unwrap() + }) + }) + .collect(); + runs.into_iter().map(|run| run.join().unwrap()).collect() + }); + ids.sort(); + + assert_eq!(ids, (1..=RUNS).collect::>()); + } } From 96abe077d904b5d76975d53abb4d945a1f3624ae Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:42:18 -0400 Subject: [PATCH 35/90] test: the local status test checks only what the store decides The test of how a run ended re-asserted decisions lib.rs makes and already pins: a dropped run fails with DROPPED_UNFINISHED, and cancelling a run does not change how it ends. It now checks the store's own mapping of how a run ended, case by case: a success is completed with no error, and a failure is failed with its reason. --- crates/tracel-experiment/src/local/status.rs | 47 +++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/crates/tracel-experiment/src/local/status.rs b/crates/tracel-experiment/src/local/status.rs index 9ae8b10a..1923b9d3 100644 --- a/crates/tracel-experiment/src/local/status.rs +++ b/crates/tracel-experiment/src/local/status.rs @@ -21,7 +21,7 @@ pub struct Status { error: Option, } -#[derive(Clone, Copy, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum StatusKind { Running, @@ -77,9 +77,17 @@ fn now() -> String { #[cfg(test)] mod tests { + use rstest::rstest; use serde_json::{Value, json}; - use crate::local::test_support::{read_json, run}; + use super::*; + use crate::local::test_support::{EXPERIMENT, read_json, run}; + + const REASON: &str = "loss is NaN"; + + fn running() -> Status { + Status::running(EXPERIMENT.to_string(), 1) + } fn is_utc_seconds(timestamp: &Value) -> bool { timestamp.as_str().is_some_and(|timestamp| { @@ -130,25 +138,20 @@ mod tests { assert!(!run_dir.join("status.json.tmp").exists()); } - #[test] - fn a_run_says_how_it_ended_in_its_status() { - let dir = tempfile::tempdir().unwrap(); - run(dir.path(), "mnist").fail("diverged").unwrap(); - let cancelled = run(dir.path(), "mnist"); - cancelled.cancel().unwrap(); - cancelled.finish().unwrap(); - drop(run(dir.path(), "mnist")); - - let status = |num: u32| read_json(&dir.path().join(format!("mnist/{num}/status.json"))); - let (failed, cancelled, dropped) = (status(1), status(2), status(3)); - assert_eq!(failed["status"], "failed"); - assert_eq!(failed["error"], "diverged"); - assert_eq!(cancelled["status"], "completed"); - assert_eq!(cancelled["error"], Value::Null); - assert_eq!(dropped["status"], "failed"); - assert_eq!( - dropped["error"], - "the run was dropped without finish() or fail()" - ); + #[rstest] + #[case::succeeded(ExperimentCompletion::Success, StatusKind::Completed, None)] + #[case::failed( + ExperimentCompletion::Failed(REASON.to_string()), + StatusKind::Failed, + Some(REASON) + )] + fn an_ended_status_says_how_the_run_ended( + #[case] completion: ExperimentCompletion, + #[case] status: StatusKind, + #[case] error: Option<&str>, + ) { + let ended = running().ended(completion); + + assert_eq!((ended.status, ended.error.as_deref()), (status, error)); } } From acdbb7120523bcd4da2aabcf5aeb5b3e301573d1 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:43:11 -0400 Subject: [PATCH 36/90] test: a local artifact never saved does not load The reader's not-found branch had no test. Loading an artifact a run never saved now fails, decoded as a type that accepts an empty bundle, so the test cannot pass on a missing directory read as an empty one. --- crates/tracel-experiment/src/local/artifacts.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/tracel-experiment/src/local/artifacts.rs b/crates/tracel-experiment/src/local/artifacts.rs index ff1d70ba..7ae470d1 100644 --- a/crates/tracel-experiment/src/local/artifacts.rs +++ b/crates/tracel-experiment/src/local/artifacts.rs @@ -363,6 +363,16 @@ mod tests { ); } + #[rstest] + fn an_artifact_never_saved_does_not_load(local_run: LocalRun) { + let loaded = + local_run + .run + .use_artifact::(local_run.run.id().clone(), "missing", &()); + + assert!(loaded.is_err()); + } + #[test] fn an_artifact_with_a_nested_name_loads_back_with_its_nested_files() { let dir = tempfile::tempdir().unwrap(); From b1766e6759a13d91bd5bf1f656cbc5399d4726d7 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:44:16 -0400 Subject: [PATCH 37/90] style: drop the line comment in Experiments::run The comment restated what the method's documentation already says: a panic on any thread while the code runs is logged to the run. --- crates/tracel-experiment/src/provider.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/tracel-experiment/src/provider.rs b/crates/tracel-experiment/src/provider.rs index 767cdd7d..c31bf325 100644 --- a/crates/tracel-experiment/src/provider.rs +++ b/crates/tracel-experiment/src/provider.rs @@ -144,8 +144,6 @@ impl Experiments { E: From + fmt::Display, { let run = self.start(name)?; - // Worker-thread panics (a kernel compiler, a data loader) land in the - // run's log even when they never unwind the run itself. let _panic_watch = run.capture_panics(); match run.in_scope(|| f(&run)) { Ok(output) => { From 32f54820ffadaf9804e3c292e6a59cbdc58c4401 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:44:37 -0400 Subject: [PATCH 38/90] test: each start hook is shown the run it was called with The mock provider gave every run the id 7, so the hook test could not show which run a hook saw. It now numbers its runs from 1, and the test starts a second hooked run after an unhooked one, so the ids show each hook called with each run of its own experiments, in order. The fixture keeps an Arc of the provider instead of sharing the provider's attributes through a second Arc>. --- crates/tracel-experiment/src/provider.rs | 44 ++++++++++++------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/tracel-experiment/src/provider.rs b/crates/tracel-experiment/src/provider.rs index c31bf325..a768362d 100644 --- a/crates/tracel-experiment/src/provider.rs +++ b/crates/tracel-experiment/src/provider.rs @@ -166,16 +166,15 @@ mod tests { use serde_json::json; use super::*; - use crate::ExperimentId; use crate::session::{Event, ExperimentCompletion}; use crate::test_support::{MockSession, create_run_with_id}; type BoxError = Box; - /// Creates runs numbered 7 on one session, and keeps the attributes each was created with. + #[derive(Default)] struct MockProvider { session: Arc, - attributes: Arc>>>, + attributes: Mutex>>, } impl ExperimentProvider for MockProvider { @@ -184,39 +183,37 @@ mod tests { _name: String, attributes: HashMap, ) -> Result { - self.attributes.lock().unwrap().push(attributes); - Ok(create_run_with_id("7", self.session.clone())) + let mut created = self.attributes.lock().unwrap(); + created.push(attributes); + Ok(create_run_with_id( + created.len().to_string(), + self.session.clone(), + )) } } struct Fixture { - session: Arc, - attributes: Arc>>>, + provider: Arc, experiments: Experiments, } impl Fixture { fn new() -> Self { - let session = Arc::new(MockSession::default()); - let attributes = Arc::default(); - let experiments = Experiments::new(Arc::new(MockProvider { - session: session.clone(), - attributes: Arc::clone(&attributes), - })); + let provider = Arc::new(MockProvider::default()); Self { - session, - attributes, - experiments, + experiments: Experiments::new(provider.clone()), + provider, } } fn completions(&self) -> Vec { - self.session.completions.lock().unwrap().clone() + self.provider.session.completions.lock().unwrap().clone() } /// The arguments the runs recorded, in order. fn arguments(&self) -> Vec { - self.session + self.provider + .session .events .lock() .unwrap() @@ -327,7 +324,7 @@ mod tests { experiments.start("train").unwrap().finish().unwrap(); assert_eq!( - *fixture.attributes.lock().unwrap(), + *fixture.provider.attributes.lock().unwrap(), [HashMap::from([ ("kind".to_string(), json!("example")), ("seed".to_string(), json!(42)), @@ -371,19 +368,22 @@ mod tests { .on_start(hook("first")) .on_start(hook("second")); - experiments.start("train").unwrap().finish().unwrap(); + let first = experiments.start("train").unwrap(); fixture .experiments .start("train") .unwrap() .finish() .unwrap(); + let second = experiments.start("train").unwrap(); assert_eq!( *called.lock().unwrap(), [ - ("first", ExperimentId::from("7")), - ("second", ExperimentId::from("7")) + ("first", first.id().clone()), + ("second", first.id().clone()), + ("first", second.id().clone()), + ("second", second.id().clone()) ] ); } From f8b4485e6887c9544529d67baaf5429774b12e20 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:44:59 -0400 Subject: [PATCH 39/90] test: split the started-arguments test into the two rules it pins a_started_experiment_records_the_arguments_logging_replaces read badly and claimed more than it showed: it shows that only experiments given arguments record them when a run starts, and that arguments logged on a run are recorded after those it started with, which is what makes the last ones the run's. Each rule now has its own test. --- crates/tracel-experiment/src/provider.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/tracel-experiment/src/provider.rs b/crates/tracel-experiment/src/provider.rs index a768362d..7915a3ac 100644 --- a/crates/tracel-experiment/src/provider.rs +++ b/crates/tracel-experiment/src/provider.rs @@ -333,13 +333,12 @@ mod tests { } #[test] - fn a_started_experiment_records_the_arguments_logging_replaces() { + fn only_experiments_given_arguments_record_them_when_a_run_starts() { let fixture = Fixture::new(); let experiments = fixture.experiments.clone().arguments(json!({"epochs": 2})); - let run = experiments.start("train").unwrap(); + experiments.start("train").unwrap().finish().unwrap(); let recorded = fixture.arguments(); - run.log_args(&json!({"epochs": 2, "fold": 1})).unwrap(); fixture .experiments .start("train") @@ -348,6 +347,17 @@ mod tests { .unwrap(); assert_eq!(recorded, [json!({"epochs": 2})]); + assert_eq!(fixture.arguments(), recorded); + } + + #[test] + fn arguments_logged_on_a_run_are_recorded_after_those_it_started_with() { + let fixture = Fixture::new(); + let experiments = fixture.experiments.clone().arguments(json!({"epochs": 2})); + let run = experiments.start("train").unwrap(); + + run.log_args(&json!({"epochs": 2, "fold": 1})).unwrap(); + assert_eq!( fixture.arguments(), [json!({"epochs": 2}), json!({"epochs": 2, "fold": 1})] From f80a6b89afeed2ce070836b5f625196dfda14b3d Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:45:16 -0400 Subject: [PATCH 40/90] test: provider tests match the panic reason's end and use a plain flag The panic test accepted any reason containing the message: it now requires the reason to end with it, wherever the panic hook puts the location before it. The test that run runs nothing when the experiment does not start flagged the call with an AtomicBool, where a plain bool does, since run takes an FnOnce with no Send or 'static bound. --- crates/tracel-experiment/src/provider.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/tracel-experiment/src/provider.rs b/crates/tracel-experiment/src/provider.rs index 7915a3ac..cdb8e187 100644 --- a/crates/tracel-experiment/src/provider.rs +++ b/crates/tracel-experiment/src/provider.rs @@ -161,7 +161,6 @@ impl Experiments { #[cfg(test)] mod tests { use std::sync::Mutex; - use std::sync::atomic::{AtomicBool, Ordering}; use serde_json::json; @@ -263,7 +262,7 @@ mod tests { assert!(panicked.is_err()); match fixture.completions().as_slice() { [ExperimentCompletion::Failed(reason)] => { - assert!(reason.contains("loss is NaN"), "{reason}"); + assert!(reason.ends_with("loss is NaN"), "{reason}"); } other => panic!("unexpected completions: {other:?}"), } @@ -298,17 +297,17 @@ mod tests { )) } } - let ran = AtomicBool::new(false); + let mut ran = false; let error = Experiments::new(Arc::new(Unreachable)) .run("train", |_run| -> Result<(), BoxError> { - ran.store(true, Ordering::SeqCst); + ran = true; Ok(()) }) .unwrap_err(); assert_eq!(error.to_string(), "the backend is unreachable"); - assert!(!ran.load(Ordering::SeqCst)); + assert!(!ran); } #[test] From 49b421943506be0096e525ad5ef1415b8c046aca Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:45:30 -0400 Subject: [PATCH 41/90] test: a cancelled run keeps logging after its stop warning cancel_does_not_prevent_logging_before_drop checked only the last event. It now checks every message the run logged, the stop warning then the one logged after it, and is named for that. It moves next to the other stop request tests. --- crates/tracel-experiment/src/lib.rs | 33 ++++++++--------- crates/tracel-experiment/src/test_support.rs | 37 +++++++++++++++++++- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/crates/tracel-experiment/src/lib.rs b/crates/tracel-experiment/src/lib.rs index 608a55d7..e8d068ab 100644 --- a/crates/tracel-experiment/src/lib.rs +++ b/crates/tracel-experiment/src/lib.rs @@ -878,8 +878,12 @@ impl Drop for ExperimentRun { #[cfg(test)] mod tests { + use rstest::rstest; + use crate::activity::ActivityEvent; - use crate::test_support::{MockSession, create_run, create_run_with_control}; + use crate::test_support::{ + MockRun, MockSession, create_run, create_run_with_control, mock_run, + }; use super::*; @@ -1227,21 +1231,6 @@ mod tests { ); } - #[test] - fn cancel_does_not_prevent_logging_before_drop() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - - run.cancel().unwrap(); - run.log_info("still-logging"); - - let events = session.events.lock().unwrap(); - match events.last() { - Some(Event::Log { record, .. }) => assert_eq!(record.message, "still-logging"), - event => panic!("unexpected event: {event:?}"), - } - } - /// The level, message, attributes and activity of each log `session` recorded. fn logs(session: &MockSession) -> Vec<(LogLevel, String, usize, Option)> { session @@ -1261,6 +1250,18 @@ mod tests { .collect() } + #[rstest] + fn a_cancelled_run_keeps_logging_after_the_stop_warning(mock_run: MockRun) { + mock_run.run.cancel().unwrap(); + + mock_run.run.log_info("still logging"); + + assert_eq!( + mock_run.session.log_messages(), + [STOP_REQUESTED, "still logging"] + ); + } + #[test] fn a_stop_request_is_logged_once_while_the_run_is_active() { let session = Arc::new(MockSession::default()); diff --git a/crates/tracel-experiment/src/test_support.rs b/crates/tracel-experiment/src/test_support.rs index e12628f9..8a85a25f 100644 --- a/crates/tracel-experiment/src/test_support.rs +++ b/crates/tracel-experiment/src/test_support.rs @@ -1,10 +1,12 @@ use std::sync::{Arc, Mutex}; +use rstest::fixture; + use crate::activity::ActivityEvent; use crate::error::ExperimentError; use crate::reader::{ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact}; use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; -use crate::{ArtifactKind, ExperimentId, ExperimentRun, ExperimentRunControl}; +use crate::{ArtifactKind, ExperimentId, ExperimentRun, ExperimentRunControl, LogRecord}; #[derive(Default)] pub struct MockSession { @@ -25,6 +27,25 @@ impl MockSession { }) .collect() } + + pub fn logs(&self) -> Vec { + self.events + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + Event::Log { record, .. } => Some(record.clone()), + _ => None, + }) + .collect() + } + + pub fn log_messages(&self) -> Vec { + self.logs() + .into_iter() + .map(|record| record.message) + .collect() + } } impl ExperimentSession for MockSession { @@ -90,3 +111,17 @@ pub fn create_run_with_control( control, ) } + +pub struct MockRun { + pub run: ExperimentRun, + pub session: Arc, +} + +#[fixture] +pub fn mock_run() -> MockRun { + let session = Arc::new(MockSession::default()); + MockRun { + run: create_run(session.clone()), + session, + } +} From f54c2b46b43cd373a4df7a04532b06b1cad365e1 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:36:17 -0400 Subject: [PATCH 42/90] fix: TRACEL_ENV takes only digits as a staging number u8::from_str accepts a leading +, so staging-+2 and Staging(+2) named the second staging console. The number after staging- or inside Staging() now has to be ASCII digits, as the accepted spellings say. --- crates/tracel-console/src/env.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/tracel-console/src/env.rs b/crates/tracel-console/src/env.rs index fdf454f2..86e31b40 100644 --- a/crates/tracel-console/src/env.rs +++ b/crates/tracel-console/src/env.rs @@ -42,6 +42,7 @@ fn parse_env(value: Option<&str>) -> Result { .strip_prefix("Staging(") .and_then(|number| number.strip_suffix(')')) .or_else(|| value.strip_prefix("staging-")) + .filter(|number| number.bytes().all(|byte| byte.is_ascii_digit())) .and_then(|number| number.parse::().ok()) .map(Env::Staging) .ok_or_else(|| ConsoleError::InvalidSetting { @@ -299,8 +300,10 @@ mod tests { "Staging()", "Staging(1", "Staging(256)", + "Staging(+2)", "staging-", "staging--1", + "staging-+2", "staging-256", ] { let error = parse_env(Some(value)).unwrap_err(); From b28e64ecee529a2bbf072f281b4fc52f01900f2a Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:36:50 -0400 Subject: [PATCH 43/90] test: target tests assert only what can fail A console target's services and an offline target's experiments and inference are built without fallible work, so asserting is_ok() on them could never fail. The console test goes; the offline test keeps only its check that building them creates no directory. --- crates/tracel/src/target.rs | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/crates/tracel/src/target.rs b/crates/tracel/src/target.rs index 6194a9e6..77a17766 100644 --- a/crates/tracel/src/target.rs +++ b/crates/tracel/src/target.rs @@ -332,8 +332,6 @@ impl From for TargetError { mod tests { use std::path::Path; - use tracel_console::TracelCredentials; - use super::*; fn target(vars: &[(&str, &str)]) -> Result { @@ -452,13 +450,13 @@ mod tests { } #[test] - fn an_offline_target_serves_experiments_and_inference_without_touching_the_disk() { + fn building_offline_experiments_and_inference_creates_no_directory() { let root = tempfile::tempdir().unwrap(); let dir = root.path().join("runs"); let offline = Target::Offline { dir: dir.clone() }; - assert!(offline.experiments().is_ok()); - assert!(offline.inference().is_ok()); + let _ = (offline.experiments(), offline.inference()); + assert!(!dir.exists()); } @@ -484,22 +482,6 @@ mod tests { )); } - #[test] - fn a_console_target_builds_its_services_without_a_request() { - let console = Target::Console { - env: Env::Development, - credentials: CredentialSource::Explicit(TracelCredentials::api_key( - "tcl_key_not_checked_until_the_first_request", - )), - project: ProjectRef::new("owner", "project"), - }; - - assert!(console.experiments().is_ok()); - assert!(console.models().is_ok()); - assert!(console.datasets().is_ok()); - assert!(console.inference().is_ok()); - } - #[test] fn a_missing_console_setting_names_the_variable_to_set() { for (error, variable, fallback) in [ From 91244f61bebedd51afd8c443485dfb091684f9fb Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:41:09 -0400 Subject: [PATCH 44/90] feat: console settings can be read without the process environment env_from_environment, CredentialSource::from_env, ProjectRef::from_env and ProjectHandle::from_env read the process environment, tracel.toml in the current directory and the tracel login session in the home directory, so nothing built on them could be tested. Each now delegates to a from_vars variant (and resolve to resolve_with) that takes the variables lookup, the directory holding tracel.toml and the session store for the console's URL. FileSessionStore and SessionStoreError are re-exported for that store. The project tests go through ProjectRef::from_vars with a tracel.toml in a temporary directory. The test that only checked a clone of an explicit credential gives way to tests of which credential TRACEL_API_KEY names, of the stored login being read from the store for the console's URL, and of an explicit credential not opening the store. --- Cargo.lock | 2 + crates/tracel-console/Cargo.toml | 4 + crates/tracel-console/src/console.rs | 31 ++++- crates/tracel-console/src/env.rs | 190 ++++++++++++++++++++++----- crates/tracel-console/src/lib.rs | 8 +- 5 files changed, 192 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c2683d5..ba7f5acc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7280,9 +7280,11 @@ dependencies = [ "chrono", "crossbeam", "reqwest", + "rstest", "serde", "serde_json", "serde_with", + "tempfile", "thiserror 2.0.21", "toml", "tracel-artifact", diff --git a/crates/tracel-console/Cargo.toml b/crates/tracel-console/Cargo.toml index aa7ce2c0..a05825d2 100644 --- a/crates/tracel-console/Cargo.toml +++ b/crates/tracel-console/Cargo.toml @@ -29,3 +29,7 @@ tracel-experiment-remote = { workspace = true } tracel-inference = { workspace = true } tracel-models = { workspace = true } url = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/tracel-console/src/console.rs b/crates/tracel-console/src/console.rs index 91b82508..6fbfdd01 100644 --- a/crates/tracel-console/src/console.rs +++ b/crates/tracel-console/src/console.rs @@ -1,11 +1,16 @@ +use std::ffi::OsString; use std::fmt; +use std::path::Path; use std::sync::{Arc, Mutex, OnceLock, PoisonError}; use std::time::{Duration, SystemTime}; use tracel_artifact::ReqwestTransferClient; use tracel_client::{ ClientError, - console::{Client, Env, TracelCredentials, user::response::UserResponseSchema}, + console::{ + Client, Env, FileSessionStore, SessionStoreError, TracelCredentials, + user::response::UserResponseSchema, + }, }; use tracel_datasets::DatasetRegistry; use tracel_experiment::Experiments; @@ -14,7 +19,7 @@ use tracel_models::ModelRegistry; use url::Url; use crate::datasets::ConsoleDatasetOps; -use crate::env::{CredentialSource, ProjectRef, env_from_environment}; +use crate::env::{CredentialSource, ProjectRef, env_from_environment, env_from_vars}; use crate::experiment::ConsoleExperimentProvider; use crate::inference::ConsoleInferenceProvider; use crate::models::ConsoleModelOps; @@ -303,9 +308,25 @@ impl ProjectHandle { /// directory. Fails with [`ConsoleError::NoNamespace`] or [`ConsoleError::NoProject`] when /// neither names it. pub fn from_env() -> Result { - let console = Console::from_env()?; - let project = ProjectRef::from_env()?; - Ok(console.project(project.namespace, project.name)) + Self::from_vars( + |name| std::env::var_os(name), + Path::new("."), + FileSessionStore::for_server, + ) + } + + /// [`from_env`](Self::from_env), reading the variables from `lookup`, `tracel.toml` from + /// `dir`, and the stored `tracel login` session from the store `sessions` gives for the + /// console's URL. + pub fn from_vars( + lookup: impl Fn(&str) -> Option, + dir: &Path, + sessions: impl FnOnce(&Url) -> Result, + ) -> Result { + let env = env_from_vars(&lookup)?; + let credentials = CredentialSource::from_vars(&lookup).resolve_with(&env, sessions)?; + let project = ProjectRef::from_vars(&lookup, dir)?; + Ok(Console::connect(env, credentials).project(project.namespace, project.name)) } /// Returns the console this project is reached through. diff --git a/crates/tracel-console/src/env.rs b/crates/tracel-console/src/env.rs index 86e31b40..9def970b 100644 --- a/crates/tracel-console/src/env.rs +++ b/crates/tracel-console/src/env.rs @@ -1,17 +1,22 @@ //! Reading which console to reach, with which credential, and which project, from the //! environment and `tracel.toml`. +use std::ffi::OsString; use std::path::Path; use std::sync::Arc; use serde::Deserialize; use tracel_client::ClientError; use tracel_client::console::auth::DeviceAuthClient; -use tracel_client::console::{AppSession, Env, FileSessionStore, SessionStore, TracelCredentials}; +use tracel_client::console::{ + AppSession, Env, FileSessionStore, SessionStore, SessionStoreError, TracelCredentials, +}; +use url::Url; use crate::ConsoleError; const TRACEL_ENV: &str = "TRACEL_ENV"; +const TRACEL_API_KEY: &str = "TRACEL_API_KEY"; const TRACEL_NAMESPACE: &str = "TRACEL_NAMESPACE"; const TRACEL_PROJECT: &str = "TRACEL_PROJECT"; const TRACEL_TOML: &str = "tracel.toml"; @@ -29,7 +34,12 @@ const TRACEL_ENV_VALUES: &str = /// `staging-n` spellings of the `tracel` CLI. Fails with [`ConsoleError::InvalidSetting`] for any /// other value. pub fn env_from_environment() -> Result { - let value = std::env::var_os(TRACEL_ENV).map(|value| value.to_string_lossy().into_owned()); + env_from_vars(|name| std::env::var_os(name)) +} + +/// [`env_from_environment`], reading `TRACEL_ENV` from `lookup`. +pub fn env_from_vars(lookup: impl Fn(&str) -> Option) -> Result { + let value = lookup(TRACEL_ENV).map(|value| value.to_string_lossy().into_owned()); parse_env(value.as_deref()) } @@ -66,7 +76,14 @@ pub enum CredentialSource { impl CredentialSource { /// `TRACEL_API_KEY` when it is set and not empty, else the stored `tracel login` session. pub fn from_env() -> Self { - TracelCredentials::from_env().map_or(Self::StoredLogin, Self::Explicit) + Self::from_vars(|name| std::env::var_os(name)) + } + + /// [`from_env`](Self::from_env), reading `TRACEL_API_KEY` from `lookup`. + pub fn from_vars(lookup: impl Fn(&str) -> Option) -> Self { + non_empty(lookup(TRACEL_API_KEY)).map_or(Self::StoredLogin, |key| { + Self::Explicit(TracelCredentials::api_key(key)) + }) } /// The credential this source names for `env`'s console. @@ -74,16 +91,28 @@ impl CredentialSource { /// Reads a stored session from disk, without network I/O. Fails with /// [`ConsoleError::NoCredentials`] when `tracel login` stored none for that console. pub fn resolve(&self, env: &Env) -> Result { + self.resolve_with(env, FileSessionStore::for_server) + } + + /// [`resolve`](Self::resolve), reading a stored session from the store `sessions` gives for + /// the console's URL. Only [`StoredLogin`](Self::StoredLogin) asks `sessions` for it. + pub fn resolve_with( + &self, + env: &Env, + sessions: impl FnOnce(&Url) -> Result, + ) -> Result { match self { Self::Explicit(credentials) => Ok(credentials.clone()), - Self::StoredLogin => stored_login(env), + Self::StoredLogin => { + let store = sessions(&env.get_url()).map_err(ClientError::from)?; + stored_login(env, store) + } } } } -/// The app session `tracel login` stored for `env`'s console. -fn stored_login(env: &Env) -> Result { - let store = FileSessionStore::for_server(&env.get_url()).map_err(ClientError::from)?; +/// The app session `tracel login` stored in `store` for `env`'s console. +fn stored_login(env: &Env, store: FileSessionStore) -> Result { if store.load().map_err(ClientError::from)?.is_none() { return Err(ConsoleError::NoCredentials); } @@ -120,34 +149,37 @@ impl ProjectRef { /// directory. Fails with [`ConsoleError::NoNamespace`] or [`ConsoleError::NoProject`] when /// neither names it. pub fn from_env() -> Result { - let namespace = non_empty_var(TRACEL_NAMESPACE); - let project = non_empty_var(TRACEL_PROJECT); + Self::from_vars(|name| std::env::var_os(name), Path::new(".")) + } + + /// [`from_env`](Self::from_env), reading the variables from `lookup` and `tracel.toml` from + /// `dir`. + pub fn from_vars( + lookup: impl Fn(&str) -> Option, + dir: &Path, + ) -> Result { + let namespace = non_empty(lookup(TRACEL_NAMESPACE)); + let project = non_empty(lookup(TRACEL_PROJECT)); let file = if namespace.is_some() && project.is_some() { TracelToml::default() } else { - TracelToml::read(Path::new(TRACEL_TOML)) + TracelToml::read(&dir.join(TRACEL_TOML)) }; - locate_project(namespace, project, file) - } -} + let namespace = namespace + .or(file.namespace) + .ok_or(ConsoleError::NoNamespace)?; + let name = project.or(file.project).ok_or(ConsoleError::NoProject)?; -fn locate_project( - namespace: Option, - project: Option, - file: TracelToml, -) -> Result { - let namespace = namespace - .or(file.namespace) - .ok_or(ConsoleError::NoNamespace)?; - let name = project.or(file.project).ok_or(ConsoleError::NoProject)?; - - Ok(ProjectRef { namespace, name }) + Ok(Self { namespace, name }) + } } -/// A variable's value; an empty one reads as unset. -fn non_empty_var(name: &str) -> Option { - std::env::var(name).ok().filter(|value| !value.is_empty()) +/// A variable's value; an empty or non-UTF-8 one reads as unset. +fn non_empty(value: Option) -> Option { + value + .and_then(|value| value.into_string().ok()) + .filter(|value| !value.is_empty()) } /// The project a `tracel.toml` names with its `namespace` and `project` keys. @@ -172,18 +204,29 @@ impl TracelToml { #[cfg(test)] mod tests { + use std::time::SystemTime; + + use rstest::{fixture, rstest}; + use tempfile::TempDir; + use tracel_client::console::{AccessToken, RefreshToken, StoredAppSession}; + use super::*; + const API_KEY: &str = "tcl_key"; + fn located( namespace: Option<&str>, project: Option<&str>, tracel_toml: &str, ) -> Result { - locate_project( - namespace.map(str::to_string), - project.map(str::to_string), - TracelToml::parse(tracel_toml), - ) + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("tracel.toml"), tracel_toml).unwrap(); + let lookup = |name: &str| match name { + TRACEL_NAMESPACE => namespace.map(OsString::from), + TRACEL_PROJECT => project.map(OsString::from), + _ => None, + }; + ProjectRef::from_vars(lookup, dir.path()) } #[test] @@ -252,13 +295,88 @@ mod tests { assert!(no_project.to_string().contains("tracel.toml")); } + fn api_key(value: Option<&str>) -> CredentialSource { + CredentialSource::from_vars(|name| match name { + TRACEL_API_KEY => value.map(OsString::from), + _ => None, + }) + } + + #[fixture] + fn sessions() -> TempDir { + tempfile::tempdir().unwrap() + } + + fn store_in(sessions: &TempDir) -> FileSessionStore { + FileSessionStore::new(sessions.path().join("session.json")) + } + + fn no_store(_: &Url) -> Result { + Err(SessionStoreError::new("no home directory")) + } + + #[test] + fn tracel_api_key_is_the_credential() { + assert!(matches!( + api_key(Some(API_KEY)), + CredentialSource::Explicit(TracelCredentials::ApiKey(key)) if key == API_KEY + )); + } + + #[rstest] + #[case::unset(None)] + #[case::empty(Some(""))] + fn without_tracel_api_key_the_credential_is_the_stored_login(#[case] value: Option<&str>) { + assert!(matches!(api_key(value), CredentialSource::StoredLogin)); + } + + #[test] + fn a_stored_login_is_read_from_the_store_for_its_console() { + let env = Env::Staging(2); + let mut asked_for = None; + + let _ = CredentialSource::StoredLogin.resolve_with(&env, |url| { + asked_for = Some(url.clone()); + no_store(url) + }); + + assert_eq!(asked_for, Some(env.get_url())); + } + + #[rstest] + fn a_store_without_a_session_gives_no_credential(sessions: TempDir) { + assert!(matches!( + CredentialSource::StoredLogin + .resolve_with(&Env::Production, |_| Ok(store_in(&sessions))), + Err(ConsoleError::NoCredentials) + )); + } + + #[rstest] + fn a_stored_session_is_the_credential(sessions: TempDir) { + store_in(&sessions) + .save(&StoredAppSession { + access_token: AccessToken::new("tcl_at_stored"), + access_token_expires_at: SystemTime::UNIX_EPOCH, + refresh_token: RefreshToken::new("tcl_rt_stored"), + refresh_token_expires_at: SystemTime::UNIX_EPOCH, + }) + .unwrap(); + + assert!(matches!( + CredentialSource::StoredLogin + .resolve_with(&Env::Production, |_| Ok(store_in(&sessions))), + Ok(TracelCredentials::AppSession(_)) + )); + } + #[test] - fn an_explicit_credential_is_used_as_given() { - let source = CredentialSource::Explicit(TracelCredentials::api_key("tcl_key")); + fn an_explicit_credential_needs_no_session_store() { + let source = CredentialSource::Explicit(TracelCredentials::api_key(API_KEY)); assert!(matches!( - source.resolve(&Env::Development), - Ok(TracelCredentials::ApiKey(key)) if key == "tcl_key" + source.resolve_with(&Env::Production, no_store), + Ok(TracelCredentials::ApiKey(key)) if key == API_KEY )); } diff --git a/crates/tracel-console/src/lib.rs b/crates/tracel-console/src/lib.rs index 1771ee40..be24402e 100644 --- a/crates/tracel-console/src/lib.rs +++ b/crates/tracel-console/src/lib.rs @@ -10,6 +10,8 @@ //! project from `TRACEL_ENV`, `TRACEL_API_KEY`, `TRACEL_NAMESPACE` and `TRACEL_PROJECT`, falling //! back to the `tracel login` sign-in and to `tracel.toml`. [`env_from_environment`], //! [`CredentialSource::from_env`] and [`ProjectRef::from_env`] read each of the three on its own. +//! Their `from_vars` variants, and [`ProjectHandle::from_vars`], read the variables from a lookup, +//! `tracel.toml` from a directory and the sign-in from a session store the caller gives. mod console; mod datasets; @@ -24,8 +26,10 @@ mod wire; pub use console::{Console, ProjectHandle}; pub use domain::{Namespace, NamespaceKind, Organization, Project, User, Visibility}; -pub use env::{CredentialSource, ProjectRef, env_from_environment}; +pub use env::{CredentialSource, ProjectRef, env_from_environment, env_from_vars}; pub use error::ConsoleError; pub use login::{DeviceApproval, DeviceLogin, refresh_session, sign_out}; pub use tracel_client::console::auth::IssuedAppSession; -pub use tracel_client::console::{AccessToken, Env, RefreshToken, TracelCredentials}; +pub use tracel_client::console::{ + AccessToken, Env, FileSessionStore, RefreshToken, SessionStoreError, TracelCredentials, +}; From fa1f0cca6b305751601ecb7c2460608fe82bb5a5 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:41:51 -0400 Subject: [PATCH 45/90] fix: an unreadable or malformed tracel.toml is an error naming it A tracel.toml that could not be read or parsed was taken as empty, so the error the user then saw said tracel.toml set no namespace even when the file visibly set one. A missing file still names nothing; a read or parse error is now ConsoleError::InvalidTracelToml, which names the file and the reason. --- crates/tracel-console/src/console.rs | 3 +- crates/tracel-console/src/env.rs | 57 +++++++++++++++++++++------- crates/tracel-console/src/error.rs | 10 +++++ 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/crates/tracel-console/src/console.rs b/crates/tracel-console/src/console.rs index 6fbfdd01..59cea5ee 100644 --- a/crates/tracel-console/src/console.rs +++ b/crates/tracel-console/src/console.rs @@ -306,7 +306,8 @@ impl ProjectHandle { /// The owner namespace is `TRACEL_NAMESPACE` and the project name `TRACEL_PROJECT`; either /// one unset is read from `namespace` or `project` in `tracel.toml` in the current /// directory. Fails with [`ConsoleError::NoNamespace`] or [`ConsoleError::NoProject`] when - /// neither names it. + /// neither names it, and with [`ConsoleError::InvalidTracelToml`] when `tracel.toml` + /// cannot be read. pub fn from_env() -> Result { Self::from_vars( |name| std::env::var_os(name), diff --git a/crates/tracel-console/src/env.rs b/crates/tracel-console/src/env.rs index 9def970b..238c90a4 100644 --- a/crates/tracel-console/src/env.rs +++ b/crates/tracel-console/src/env.rs @@ -2,6 +2,7 @@ //! environment and `tracel.toml`. use std::ffi::OsString; +use std::io::ErrorKind; use std::path::Path; use std::sync::Arc; @@ -147,7 +148,8 @@ impl ProjectRef { /// The owner namespace is `TRACEL_NAMESPACE` and the project name `TRACEL_PROJECT`; either /// one unset is read from `namespace` or `project` in `tracel.toml` in the current /// directory. Fails with [`ConsoleError::NoNamespace`] or [`ConsoleError::NoProject`] when - /// neither names it. + /// neither names it, and with [`ConsoleError::InvalidTracelToml`] when `tracel.toml` + /// cannot be read. pub fn from_env() -> Result { Self::from_vars(|name| std::env::var_os(name), Path::new(".")) } @@ -163,7 +165,7 @@ impl ProjectRef { let file = if namespace.is_some() && project.is_some() { TracelToml::default() } else { - TracelToml::read(&dir.join(TRACEL_TOML)) + TracelToml::read(&dir.join(TRACEL_TOML))? }; let namespace = namespace @@ -190,15 +192,20 @@ struct TracelToml { } impl TracelToml { - /// A missing, unreadable or malformed file names nothing. - fn read(path: &Path) -> Self { - std::fs::read_to_string(path) - .map(|contents| Self::parse(&contents)) - .unwrap_or_default() - } - - fn parse(contents: &str) -> Self { - toml::from_str(contents).unwrap_or_default() + /// The project the file at `path` names; a missing file names nothing. + /// + /// Fails with [`ConsoleError::InvalidTracelToml`] when the file cannot be read or is not + /// valid TOML. + fn read(path: &Path) -> Result { + let invalid = |reason: String| ConsoleError::InvalidTracelToml { + path: path.to_path_buf(), + reason, + }; + match std::fs::read_to_string(path) { + Ok(contents) => toml::from_str(&contents).map_err(|error| invalid(error.to_string())), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(Self::default()), + Err(error) => Err(invalid(error.to_string())), + } } } @@ -275,13 +282,37 @@ mod tests { } #[test] - fn a_malformed_tracel_toml_names_nothing() { + fn a_missing_tracel_toml_names_nothing() { + let dir = tempfile::tempdir().unwrap(); + assert!(matches!( - located(None, Some("env-project"), "namespace = "), + ProjectRef::from_vars(|_| None, dir.path()), Err(ConsoleError::NoNamespace) )); } + #[rstest] + #[case::malformed(|path: &Path| std::fs::write(path, "namespace = ").unwrap())] + #[case::unreadable(|path: &Path| std::fs::create_dir(path).unwrap())] + fn an_unreadable_or_malformed_tracel_toml_is_an_error_naming_it(#[case] write: fn(&Path)) { + let dir = tempfile::tempdir().unwrap(); + let tracel_toml = dir.path().join(TRACEL_TOML); + write(&tracel_toml); + + assert!(matches!( + ProjectRef::from_vars(|_| None, dir.path()), + Err(ConsoleError::InvalidTracelToml { path, .. }) if path == tracel_toml + )); + } + + #[test] + fn tracel_toml_is_not_read_when_the_environment_names_the_project() { + assert_eq!( + located(Some("env-owner"), Some("env-project"), "namespace = ").unwrap(), + ProjectRef::new("env-owner", "env-project") + ); + } + #[test] fn a_missing_namespace_or_project_says_where_to_set_it() { let no_namespace = located(None, Some("env-project"), "").unwrap_err(); diff --git a/crates/tracel-console/src/error.rs b/crates/tracel-console/src/error.rs index 1737adba..f8c10510 100644 --- a/crates/tracel-console/src/error.rs +++ b/crates/tracel-console/src/error.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use tracel_client::error::ClientError; /// Errors produced while configuring, authenticating with or calling the console. @@ -42,6 +44,14 @@ pub enum ConsoleError { (expected keys: `namespace` and `project`)" )] NoProject, + /// The `tracel.toml` that would name the project cannot be read, or is not valid TOML. + #[error("could not read {}: {reason}", path.display())] + InvalidTracelToml { + /// The file. + path: PathBuf, + /// Why it cannot be used. + reason: String, + }, /// The credential cannot be used for this request: an API key reaches project data only. #[error("this credential cannot be used for this request; API keys reach project data only")] CredentialNotAllowed, From 4a12e590f07b7e1e93d073201978379e4d4961e4 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:42:19 -0400 Subject: [PATCH 46/90] docs: the job command line docs match the generated flags A field named version now has a flag of its own, so only help and config are the command line's names. A boolean flag still reads true given no value, and takes a value only as --flag=value; --flag value no longer sets it. The crate docs and the README say so. --- README.md | 7 ++++--- crates/tracel/src/lib.rs | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6b8338b4..dbbb1d0f 100644 --- a/README.md +++ b/README.md @@ -159,11 +159,12 @@ Options: - A nested field's flag joins the keys with dots and writes `_` as `-`: the field `optimizer.weight_decay` is `--optimizer.weight-decay`. A field that is an array, a map, an optional object or an enum with data takes a JSON literal, such as `--layers '[64, 32]'`. A - boolean flag given no value is `true`. + boolean flag given no value is `true`, and takes a value only as `--flag=value`, such as + `--shuffle=false`. - The input is the default input, then the `--config` file, then the JSON document, then the flags, each merged onto the one before, so a later one wins. Launchers pass the JSON document. -- `help`, `version` and `config` are the command line's names: a field with one of them has no - flag, and is set through `--config` or the JSON document. +- `help` and `config` are the command line's names: a field with one of them has no flag, and is + set through `--config` or the JSON document. - `--completions ` prints the completion script for `bash`, `elvish`, `fish`, `powershell` or `zsh`: `cargo run -- --completions bash > cli.bash`. diff --git a/crates/tracel/src/lib.rs b/crates/tracel/src/lib.rs index 9ddc70e7..1b34c48c 100644 --- a/crates/tracel/src/lib.rs +++ b/crates/tracel/src/lib.rs @@ -125,11 +125,12 @@ //! - A nested field's flag joins the keys with dots and writes `_` as `-`: the field //! `optimizer.weight_decay` is `--optimizer.weight-decay`. A field that is an array, a map, an //! optional object or an enum with data takes a JSON literal, such as `--layers '[64, 32]'`. A -//! boolean flag given no value is `true`. +//! boolean flag given no value is `true`, and takes a value only as `--flag=value`, such as +//! `--shuffle=false`. //! - The input is the default input, then the `--config` file, then the JSON document, then the //! flags, each merged onto the one before, so a later one wins. Launchers pass the JSON document. -//! - `help`, `version` and `config` are the command line's names: a field with one of them has no -//! flag, and is set through `--config` or the JSON document. +//! - `help` and `config` are the command line's names: a field with one of them has no flag, and is +//! set through `--config` or the JSON document. //! - ` --completions ` prints the completion script for `bash`, `elvish`, `fish`, //! `powershell` or `zsh`. //! From 0e94e812446e7d52937de53fbfa0781bc0955165 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:43:06 -0400 Subject: [PATCH 47/90] fix: a console target reports a missing setting as the console does TargetError copied ConsoleError's missing credential, namespace and project into its own MissingSetting, and its invalid TRACEL_ENV into InvalidSetting, then rebuilt their messages from the variable name, so the same missing setting read differently from Target::from_env than from ProjectHandle::from_env. Console errors now pass through unchanged as TargetError::Console, and InvalidSetting is left for TRACEL_STATION_URL. The tests of the copied messages go with them; the console's own tests cover the messages. --- crates/tracel/src/target.rs | 93 +++---------------------------------- 1 file changed, 6 insertions(+), 87 deletions(-) diff --git a/crates/tracel/src/target.rs b/crates/tracel/src/target.rs index 77a17766..e382fb90 100644 --- a/crates/tracel/src/target.rs +++ b/crates/tracel/src/target.rs @@ -17,9 +17,6 @@ use crate::running_job; const TRACEL_TARGET: &str = "TRACEL_TARGET"; const TRACEL_RUNS_DIR: &str = "TRACEL_RUNS_DIR"; -const TRACEL_API_KEY: &str = "TRACEL_API_KEY"; -const TRACEL_NAMESPACE: &str = "TRACEL_NAMESPACE"; -const TRACEL_PROJECT: &str = "TRACEL_PROJECT"; #[cfg(feature = "station")] const TRACEL_STATION_URL: &str = "TRACEL_STATION_URL"; @@ -89,9 +86,9 @@ impl Target { doc = "| `TRACEL_STATION_URL` | the Station's base URL | `http://localhost:8000` |" )] /// - /// Fails with [`TargetError::UnknownTarget`] when `TRACEL_TARGET` names no target, and with - /// [`TargetError::MissingSetting`] or [`TargetError::InvalidSetting`] naming the variable to - /// set when the target cannot be filled in. + /// Fails with [`TargetError::UnknownTarget`] when `TRACEL_TARGET` names no target, with + /// [`TargetError::Console`] when the console target cannot be filled in, and with + /// [`TargetError::InvalidSetting`] when `TRACEL_STATION_URL` is not an http or https URL. pub fn from_env() -> Result { Self::from_vars(|name| std::env::var_os(name)) } @@ -258,13 +255,7 @@ pub enum TargetError { /// The `tracel` feature that adds the target. feature: &'static str, }, - /// A setting the target needs is not set. - #[error("{variable} is not set{}", fallback(.variable))] - MissingSetting { - /// The environment variable to set. - variable: &'static str, - }, - /// An environment variable holds a value the SDK does not accept. + /// `TRACEL_STATION_URL` holds a value that is not an http or https URL. #[error("invalid {variable} value `{value}`: expected {expected}")] InvalidSetting { /// The environment variable. @@ -282,50 +273,9 @@ pub enum TargetError { /// The kind of target that does not serve it. target: &'static str, }, - /// The console settings could not be read for another reason, such as a stored sign-in - /// that cannot be read. + /// The console target cannot be filled in or reached, as the console says. #[error(transparent)] - Console(ConsoleError), -} - -/// What else provides a setting whose variable is unset, as [`TargetError::MissingSetting`] says. -fn fallback(variable: &str) -> &'static str { - match variable { - TRACEL_API_KEY => ", and there is no `tracel login` sign-in for this console", - TRACEL_NAMESPACE => { - ", and tracel.toml sets no `namespace` (expected keys: `namespace` and `project`)" - } - TRACEL_PROJECT => { - ", and tracel.toml sets no `project` (expected keys: `namespace` and `project`)" - } - _ => "", - } -} - -impl From for TargetError { - fn from(error: ConsoleError) -> Self { - match error { - ConsoleError::NoCredentials => Self::MissingSetting { - variable: TRACEL_API_KEY, - }, - ConsoleError::NoNamespace => Self::MissingSetting { - variable: TRACEL_NAMESPACE, - }, - ConsoleError::NoProject => Self::MissingSetting { - variable: TRACEL_PROJECT, - }, - ConsoleError::InvalidSetting { - variable, - value, - expected, - } => Self::InvalidSetting { - variable, - value, - expected, - }, - error => Self::Console(error), - } - } + Console(#[from] ConsoleError), } #[cfg(test)] @@ -481,35 +431,4 @@ mod tests { }) )); } - - #[test] - fn a_missing_console_setting_names_the_variable_to_set() { - for (error, variable, fallback) in [ - (ConsoleError::NoCredentials, TRACEL_API_KEY, "tracel login"), - (ConsoleError::NoNamespace, TRACEL_NAMESPACE, "tracel.toml"), - (ConsoleError::NoProject, TRACEL_PROJECT, "tracel.toml"), - ] { - let error = TargetError::from(error); - - assert!( - matches!(error, TargetError::MissingSetting { variable: held } if held == variable), - "{error:?}" - ); - assert!(error.to_string().contains(variable), "{error}"); - assert!(error.to_string().contains(fallback), "{error}"); - } - } - - #[test] - fn a_missing_namespace_or_project_names_the_keys_of_tracel_toml() { - for (error, key) in [ - (ConsoleError::NoNamespace, "no `namespace`"), - (ConsoleError::NoProject, "no `project`"), - ] { - let message = TargetError::from(error).to_string(); - - assert!(message.contains(key), "{message}"); - assert!(message.contains("`namespace` and `project`"), "{message}"); - } - } } From 7f41fd3fc323db91952b9edec9301ae3764fd9b2 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:45:12 -0400 Subject: [PATCH 48/90] fix: a console target's services share one console connection Target::Console held the console, credential source and project, and each experiments(), models(), datasets() or inference() call built a new Console from them: one credential verification and one owner kind lookup per service. Target::Console now holds the ProjectHandle that ProjectHandle::from_vars reads, so every service reuses its connection, and a missing credential is reported by Target::from_env rather than by the first service built. Target::from_vars passes its lookup, the directory holding tracel.toml and the session store down to ProjectHandle::from_vars. The tests show that a console target is the project its variables name, and that one without a credential fails with the console's NoCredentials. The docs explaining the per-call construction go. --- crates/tracel/Cargo.toml | 4 +- crates/tracel/src/target.rs | 128 ++++++++++++++++++++---------------- 2 files changed, 73 insertions(+), 59 deletions(-) diff --git a/crates/tracel/Cargo.toml b/crates/tracel/Cargo.toml index ffb464f7..fcbab192 100644 --- a/crates/tracel/Cargo.toml +++ b/crates/tracel/Cargo.toml @@ -13,7 +13,7 @@ rust-version.workspace = true [features] default = [] -station = ["dep:tracel-station", "dep:url", "tracel-runtime/station"] +station = ["dep:tracel-station", "tracel-runtime/station"] burn = ["tracel-experiment/burn", "tracel-datasets/burn"] schema = ["tracel-runtime/schema"] @@ -28,7 +28,7 @@ tracel-datasets.workspace = true tracel-models.workspace = true serde_json.workspace = true thiserror.workspace = true -url = { workspace = true, optional = true } +url.workspace = true [dev-dependencies] serde = { workspace = true, features = ["std"] } diff --git a/crates/tracel/src/target.rs b/crates/tracel/src/target.rs index e382fb90..70256c86 100644 --- a/crates/tracel/src/target.rs +++ b/crates/tracel/src/target.rs @@ -1,8 +1,8 @@ use std::ffi::OsString; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; -use tracel_console::{Console, ConsoleError, CredentialSource, Env, ProjectHandle, ProjectRef}; +use tracel_console::{ConsoleError, FileSessionStore, ProjectHandle, SessionStoreError}; use tracel_datasets::DatasetRegistry; use tracel_experiment::Experiments; use tracel_experiment::local::LocalExperiments; @@ -10,7 +10,6 @@ use tracel_inference::{InferenceModule, NoopInferenceProvider}; use tracel_models::ModelRegistry; #[cfg(feature = "station")] use tracel_station::Station; -#[cfg(feature = "station")] use url::Url; use crate::running_job; @@ -35,8 +34,7 @@ const TARGETS: &str = "offline, console or station"; /// /// Build one directly, or read it from the environment with [`Target::from_env`]. Neither /// performs network I/O, and neither do the services a target builds: the first request that -/// needs a server reaches it. Each call to a service method binds a new connection; to share one -/// across services, build them from a [`ProjectHandle`] instead. +/// needs a server reaches it. #[derive(Debug, Clone)] #[non_exhaustive] pub enum Target { @@ -45,14 +43,11 @@ pub enum Target { /// The directory runs are recorded under. dir: PathBuf, }, - /// A project on the Tracel console. + /// A project on the Tracel console. The services the target builds share its connection, so + /// the credential is verified once. Console { - /// The console to reach. - env: Env, - /// Where the credential comes from. - credentials: CredentialSource, /// The project experiments, models and datasets belong to. - project: ProjectRef, + project: ProjectHandle, }, /// A Tracel Station (requires the `station` feature). #[cfg(feature = "station")] @@ -86,16 +81,26 @@ impl Target { doc = "| `TRACEL_STATION_URL` | the Station's base URL | `http://localhost:8000` |" )] /// - /// Fails with [`TargetError::UnknownTarget`] when `TRACEL_TARGET` names no target, with - /// [`TargetError::Console`] when the console target cannot be filled in, and with - /// [`TargetError::InvalidSetting`] when `TRACEL_STATION_URL` is not an http or https URL. + /// The console target is the project [`ProjectHandle::from_env`] reads, and fails as it + /// does, with [`TargetError::Console`]. Fails with [`TargetError::UnknownTarget`] when + /// `TRACEL_TARGET` names no target, and with [`TargetError::InvalidSetting`] when + /// `TRACEL_STATION_URL` is not an http or https URL. pub fn from_env() -> Result { - Self::from_vars(|name| std::env::var_os(name)) + Self::from_vars( + |name| std::env::var_os(name), + Path::new("."), + FileSessionStore::for_server, + ) } - /// [`from_env`](Self::from_env) over the values `lookup` gives this crate's variables. The - /// console's own variables are read by [`tracel_console`]. - fn from_vars(lookup: impl Fn(&str) -> Option) -> Result { + /// [`from_env`](Self::from_env), reading the variables from `lookup`, `tracel.toml` from + /// `dir`, and the stored `tracel login` session from the store `sessions` gives for the + /// console's URL. + fn from_vars( + lookup: impl Fn(&str) -> Option, + dir: &Path, + sessions: impl FnOnce(&Url) -> Result, + ) -> Result { let var = |name: &str| lookup(name).filter(|value| !value.is_empty()); let target = var(TRACEL_TARGET).map(|value| value.to_string_lossy().into_owned()); @@ -104,9 +109,7 @@ impl Target { dir: var(TRACEL_RUNS_DIR).map_or_else(|| DEFAULT_RUNS_DIR.into(), PathBuf::from), }), Some("console") => Ok(Target::Console { - env: tracel_console::env_from_environment()?, - credentials: CredentialSource::from_env(), - project: ProjectRef::from_env()?, + project: ProjectHandle::from_vars(&lookup, dir, sessions)?, }), #[cfg(feature = "station")] Some("station") => { @@ -153,11 +156,7 @@ impl Target { pub fn experiments(&self) -> Result { let experiments = match self { Target::Offline { dir } => Experiments::new(Arc::new(LocalExperiments::new(dir))), - Target::Console { - env, - credentials, - project, - } => console_project(env, credentials, project)?.experiments(), + Target::Console { project } => project.experiments(), #[cfg(feature = "station")] Target::Station { url } => Station::connect(url.clone()).experiments(), }; @@ -173,11 +172,7 @@ impl Target { service: "models", target: "offline", }), - Target::Console { - env, - credentials, - project, - } => Ok(console_project(env, credentials, project)?.models()), + Target::Console { project } => Ok(project.models()), #[cfg(feature = "station")] Target::Station { url } => Ok(Station::connect(url.clone()).models()), } @@ -192,11 +187,7 @@ impl Target { service: "datasets", target: "offline", }), - Target::Console { - env, - credentials, - project, - } => Ok(console_project(env, credentials, project)?.datasets()), + Target::Console { project } => Ok(project.datasets()), #[cfg(feature = "station")] Target::Station { url } => Ok(Station::connect(url.clone()).datasets()), } @@ -210,27 +201,13 @@ impl Target { pub fn inference(&self) -> Result { match self { Target::Offline { .. } => Ok(noop_inference()), - Target::Console { - env, - credentials, - project, - } => Ok(console_project(env, credentials, project)?.inference()), + Target::Console { project } => Ok(project.inference()), #[cfg(feature = "station")] Target::Station { .. } => Ok(noop_inference()), } } } -/// Binds `project` on the console `env` names, with the credential `credentials` provides. -fn console_project( - env: &Env, - credentials: &CredentialSource, - project: &ProjectRef, -) -> Result { - let console = Console::connect(env.clone(), credentials.resolve(env)?); - Ok(console.project(&project.namespace, &project.name)) -} - fn noop_inference() -> InferenceModule { InferenceModule::new(Arc::new(NoopInferenceProvider::new())) } @@ -280,16 +257,21 @@ pub enum TargetError { #[cfg(test)] mod tests { - use std::path::Path; + use tracel_console::ProjectRef; use super::*; fn target(vars: &[(&str, &str)]) -> Result { - Target::from_vars(|name| { - vars.iter() - .find(|(variable, _)| *variable == name) - .map(|(_, value)| OsString::from(value)) - }) + let dir = tempfile::tempdir().unwrap(); + Target::from_vars( + |name| { + vars.iter() + .find(|(variable, _)| *variable == name) + .map(|(_, value)| OsString::from(value)) + }, + dir.path(), + |_| Ok(FileSessionStore::new(dir.path().join("session.json"))), + ) } fn offline_dir(target: Result) -> PathBuf { @@ -431,4 +413,36 @@ mod tests { }) )); } + + #[test] + fn a_console_target_is_the_project_its_variables_name() { + let mnist = ProjectRef::new("alice", "mnist"); + + let console = target(&[ + (TRACEL_TARGET, "console"), + ("TRACEL_API_KEY", "tcl_key"), + ("TRACEL_NAMESPACE", mnist.namespace.as_str()), + ("TRACEL_PROJECT", mnist.name.as_str()), + ]); + + assert!(matches!( + console, + Ok(Target::Console { project }) + if ProjectRef::new(project.owner(), project.name()) == mnist + )); + } + + #[test] + fn a_console_target_without_a_credential_fails_as_the_console_does() { + let console = target(&[ + (TRACEL_TARGET, "console"), + ("TRACEL_NAMESPACE", "alice"), + ("TRACEL_PROJECT", "mnist"), + ]); + + assert!(matches!( + console, + Err(TargetError::Console(ConsoleError::NoCredentials)) + )); + } } From 65b6263bc65f6b4ccd240b6faccfee1204c8a186 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:49:55 -0400 Subject: [PATCH 49/90] fix: an artifact name several artifacts share loads the newest The console and the Station allow several artifacts of one experiment to share a name, and loading such an artifact by name failed as ambiguous. It now loads the newest of the exact matches by created_at, and the Ambiguous error is gone. Both lookups pick from the listed ArtifactResponse directly instead of through a generic helper, and the ambiguity test becomes a test that the newest is picked. --- Cargo.lock | 1 + crates/tracel-console/src/experiment.rs | 103 ++++++++++++-------- crates/tracel-station/Cargo.toml | 3 + crates/tracel-station/src/experiment/mod.rs | 99 +++++++++++-------- 4 files changed, 120 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba7f5acc..0a5e4476 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7422,6 +7422,7 @@ name = "tracel-station" version = "0.12.0" dependencies = [ "chrono", + "rstest", "serde", "serde_json", "serde_with", diff --git a/crates/tracel-console/src/experiment.rs b/crates/tracel-console/src/experiment.rs index 4e505129..2675ac0f 100644 --- a/crates/tracel-console/src/experiment.rs +++ b/crates/tracel-console/src/experiment.rs @@ -28,6 +28,7 @@ use tracel_experiment_remote::{ArtifactUploadError, ArtifactUploader, RemoteExpe use url::Url; use crate::console::ProjectScope; +use crate::wire::console_timestamp; use crate::{Namespace, NamespaceKind}; /// Experiment provider backed by the console's experiment run protocol. @@ -250,7 +251,7 @@ impl ExperimentArtifactClient { Ok(bundle) } - /// The artifact of the experiment named `name`. + /// The artifact of the experiment named `name`, the newest when several are. fn fetch(&self, name: &str) -> Result { let listed = self.scope.client()?.list_artifacts_by_name( &self.scope.owner, @@ -258,26 +259,24 @@ impl ExperimentArtifactClient { self.experiment_num, name, )?; - named(listed.items, name, |artifact| &artifact.name) + newest_named(listed.items, name) } } -/// The one item of `items` whose name, which `name_of` gives, is `name`. +/// The newest of `artifacts` named exactly `name`. /// /// The console's name filter lists the artifacts whose names contain the name asked for, so -/// asking for `model` can list `model-2` as well. -fn named(items: Vec, name: &str, name_of: impl Fn(&T) -> &str) -> Result { - let mut matching = items.into_iter().filter(|item| name_of(item) == name); - let found = matching - .next() - .ok_or_else(|| ArtifactError::NotFound(name.to_owned()))?; - match matching.count() { - 0 => Ok(found), - others => Err(ArtifactError::Ambiguous { - name: name.to_owned(), - count: others + 1, - }), - } +/// asking for `model` can list `model-2` as well, and an experiment can hold several artifacts of +/// one name. +fn newest_named( + artifacts: Vec, + name: &str, +) -> Result { + artifacts + .into_iter() + .filter(|artifact| artifact.name == name) + .max_by_key(|artifact| console_timestamp(&artifact.created_at)) + .ok_or_else(|| ArtifactError::NotFound(name.to_owned())) } fn artifact_kind_name(kind: ArtifactKind) -> &'static str { @@ -292,8 +291,6 @@ fn artifact_kind_name(kind: ArtifactKind) -> &'static str { enum ArtifactError { #[error("Artifact not found: {0}")] NotFound(String), - #[error("Artifact name is ambiguous: {count} artifacts are named {name}")] - Ambiguous { name: String, count: usize }, #[error(transparent)] Client(#[from] ClientError), #[error(transparent)] @@ -383,46 +380,66 @@ impl ArtifactUploader for ConsoleArtifactUploader { #[cfg(test)] mod tests { + use rstest::{fixture, rstest}; use tracel_client::console::Env; + use tracel_client::console::artifact::response::ArtifactSourceResponse; use super::*; - /// The id of the artifact `named` picks among `(id, name)` pairs. - fn pick( - artifacts: &[(&'static str, &'static str)], - name: &str, - ) -> Result<&'static str, ArtifactError> { - named(artifacts.to_vec(), name, |(_, name)| name).map(|(id, _)| id) + fn artifact(name: &str, created_at: &str) -> ArtifactResponse { + ArtifactResponse { + id: format!("{name} at {created_at}"), + created_at: created_at.to_string(), + name: name.to_string(), + kind: "model".to_string(), + bucket_id: "bucket".to_string(), + experiment: ArtifactSourceResponse { + id: 1, + experiment_num: 1, + }, + manifest: Value::Null, + } } - #[test] - fn an_artifact_is_picked_by_its_exact_name() { - let listed = [("2", "model-2"), ("1", "model"), ("3", "model.bpk")]; + #[fixture] + fn listed() -> Vec { + vec![ + artifact("model-2", "2026-10-01T00:00:00Z"), + artifact("model", "2026-10-01T00:00:00Z"), + artifact("model.bpk", "2026-10-01T00:00:00Z"), + ] + } - assert_eq!(pick(&listed, "model").unwrap(), "1"); - assert_eq!(pick(&listed, "model-2").unwrap(), "2"); + #[rstest] + #[case::a_name_others_extend("model")] + #[case::a_name_extending_another("model-2")] + fn an_artifact_is_picked_by_its_exact_name(listed: Vec, #[case] name: &str) { + assert_eq!(newest_named(listed, name).unwrap().name, name); } - #[test] - fn a_name_no_artifact_has_exactly_is_not_found() { - let error = pick(&[("2", "model-2")], "model").unwrap_err(); + #[rstest] + #[case::only_longer_names_listed(vec![artifact("model-2", "2026-10-01T00:00:00Z")])] + #[case::nothing_listed(vec![])] + fn a_name_no_artifact_has_exactly_is_not_found(#[case] listed: Vec) { + let asked = "model"; - assert!(matches!(&error, ArtifactError::NotFound(name) if name == "model")); assert!(matches!( - pick(&[], "model"), - Err(ArtifactError::NotFound(_)) + newest_named(listed, asked), + Err(ArtifactError::NotFound(name)) if name == asked )); } #[test] - fn a_name_several_artifacts_have_is_ambiguous() { - let error = pick(&[("1", "model"), ("2", "model-2"), ("3", "model")], "model").unwrap_err(); - - assert!(matches!(&error, ArtifactError::Ambiguous { count: 2, .. })); - assert_eq!( - error.to_string(), - "Artifact name is ambiguous: 2 artifacts are named model" - ); + fn the_newest_of_same_named_artifacts_is_picked() { + let newest = artifact("model", "2026-10-03T00:00:00Z"); + let listed = vec![ + artifact("model", "2026-10-01T00:00:00Z"), + newest.clone(), + artifact("model-2", "2026-10-04T00:00:00Z"), + artifact("model", "2026-10-02T00:00:00Z"), + ]; + + assert_eq!(newest_named(listed, "model").unwrap().id, newest.id); } fn page(api_url: &str, owner: Namespace, project: &str, num: i32) -> Option { diff --git a/crates/tracel-station/Cargo.toml b/crates/tracel-station/Cargo.toml index fd39cbe0..f59d818d 100644 --- a/crates/tracel-station/Cargo.toml +++ b/crates/tracel-station/Cargo.toml @@ -24,3 +24,6 @@ tracel-experiment = { workspace = true } tracel-experiment-remote = { workspace = true } tracel-models = { workspace = true } url = { workspace = true } + +[dev-dependencies] +rstest = { workspace = true } diff --git a/crates/tracel-station/src/experiment/mod.rs b/crates/tracel-station/src/experiment/mod.rs index 16fc7087..2c94fc0d 100644 --- a/crates/tracel-station/src/experiment/mod.rs +++ b/crates/tracel-station/src/experiment/mod.rs @@ -24,6 +24,7 @@ use tracel_experiment_remote::RemoteExperimentSession; use self::artifacts::{StationArtifactReader, StationArtifactUploader}; use crate::station::StationInner; +use crate::wire::station_timestamp; #[derive(Debug, thiserror::Error)] enum RunError { @@ -161,7 +162,7 @@ impl ExperimentArtifactClient { Ok(bundle) } - /// The artifact of the experiment named `name`. + /// The artifact of the experiment named `name`, the newest when several are. pub fn fetch(&self, name: &str) -> Result { let listed = self.client.experiments().list_artifacts( self.exp_path.experiment_num(), @@ -169,26 +170,24 @@ impl ExperimentArtifactClient { name: Some(name.to_string()), }, )?; - named(listed.items, name, |artifact| &artifact.name) + newest_named(listed.items, name) } } -/// The one item of `items` whose name, which `name_of` gives, is `name`. +/// The newest of `artifacts` named exactly `name`. /// /// The Station's name filter lists the artifacts whose names contain the name asked for, so -/// asking for `model` can list `model-2` as well. -fn named(items: Vec, name: &str, name_of: impl Fn(&T) -> &str) -> Result { - let mut matching = items.into_iter().filter(|item| name_of(item) == name); - let found = matching - .next() - .ok_or_else(|| ArtifactError::NotFound(name.to_owned()))?; - match matching.count() { - 0 => Ok(found), - others => Err(ArtifactError::Ambiguous { - name: name.to_owned(), - count: others + 1, - }), - } +/// asking for `model` can list `model-2` as well, and an experiment can hold several artifacts of +/// one name. +fn newest_named( + artifacts: Vec, + name: &str, +) -> Result { + artifacts + .into_iter() + .filter(|artifact| artifact.name == name) + .max_by_key(|artifact| station_timestamp(&artifact.created_at)) + .ok_or_else(|| ArtifactError::NotFound(name.to_owned())) } fn artifact_kind_name(kind: ArtifactKind) -> &'static str { @@ -203,8 +202,6 @@ fn artifact_kind_name(kind: ArtifactKind) -> &'static str { pub enum ArtifactError { #[error("Artifact not found: {0}")] NotFound(String), - #[error("Artifact name is ambiguous: {count} artifacts are named {name}")] - Ambiguous { name: String, count: usize }, #[error(transparent)] Client(#[from] ClientError), #[error(transparent)] @@ -266,43 +263,59 @@ fn create_run( #[cfg(test)] mod tests { + use rstest::{fixture, rstest}; + use super::*; - /// The id of the artifact `named` picks among `(id, name)` pairs. - fn pick( - artifacts: &[(&'static str, &'static str)], - name: &str, - ) -> Result<&'static str, ArtifactError> { - named(artifacts.to_vec(), name, |(_, name)| name).map(|(id, _)| id) + fn artifact(name: &str, created_at: &str) -> ArtifactResponse { + ArtifactResponse { + id: format!("{name} at {created_at}"), + created_at: created_at.to_string(), + name: name.to_string(), + kind: "model".to_string(), + experiment_num: 1, + manifest: Value::Null, + } } - #[test] - fn an_artifact_is_picked_by_its_exact_name() { - let listed = [("2", "model-2"), ("1", "model"), ("3", "model.bpk")]; + #[fixture] + fn listed() -> Vec { + vec![ + artifact("model-2", "2026-10-01T00:00:00Z"), + artifact("model", "2026-10-01T00:00:00Z"), + artifact("model.bpk", "2026-10-01T00:00:00Z"), + ] + } - assert_eq!(pick(&listed, "model").unwrap(), "1"); - assert_eq!(pick(&listed, "model-2").unwrap(), "2"); + #[rstest] + #[case::a_name_others_extend("model")] + #[case::a_name_extending_another("model-2")] + fn an_artifact_is_picked_by_its_exact_name(listed: Vec, #[case] name: &str) { + assert_eq!(newest_named(listed, name).unwrap().name, name); } - #[test] - fn a_name_no_artifact_has_exactly_is_not_found() { - let error = pick(&[("2", "model-2")], "model").unwrap_err(); + #[rstest] + #[case::only_longer_names_listed(vec![artifact("model-2", "2026-10-01T00:00:00Z")])] + #[case::nothing_listed(vec![])] + fn a_name_no_artifact_has_exactly_is_not_found(#[case] listed: Vec) { + let asked = "model"; - assert!(matches!(&error, ArtifactError::NotFound(name) if name == "model")); assert!(matches!( - pick(&[], "model"), - Err(ArtifactError::NotFound(_)) + newest_named(listed, asked), + Err(ArtifactError::NotFound(name)) if name == asked )); } #[test] - fn a_name_several_artifacts_have_is_ambiguous() { - let error = pick(&[("1", "model"), ("2", "model-2"), ("3", "model")], "model").unwrap_err(); - - assert!(matches!(&error, ArtifactError::Ambiguous { count: 2, .. })); - assert_eq!( - error.to_string(), - "Artifact name is ambiguous: 2 artifacts are named model" - ); + fn the_newest_of_same_named_artifacts_is_picked() { + let newest = artifact("model", "2026-10-03T00:00:00Z"); + let listed = vec![ + artifact("model", "2026-10-01T00:00:00Z"), + newest.clone(), + artifact("model-2", "2026-10-04T00:00:00Z"), + artifact("model", "2026-10-02T00:00:00Z"), + ]; + + assert_eq!(newest_named(listed, "model").unwrap().id, newest.id); } } From 5e42946c26014a55cac2fe45eed68b3202d87a64 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:51:24 -0400 Subject: [PATCH 50/90] test: experiments are linked to a job through the runtime's own seam The linking tests called the private link with a hand-built RunningJob and recorded the announced lines in an Arc>, so linked inside a job and JobScope::current() went untested. linked now reads the JobScope directly, and RunningJob and the announce parameter go. The tests run a job through Jobs::prepare and PreparedJob::run, and the experiments it starts come from a provider that records the attributes and arguments each one starts with: in the job, on a thread the job spawns, with no number or input, and outside any job. The announced line keeps its own test, which now covers a run with no location too. --- Cargo.lock | 1 + crates/tracel/Cargo.toml | 1 + crates/tracel/src/running_job.rs | 293 +++++++++++++++++-------------- 3 files changed, 159 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a5e4476..9e4662e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7226,6 +7226,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" name = "tracel" version = "0.12.0" dependencies = [ + "rstest", "serde", "serde_json", "tempfile", diff --git a/crates/tracel/Cargo.toml b/crates/tracel/Cargo.toml index fcbab192..652822a1 100644 --- a/crates/tracel/Cargo.toml +++ b/crates/tracel/Cargo.toml @@ -31,5 +31,6 @@ thiserror.workspace = true url.workspace = true [dev-dependencies] +rstest.workspace = true serde = { workspace = true, features = ["std"] } tempfile.workspace = true diff --git a/crates/tracel/src/running_job.rs b/crates/tracel/src/running_job.rs index 5a85b9ee..fa612930 100644 --- a/crates/tracel/src/running_job.rs +++ b/crates/tracel/src/running_job.rs @@ -10,53 +10,26 @@ use tracel_runtime::JobScope; /// number, as a number when it is one. const JOB_NUM_ATTRIBUTE: &str = "tracel.job_num"; -/// `experiments`, linked to the job the current thread runs, when it runs one, as [`link`] links -/// them, each run started from them printing where it records to stderr; as they are otherwise. -/// The link holds on any thread a run is started on. +/// `experiments`, linked to the job the current thread runs, when it runs one; as they are +/// otherwise. +/// +/// Each experiment started from linked experiments, on any thread, belongs to the job: it records +/// the job's number as its `tracel.job_num` attribute and the job's input, unless it is `null`, as +/// its arguments, and it prints where it records to stderr as it starts. pub fn linked(experiments: Experiments) -> Experiments { - match RunningJob::current() { - Some(job) => link(experiments, job, |line| eprintln!("{line}")), - None => experiments, - } -} - -/// What a job gives the experiments started in it. -struct RunningJob { - /// The number the job's launcher gave it. - num: Option, - /// The input the job runs with, as JSON. - input: Value, -} - -impl RunningJob { - /// The job the current thread runs, as its [`JobScope`] gives it, while it runs one. - fn current() -> Option { - let scope = JobScope::current()?; - Some(Self { - num: scope.num().map(str::to_string), - input: scope.input().clone(), - }) - } -} - -/// `experiments`, each started from which belongs to `job`: it records the job's number as its -/// `tracel.job_num` attribute and the job's input, unless it is `null`, as its arguments, and it -/// gives `announce` the line saying where it records as it starts. -fn link( - experiments: Experiments, - job: RunningJob, - announce: impl Fn(&str) + Send + Sync + 'static, -) -> Experiments { + let Some(job) = JobScope::current() else { + return experiments; + }; let attributes = job - .num - .map(|num| (JOB_NUM_ATTRIBUTE.to_string(), job_num(&num))); - let experiments = match job.input { + .num() + .map(|num| (JOB_NUM_ATTRIBUTE.to_string(), job_num(num))); + let experiments = match job.input() { Value::Null => experiments, - input => experiments.arguments(input), + input => experiments.arguments(input.clone()), }; experiments .attributes(HashMap::from_iter(attributes)) - .on_start(move |run| announce(&recording(run))) + .on_start(|run| eprintln!("{}", recording(run))) } /// The line saying where `run` records: its number and its page on the console, or its directory @@ -79,107 +52,157 @@ fn job_num(num: &str) -> Value { #[cfg(test)] mod tests { - use std::path::Path; + use std::path::PathBuf; use std::sync::{Arc, Mutex}; + use rstest::{fixture, rstest}; use serde_json::json; - use tracel_experiment::ExperimentProvider; - use tracel_experiment::local::LocalExperiments; + use tracel_experiment::error::ExperimentError; + use tracel_experiment::reader::{ + ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact, + }; + use tracel_experiment::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; + use tracel_experiment::{ArtifactKind, CancelToken, ExperimentId, ExperimentProvider}; + use tracel_runtime::{BoxError, Execution, Jobs}; use super::*; - /// What a test keeps of what it is given. - type Kept = Arc>>; + const ID: &str = "7"; + const PAGE: &str = "https://console.tracel.ai/users/alice/projects/mnist/experiments/7"; + + #[derive(Debug, Default, PartialEq)] + struct Recorded { + attributes: HashMap, + arguments: Option, + } + + #[derive(Clone, Default)] + struct Recorder(Arc>); - /// A job numbered `num`, running with `input`. - fn job(num: Option<&str>, input: Value) -> RunningJob { - RunningJob { - num: num.map(str::to_string), - input, + impl Recorder { + fn experiments(&self) -> Experiments { + Experiments::new(Arc::new(self.clone())) + } + + fn recorded(&self) -> Recorded { + std::mem::take(&mut self.0.lock().unwrap()) } } - fn offline(dir: &Path) -> Experiments { - Experiments::new(Arc::new(LocalExperiments::new(dir))) + impl ExperimentProvider for Recorder { + fn create_experiment( + &self, + _: String, + attributes: HashMap, + ) -> Result { + self.0.lock().unwrap().attributes = attributes; + Ok(ExperimentRun::new( + ID, + self.clone(), + self.clone(), + CancelToken::new(), + )) + } } - /// `experiments` linked to `job`, and the lines its runs announce. - fn linked_to(experiments: Experiments, job: RunningJob) -> (Experiments, Kept) { - let announced = Arc::new(Mutex::new(Vec::new())); - let experiments = link(experiments, job, { - let announced = announced.clone(); - move |line| announced.lock().unwrap().push(line.to_string()) - }); - (experiments, announced) + impl ExperimentSession for Recorder { + fn record_event(&self, event: Event) -> Result<(), ExperimentError> { + if let Event::Args(arguments) = event { + self.0.lock().unwrap().arguments = Some(arguments); + } + Ok(()) + } + + fn save_artifact( + &self, + _: &str, + _: ArtifactKind, + _: Box, + ) -> Result<(), ExperimentError> { + Ok(()) + } + + fn finish(&self, _: ExperimentCompletion) -> Result<(), ExperimentError> { + Ok(()) + } + } + + impl ExperimentArtifactReader for Recorder { + fn load_artifact_raw( + &self, + _: ExperimentId, + _: &str, + ) -> Result { + Err(ExperimentReaderError::new("a recorder keeps no artifacts")) + } } - /// The lines of the events file of run `num` of `train`, under `dir`. - fn events(dir: &Path, num: u64) -> Vec { - std::fs::read_to_string(dir.join(format!("train/{num}/events.jsonl"))) + type Start = fn(Experiments) -> Result<(), BoxError>; + + fn start(experiments: Experiments) -> Result<(), BoxError> { + experiments.start("train")?.finish()?; + Ok(()) + } + + fn start_on_another_thread(experiments: Experiments) -> Result<(), BoxError> { + std::thread::spawn(move || start(experiments)) + .join() .unwrap() - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .collect() } - #[test] - fn an_experiment_started_in_a_job_belongs_to_it() { - let dir = tempfile::tempdir().unwrap(); - let job = job(Some("12"), json!({"epochs": 2})); - - let (experiments, announced) = linked_to(offline(dir.path()), job); - experiments.start("train").unwrap().finish().unwrap(); - - let runs = dir.path().canonicalize().unwrap(); - assert!(events(dir.path(), 1).contains(&json!({ - "type": "attribute", - "key": "tracel.job_num", - "value": 12 - }))); - assert!(events(dir.path(), 1).contains(&json!({ - "type": "arguments", - "value": {"epochs": 2} - }))); + fn recorded_in_a_job(execution: Execution, input: Value, start: Start) -> Recorded { + let recorder = Recorder::default(); + let train = { + let recorder = recorder.clone(); + move |_: Value| start(linked(recorder.experiments())) + }; + Jobs::new() + .job("train", train) + .prepare("train", input) + .unwrap() + .run(execution) + .into_result() + .unwrap(); + recorder.recorded() + } + + #[fixture] + fn input() -> Value { + json!({"epochs": 2}) + } + + #[rstest] + #[case::on_the_jobs_thread(start)] + #[case::on_a_thread_the_job_spawns(start_on_another_thread)] + fn an_experiment_started_in_a_job_belongs_to_it(#[case] start: Start, input: Value) { + let num = "12"; + + let recorded = recorded_in_a_job(Execution::new().with_num(num), input.clone(), start); + assert_eq!( - *announced.lock().unwrap(), - [format!( - "Recording offline in {}", - runs.join("train/1").display() - )] + recorded, + Recorded { + attributes: HashMap::from([(JOB_NUM_ATTRIBUTE.to_string(), job_num(num))]), + arguments: Some(input), + } ); } #[test] fn an_experiment_started_in_a_job_without_a_number_or_input_records_neither() { - let dir = tempfile::tempdir().unwrap(); - let job = job(None, Value::Null); - let (experiments, announced) = linked_to(offline(dir.path()), job); - - experiments.start("train").unwrap().finish().unwrap(); - - assert!( - events(dir.path(), 1) - .iter() - .all(|event| event["type"] != "attribute" && event["type"] != "arguments") + assert_eq!( + recorded_in_a_job(Execution::new(), Value::Null, start), + Recorded::default() ); - assert_eq!(announced.lock().unwrap().len(), 1); } #[test] fn outside_a_job_experiments_are_not_linked() { - let dir = tempfile::tempdir().unwrap(); + let recorder = Recorder::default(); - linked(offline(dir.path())) - .start("train") - .unwrap() - .finish() - .unwrap(); + start(linked(recorder.experiments())).unwrap(); - assert!( - events(dir.path(), 1) - .iter() - .all(|event| event["type"] != "attribute" && event["type"] != "arguments") - ); + assert_eq!(recorder.recorded(), Recorded::default()); } #[test] @@ -188,30 +211,28 @@ mod tests { assert_eq!(job_num("job-42"), json!("job-42")); } - #[test] - fn an_experiment_announces_its_page_or_its_directory() { - let dir = tempfile::tempdir().unwrap(); - let experiments = LocalExperiments::new(dir.path()); - let page = "https://console.tracel.ai/users/me/projects/mnist/experiments/2"; - - let offline = experiments - .create_experiment("mnist".to_string(), HashMap::new()) - .unwrap(); - let on_the_console = experiments - .create_experiment("mnist".to_string(), HashMap::new()) - .unwrap() - .with_location(ExperimentLocation::Url(page.to_string())); - - assert_eq!( - recording(&offline), - format!( - "Recording offline in {}", - dir.path().canonicalize().unwrap().join("mnist/1").display() - ) - ); - assert_eq!( - recording(&on_the_console), - format!("Recording experiment 2: {page}") + #[rstest] + #[case::on_the_console(Some(ExperimentLocation::Url(PAGE.to_string())), PAGE)] + #[case::offline( + Some(ExperimentLocation::Dir(PathBuf::from("runs/mnist/1"))), + "runs/mnist/1" + )] + #[case::without_a_location(None, ID)] + fn an_experiment_announces_where_it_records( + #[case] location: Option, + #[case] named: &str, + ) { + let run = ExperimentRun::new( + ID, + Recorder::default(), + Recorder::default(), + CancelToken::new(), ); + let run = match location { + Some(location) => run.with_location(location), + None => run, + }; + + assert!(recording(&run).contains(named)); } } From bff2c4143f9f93bd9dcaf294bfb4b38a12e0e9b5 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:52:11 -0400 Subject: [PATCH 51/90] docs: the crates name their registries instead of capabilities The model and dataset crates' descriptions, the dataset crate's burn feature note and the console crate's docs still called the registries capabilities, a name the code no longer uses. --- crates/tracel-console/src/lib.rs | 3 ++- crates/tracel-datasets/Cargo.toml | 4 ++-- crates/tracel-models/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/tracel-console/src/lib.rs b/crates/tracel-console/src/lib.rs index be24402e..a3846832 100644 --- a/crates/tracel-console/src/lib.rs +++ b/crates/tracel-console/src/lib.rs @@ -4,7 +4,8 @@ //! //! [`Console`] owns one connection to the console. Creating it performs no I/O: the first //! request verifies the credential. Project handles are cheap views over that shared client and -//! vend backend-agnostic, project-scoped capabilities without performing I/O when created. +//! vend backend-agnostic, project-scoped registries and services without performing I/O when +//! created. //! //! [`Console::from_env`] and [`ProjectHandle::from_env`] read the console, the credential and the //! project from `TRACEL_ENV`, `TRACEL_API_KEY`, `TRACEL_NAMESPACE` and `TRACEL_PROJECT`, falling diff --git a/crates/tracel-datasets/Cargo.toml b/crates/tracel-datasets/Cargo.toml index ea2bc515..9dcfd726 100644 --- a/crates/tracel-datasets/Cargo.toml +++ b/crates/tracel-datasets/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tracel-datasets" -description = "Backend-agnostic dataset domain and capability for Tracel." +description = "Backend-agnostic dataset domain and registry for Tracel." edition.workspace = true version.workspace = true readme.workspace = true @@ -13,7 +13,7 @@ categories.workspace = true [features] default = [] -# Burn `Dataset` adapters over the capability. Off by default so that backends composing +# Burn `Dataset` adapters over dataset versions. Off by default so that backends composing # this crate stay burn-free. burn = ["dep:burn"] diff --git a/crates/tracel-models/Cargo.toml b/crates/tracel-models/Cargo.toml index 0eb5d36d..e3ae71c0 100644 --- a/crates/tracel-models/Cargo.toml +++ b/crates/tracel-models/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tracel-models" -description = "Backend-agnostic model domain and capability for Tracel." +description = "Backend-agnostic model domain and registry for Tracel." edition.workspace = true version.workspace = true readme.workspace = true From 2d6af690cb4e7d8d6462387347fc7139628993cf Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:53:11 -0400 Subject: [PATCH 52/90] test: the example bundle decodes without an empty metadata check ExampleConfig's decode read and parsed its metadata file only to reach an if-let whose body was a comment, so it checked nothing. It goes, and decode reads the config alone. --- crates/tracel-artifact/src/bundle/memory/tests.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/crates/tracel-artifact/src/bundle/memory/tests.rs b/crates/tracel-artifact/src/bundle/memory/tests.rs index f5ae034e..2bb7707d 100644 --- a/crates/tracel-artifact/src/bundle/memory/tests.rs +++ b/crates/tracel-artifact/src/bundle/memory/tests.rs @@ -96,18 +96,6 @@ impl BundleDecode for ExampleConfig { let config: ExampleConfig = serde_json::from_str(&config_content) .map_err(|e| format!("Failed to parse config JSON: {}", e))?; - // Optionally validate metadata exists (but don't require it for decoding) - if let Ok(mut metadata_reader) = source.open(&settings.metadata_filename) { - let mut metadata_content = String::new(); - if metadata_reader - .read_to_string(&mut metadata_content) - .is_ok() - && let Ok(_metadata) = serde_json::from_str::(&metadata_content) - { - // Could validate version compatibility here - } - } - Ok(config) } } From 833366c2328a7a4f2f91feac44479329a2e1b0c2 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:53:21 -0400 Subject: [PATCH 53/90] docs: the README says TRACEL_TARGET=station needs the station feature The README's variable table listed only offline and console, though Target::from_env reads station too when tracel is built with its station feature. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dbbb1d0f..f9e168e6 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ complete, runnable version. | Variable | Value | Default | | --- | --- | --- | -| `TRACEL_TARGET` | `offline` or `console` | `offline` | +| `TRACEL_TARGET` | `offline`, `console`, or `station` with the `station` feature of `tracel` | `offline` | | `TRACEL_RUNS_DIR` | the directory offline runs are recorded under | `./runs` | | `TRACEL_ENV` | the console to reach | `Production` | | `TRACEL_API_KEY` | an API key or a job token | the stored `tracel login` session | From ef550ade9340d4550c61433d9fb4e83564480180 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 09:54:53 -0400 Subject: [PATCH 54/90] ci: lint and tests build the station, burn and schema features No workspace member turns these features on, so CI never built the code behind them: the station runner's tests stopped compiling on main without CI noticing. Lint, unit, integration and documentation tests now pass tracel/station, tracel/burn and tracel/schema, which reach every feature-gated module of the SDK. --all-features would also turn on the mnist example's cuda backend, which the runner cannot build. --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00d71fff..14bd9d33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI env: CARGO_TERM_COLOR: always + FEATURES: tracel/station,tracel/burn,tracel/schema # For now we execute CI only on PR to save on CI time on: @@ -33,16 +34,16 @@ jobs: run: cargo xtask check format # -------------------------------------------------------------------------------- - name: Lint - run: cargo xtask check lint + run: cargo xtask check --features "$FEATURES" lint # -------------------------------------------------------------------------------- - name: Typos uses: tracel-ai/github-actions/check-typos@v11 # -------------------------------------------------------------------------------- - name: Unit Tests - run: cargo xtask test unit + run: cargo xtask test --features "$FEATURES" unit # -------------------------------------------------------------------------------- - name: Integration Tests - run: cargo xtask test integration + run: cargo xtask test --features "$FEATURES" integration # -------------------------------------------------------------------------------- - name: Documentation Tests - run: cargo xtask doc tests + run: cargo xtask doc --features "$FEATURES" tests From 5bc42400ea9966dd09666f6a886a45c22344e463 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 10:38:34 -0400 Subject: [PATCH 55/90] refactor: the process reads its launch variables once, in Jobs::run execute took a lookup over the environment and passed it on to describe and to the job's execution, so reading TRACEL_DESCRIBE and TRACEL_JOB_NUM was spread over three functions. Jobs::run now reads both into a Launch, which treats an empty value as unset, and execute and describe take what it holds. The tests build a Launch instead of faking the environment. --- crates/tracel-runtime/src/process/describe.rs | 42 ++--- crates/tracel-runtime/src/process/mod.rs | 151 ++++++++++-------- 2 files changed, 98 insertions(+), 95 deletions(-) diff --git a/crates/tracel-runtime/src/process/describe.rs b/crates/tracel-runtime/src/process/describe.rs index b097271b..ab8550d5 100644 --- a/crates/tracel-runtime/src/process/describe.rs +++ b/crates/tracel-runtime/src/process/describe.rs @@ -1,32 +1,26 @@ //! Describing the registered jobs instead of running one. -use std::ffi::OsString; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; -use tracel_job::{DefinitionsFile, TRACEL_DESCRIBE}; +use tracel_job::DefinitionsFile; use crate::JobDefinition; use crate::process::file::write_atomically; -/// Writes the definitions file of the jobs `definitions` gives when `TRACEL_DESCRIBE`, among the -/// environment variables `lookup` gives, names a path. -/// -/// Returns whether it did; the program then runs no job. +/// Writes the definitions file of the jobs `definitions` gives to `path`. pub fn describe<'a>( definitions: impl IntoIterator, - lookup: impl Fn(&str) -> Option, -) -> Result { - let Some(path) = lookup(TRACEL_DESCRIBE).filter(|path| !path.is_empty()) else { - return Ok(false); - }; - let path = PathBuf::from(path); + path: &Path, +) -> Result<(), DescribeError> { let file = DefinitionsFile::new( env!("CARGO_PKG_VERSION"), definitions.into_iter().cloned().collect(), ); - write_atomically(&path, &file.to_json()).map_err(|source| DescribeError { path, source })?; - Ok(true) + write_atomically(path, &file.to_json()).map_err(|source| DescribeError { + path: path.to_path_buf(), + source, + }) } /// The definitions file could not be written. @@ -39,8 +33,6 @@ pub struct DescribeError { #[cfg(test)] mod tests { - use std::path::Path; - use serde_json::{Value, json}; use super::*; @@ -62,18 +54,12 @@ mod tests { ] } - fn describe_to(path: &Path) -> Result { - describe(&definitions(), |name| { - (name == TRACEL_DESCRIBE).then(|| path.as_os_str().to_owned()) - }) - } - #[test] fn the_definitions_file_lists_every_job() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("jobs.json"); - assert!(describe_to(&path).unwrap()); + describe(&definitions(), &path).unwrap(); let file: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); assert_eq!( @@ -104,14 +90,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("missing").join("jobs.json"); - let error = describe_to(&path).unwrap_err(); + let error = describe(&definitions(), &path).unwrap_err(); assert_eq!(error.path, path); } - - #[test] - fn an_unset_or_empty_tracel_describe_writes_nothing() { - assert!(!describe(&definitions(), |_| None).unwrap()); - assert!(!describe(&definitions(), |_| Some(OsString::new())).unwrap()); - } } diff --git a/crates/tracel-runtime/src/process/mod.rs b/crates/tracel-runtime/src/process/mod.rs index 449a4fe7..771f28db 100644 --- a/crates/tracel-runtime/src/process/mod.rs +++ b/crates/tracel-runtime/src/process/mod.rs @@ -6,15 +6,15 @@ mod error; mod file; use std::ffi::OsString; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::ExitCode; use clap::Command; use clap::error::{ContextKind, ContextValue, ErrorKind}; use clap_complete::Shell; use serde_json::Value; -use tracel_job::TRACEL_JOB_NUM; use tracel_job::cli::{command, completions, job_input}; +use tracel_job::{TRACEL_DESCRIBE, TRACEL_JOB_NUM}; use crate::{Execution, Jobs}; use describe::describe; @@ -47,7 +47,11 @@ impl Jobs { /// [`job_num`](crate::job_num) gives the number its launcher gave it, and /// [`job_input`](crate::job_input) the input it runs with. pub fn run(self) -> ExitCode { - let exit = match execute(&self, std::env::args_os(), |name| std::env::var_os(name)) { + let launch = Launch::new( + std::env::var_os(TRACEL_DESCRIBE), + std::env::var_os(TRACEL_JOB_NUM), + ); + let exit = match execute(&self, std::env::args_os(), launch) { Ok(()) => tracel_job::ExitCode::Completed, Err(error) => { error.print(); @@ -58,21 +62,49 @@ impl Jobs { } } -/// Does what the command-line arguments `args`, the program's name first, and the environment -/// variables `vars` looks up ask of `jobs`: writes their definitions, prints the help, the version -/// or a completion script, or runs the job named. +/// What the program's launcher asks of it through [`TRACEL_DESCRIBE`] and [`TRACEL_JOB_NUM`]. +struct Launch { + /// Where to write the jobs' definitions, instead of running a job. + describe: Option, + /// The number of the job the program runs. + num: Option, +} + +impl Launch { + /// The launch the values of `TRACEL_DESCRIBE` and `TRACEL_JOB_NUM` ask for, an empty value + /// counting as unset. + fn new(describe: Option, num: Option) -> Self { + let set = |value: Option| value.filter(|value| !value.is_empty()); + Self { + describe: set(describe).map(PathBuf::from), + num: set(num).map(|num| num.to_string_lossy().into_owned()), + } + } + + /// The execution of the job the program runs: numbered when its launcher numbered it. + fn execution(&self) -> Execution { + match &self.num { + Some(num) => Execution::new().with_num(num.clone()), + None => Execution::new(), + } + } +} + +/// Does what the command-line arguments `args`, the program's name first, and the `launch` ask +/// of `jobs`: writes their definitions, prints the help, the version or a completion script, or +/// runs the job named. fn execute( jobs: &Jobs, args: impl IntoIterator, - vars: impl Fn(&str) -> Option, + launch: Launch, ) -> Result<(), RunError> { - if describe(jobs.definitions(), &vars)? { - return Ok(()); + if let Some(path) = &launch.describe { + return Ok(describe(jobs.definitions(), path)?); } let args: Vec = args.into_iter().collect(); let mut command = command_line(jobs, program_name(args.first())); match read_request(jobs, &mut command, args)? { - Request::Run { job, input } => dispatch(jobs, &job, input, execution(vars)), + Request::Run { job, input } => dispatch(jobs, &job, input, launch.execution()), Request::Completions(shell) => { let name = command.get_name().to_string(); clap_complete::generate(shell, &mut command, name, &mut std::io::stdout()); @@ -142,15 +174,6 @@ where } } -/// The execution of the job the process runs, as the environment variables `lookup` gives -/// describe it: numbered by [`TRACEL_JOB_NUM`] when its launcher sets it. -fn execution(lookup: impl Fn(&str) -> Option) -> Execution { - match lookup(TRACEL_JOB_NUM).filter(|num| !num.is_empty()) { - Some(num) => Execution::new().with_num(num.to_string_lossy()), - None => Execution::new(), - } -} - /// Runs the job `name` of `jobs` with `input`, launched as `execution` says. fn dispatch(jobs: &Jobs, name: &str, input: Value, execution: Execution) -> Result<(), RunError> { jobs.prepare(name, input)? @@ -170,9 +193,11 @@ fn program_name(program: Option<&OsString>) -> String { #[cfg(test)] mod tests { + use std::sync::{Arc, OnceLock}; + use rstest::{fixture, rstest}; use serde::{Deserialize, Serialize}; - use tracel_job::{DefinitionsFile, TRACEL_DESCRIBE}; + use tracel_job::DefinitionsFile; use super::*; use crate::{BoxError, job_num}; @@ -200,67 +225,65 @@ mod tests { ["program"].iter().chain(args).map(OsString::from) } - /// The environment variable `name` set to `value`, and no other. - fn var(name: &'static str, value: impl Into) -> impl Fn(&str) -> Option { - let value = value.into(); - move |asked| (asked == name).then(|| value.clone()) + fn describing(path: &Path) -> Launch { + Launch::new(Some(path.as_os_str().to_owned()), None) } - /// Runs the command line `program ` with no environment variables set. fn run(jobs: Jobs, args: &[&str]) -> Result<(), RunError> { - execute(&jobs, arguments(args), |_| None) + execute(&jobs, arguments(args), Launch::new(None, None)) } #[test] - fn a_launcher_asking_for_the_definitions_gets_them_and_no_job_runs() { - for args in [&[][..], &["train"], &["--bogus"]] { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("jobs.json"); - let jobs = jobs(); - - execute(&jobs, arguments(args), var(TRACEL_DESCRIBE, &path)).unwrap(); - - let file = DefinitionsFile::from_json(&std::fs::read(&path).unwrap()).unwrap(); - assert_eq!( - file.jobs, - jobs.definitions().cloned().collect::>(), - "{args:?}" - ); - } + fn an_empty_launch_variable_counts_as_unset() { + let launch = Launch::new(Some(OsString::new()), Some(OsString::new())); + + assert_eq!((launch.describe, launch.num), (None, None)); } - #[test] - fn definitions_that_cannot_be_written_fail_the_run_with_exit_code_1() { + #[rstest] + #[case::no_job(&[])] + #[case::a_job_that_fails(&["fail"])] + #[case::an_unusable_command_line(&["--bogus"])] + fn a_launcher_asking_for_the_definitions_gets_them_instead_of_a_run( + jobs: Jobs, + #[case] args: &[&str], + ) { let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("missing").join("jobs.json"); - let jobs = jobs(); + let path = dir.path().join("jobs.json"); - let failed = - execute(&jobs, arguments(&["train"]), var(TRACEL_DESCRIBE, &path)).unwrap_err(); + execute(&jobs, arguments(args), describing(&path)).unwrap(); - assert!(matches!(failed, RunError::Describe(_)), "{failed:?}"); - assert_eq!(failed.exit_code().code(), 1); + let file = DefinitionsFile::from_json(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(file.jobs, jobs.definitions().cloned().collect::>()); } - #[test] - fn printing_the_version_ends_the_run_successfully() { - let jobs = jobs().version("1.2.3"); + #[rstest] + fn definitions_that_cannot_be_written_fail_the_run(jobs: Jobs) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("missing").join("jobs.json"); + + let failed = execute(&jobs, arguments(&["train"]), describing(&path)).unwrap_err(); - execute(&jobs, arguments(&["--version"]), |_| None).unwrap(); + assert_eq!(failed.exit_code(), tracel_job::ExitCode::Failed); } - #[test] - fn the_job_is_numbered_by_tracel_job_num_when_its_launcher_sets_it() { - for (num, expected) in [(Some("12"), Some("12")), (Some(""), None), (None, None)] { - let jobs = Jobs::new().job("train", move |()| match job_num() { - seen if seen.as_deref() == expected => Ok(()), - seen => Err::<(), BoxError>(format!("numbered {seen:?}").into()), - }); - let vars = move |name: &str| num.filter(|_| name == TRACEL_JOB_NUM).map(OsString::from); + #[rstest] + #[case::numbered(Some("12"))] + #[case::unnumbered(None)] + fn the_job_is_numbered_as_its_launcher_numbers_it(#[case] num: Option<&str>) { + let seen = Arc::new(OnceLock::new()); + let jobs = Jobs::new().job("train", { + let seen = seen.clone(); + move |()| { + let _ = seen.set(job_num()); + Ok::<_, BoxError>(()) + } + }); + let launch = Launch::new(None, num.map(OsString::from)); - execute(&jobs, arguments(&["train"]), vars) - .unwrap_or_else(|error| panic!("{num:?}: {error}")); - } + execute(&jobs, arguments(&["train"]), launch).unwrap(); + + assert_eq!(seen.get(), Some(&num.map(str::to_string))); } #[rstest] From 4c398ea4c25addf9c8a2de28ef7f355a4131f132 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:26:46 -0400 Subject: [PATCH 56/90] test: an exit code reads back from its code, one case per code The exit code test walked two hand-written tables in one body. Each exit code and each code the protocol does not give is now its own rstest value, and the codes themselves are no longer restated. --- crates/tracel-job/src/run.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/tracel-job/src/run.rs b/crates/tracel-job/src/run.rs index 5fe4859a..cac2d1bf 100644 --- a/crates/tracel-job/src/run.rs +++ b/crates/tracel-job/src/run.rs @@ -42,20 +42,21 @@ impl From for std::process::ExitCode { #[cfg(test)] mod tests { + use rstest::rstest; + use super::*; - #[test] - fn each_exit_code_reads_back_from_its_code() { - for (exit, code) in [ - (ExitCode::Completed, 0), - (ExitCode::Failed, 1), - (ExitCode::Rejected, 2), - ] { - assert_eq!(exit.code(), code); - assert_eq!(ExitCode::from_code(i32::from(code)), Some(exit)); - } - for code in [-1, 3, 101, 130, 137, 256] { - assert_eq!(ExitCode::from_code(code), None); - } + #[rstest] + fn an_exit_code_reads_back_from_its_code( + #[values(ExitCode::Completed, ExitCode::Failed, ExitCode::Rejected)] exit: ExitCode, + ) { + assert_eq!(ExitCode::from_code(exit.code().into()), Some(exit)); + } + + #[rstest] + fn a_code_the_protocol_does_not_give_is_no_exit_code( + #[values(-1, 3, 101, 130, 137, 256)] code: i32, + ) { + assert_eq!(ExitCode::from_code(code), None); } } From 4e824316ccad66a62b3d29b57dd9f56d3deab843 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 13:34:27 -0400 Subject: [PATCH 57/90] test: the definitions file tests no longer test serde The file was compared as pretty-printed text, and a protocol error by its message. The text pin checked what serde writes and is gone; a file of another protocol is refused, checked by its ParseError variant. --- crates/tracel-job/src/definitions.rs | 83 ++-------------------------- 1 file changed, 5 insertions(+), 78 deletions(-) diff --git a/crates/tracel-job/src/definitions.rs b/crates/tracel-job/src/definitions.rs index 82e01f37..3eaeab47 100644 --- a/crates/tracel-job/src/definitions.rs +++ b/crates/tracel-job/src/definitions.rs @@ -113,60 +113,6 @@ mod tests { use super::*; - fn file() -> DefinitionsFile { - let jobs = vec![ - JobDefinition { - name: "echo".to_string(), - description: None, - input_schema: Some(json!({"type": "string"})), - input_example: None, - }, - JobDefinition { - name: "train".to_string(), - description: Some("Train the model".to_string()), - input_schema: None, - input_example: Some(json!({"epochs": 10, "optimizer": {"lr": 0.001}})), - }, - ]; - DefinitionsFile::new("0.10.0", jobs) - } - - #[test] - fn the_file_is_pretty_printed_json_ending_with_a_newline() { - let expected = r#"{ - "protocol": 1, - "sdk_version": "0.10.0", - "jobs": [ - { - "name": "echo", - "description": null, - "input_schema": { - "type": "string" - }, - "input_example": null - }, - { - "name": "train", - "description": "Train the model", - "input_schema": null, - "input_example": { - "epochs": 10, - "optimizer": { - "lr": 0.001 - } - } - } - ] -} -"#; - - assert_eq!(file().to_json(), expected); - assert_eq!( - DefinitionsFile::from_json(expected.as_bytes()).unwrap(), - file() - ); - } - #[test] fn the_jobs_are_ordered_by_name() { let job = |name: &str| JobDefinition { @@ -183,31 +129,12 @@ mod tests { } #[test] - fn only_a_definitions_file_of_this_protocol_parses() { - let parse = |json: &str| DefinitionsFile::from_json(json.as_bytes()); - - assert!(matches!( - parse(r#"{"protocol": 1}"#), - Err(ParseError::Invalid(_)) - )); - assert!(matches!( - parse(r#"{"jobs": []}"#), - Err(ParseError::Invalid(_)) - )); - assert!(matches!( - parse(r#"{"protocol": 1, "jo"#), - Err(ParseError::Invalid(_)) - )); - assert!(matches!( - parse(r#"{"protocol": 2, "jobs": "elsewhere"}"#), - Err(ParseError::Protocol { found: 2 }) - )); - } + fn a_file_of_another_protocol_is_refused_whatever_its_other_fields() { + let other = PROTOCOL + 1; + let document = json!({"protocol": other, "jobs": "elsewhere"}); - #[test] - fn a_protocol_error_names_both_versions() { - let error = DefinitionsFile::from_json(br#"{"protocol": 7}"#).unwrap_err(); + let parsed = DefinitionsFile::from_json(document.to_string().as_bytes()); - assert_eq!(error.to_string(), "protocol 7 is not supported, expected 1"); + assert!(matches!(parsed, Err(ParseError::Protocol { found }) if found == other)); } } From 084645685067d744afaf60d8440a05bb639eb500 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:31:13 -0400 Subject: [PATCH 58/90] test: the flags tests check one rule each, as rstest cases The flags tests listed every flag of one input holding all kinds of fields, in map order, and checked how schemars renames, flattens and escapes what it writes. Each test now checks one rule of the flags an input schema yields, over rstest cases built from small fixture types or hand-written schemas: the kind each type takes, nested names, help, choices, defaults, requirement, references and the keys that have no flag, and how an argument parses and a default renders. --- crates/tracel-job/src/cli/flags.rs | 589 ++++++++++++++--------------- 1 file changed, 292 insertions(+), 297 deletions(-) diff --git a/crates/tracel-job/src/cli/flags.rs b/crates/tracel-job/src/cli/flags.rs index 820e2d88..90a93693 100644 --- a/crates/tracel-job/src/cli/flags.rs +++ b/crates/tracel-job/src/cli/flags.rs @@ -411,14 +411,13 @@ fn description(schema: &Map) -> Option { mod tests { use std::collections::HashMap; - use rstest::rstest; + use rstest::{fixture, rstest}; use schemars::JsonSchema; use serde::Serialize; use serde_json::json; use super::*; - /// The flags of a job whose input schema is `schema` and default input `default`. fn flags_of(schema: Value, default: Option) -> Vec { flags(&JobDefinition { name: "train".to_string(), @@ -428,37 +427,19 @@ mod tests { }) } - /// The flags of a job taking `T`, with no default input. fn flags_for() -> Vec { flags_of(schemars::schema_for!(T).to_value(), None) } - /// The name and kind of each flag. - fn kinds(flags: &[Flag]) -> Vec<(String, Kind)> { - flags - .iter() - .map(|flag| (flag.name(), flag.kind.clone())) - .collect() - } - fn flag<'f>(flags: &'f [Flag], name: &str) -> &'f Flag { flags .iter() .find(|flag| flag.name() == name) - .unwrap_or_else(|| panic!("no flag --{name} in {:?}", kinds(flags))) + .unwrap_or_else(|| panic!("no flag --{name}")) } - fn choices(values: &[(&str, Option<&str>)]) -> Kind { - Kind::Choice( - values - .iter() - .map(|(value, help)| Choice { - text: value.to_string(), - value: json!(value), - help: help.map(str::to_string), - }) - .collect(), - ) + fn kind_of() -> Kind { + flag(&flags_for::>(), "field").kind.clone() } fn one_of(values: &[Value]) -> Kind { @@ -471,41 +452,27 @@ mod tests { } #[derive(JsonSchema)] - #[serde(rename_all = "camelCase")] #[expect(dead_code)] - struct Shapes { - epochs: u32, - offset: i64, - lr: f64, - decay: f32, - shuffle: bool, - tag: String, - letter: char, - #[serde(rename = "seed_value")] - seed: Option, - label: Option, + struct Field { + field: T, + } + + #[derive(JsonSchema)] + #[expect(dead_code)] + struct Nesting { optimizer: Nested, - resume_from: Option, - layers: Vec, - labels: HashMap, - pair: (u32, String), - anything: serde_json::Value, - nothing: (), - tagged: Tagged, - untagged: Untagged, - mixed: Mixed, - #[serde(flatten)] - flattened: Flattened, } - #[derive(Serialize, JsonSchema)] + #[derive(JsonSchema)] + #[expect(dead_code)] struct Nested { lr: f64, weight_decay: f64, schedule: Schedule, } - #[derive(Serialize, JsonSchema)] + #[derive(JsonSchema)] + #[expect(dead_code)] struct Schedule { warmup: u32, } @@ -539,46 +506,56 @@ mod tests { Data(u32), } - #[derive(JsonSchema)] - #[expect(dead_code)] - struct Flattened { - workers: u32, + #[rstest] + #[case::unsigned_integer(kind_of::(), Kind::Integer)] + #[case::signed_integer(kind_of::(), Kind::Integer)] + #[case::double(kind_of::(), Kind::Number)] + #[case::single(kind_of::(), Kind::Number)] + #[case::boolean(kind_of::(), Kind::Bool)] + #[case::string(kind_of::(), Kind::String)] + #[case::character(kind_of::(), Kind::String)] + fn a_scalar_field_takes_the_values_of_its_type(#[case] kind: Kind, #[case] expected: Kind) { + assert_eq!(kind, expected); + } + + #[rstest] + #[case::integer(kind_of::>(), kind_of::())] + #[case::string(kind_of::>(), kind_of::())] + #[case::unit_enum(kind_of::>(), kind_of::())] + fn an_optional_field_takes_the_values_of_the_type_it_makes_optional( + #[case] optional: Kind, + #[case] plain: Kind, + ) { + assert_eq!(optional, plain); + } + + #[rstest] + #[case::list(kind_of::>())] + #[case::map(kind_of::>())] + #[case::tuple(kind_of::<(u32, String)>())] + #[case::any_json(kind_of::())] + #[case::unit(kind_of::<()>())] + #[case::optional_object(kind_of::>())] + #[case::tagged_enum(kind_of::())] + #[case::untagged_enum(kind_of::())] + #[case::enum_with_data(kind_of::())] + fn a_field_of_any_other_type_takes_json(#[case] kind: Kind) { + assert_eq!(kind, Kind::Json); } #[test] - fn objects_nest_into_dotted_flags_and_other_values_take_json() { - let flags = flags_for::(); + fn a_nested_object_has_a_flag_per_field_named_by_its_keys() { + let mut names: Vec = flags_for::().iter().map(Flag::name).collect(); + names.sort(); assert_eq!( - kinds(&flags), + names, [ - ("anything".to_string(), Kind::Json), - ("decay".to_string(), Kind::Number), - ("epochs".to_string(), Kind::Integer), - ("label".to_string(), Kind::String), - ("labels".to_string(), Kind::Json), - ("layers".to_string(), Kind::Json), - ("letter".to_string(), Kind::String), - ("lr".to_string(), Kind::Number), - ("mixed".to_string(), Kind::Json), - ("nothing".to_string(), Kind::Json), - ("offset".to_string(), Kind::Integer), - ("optimizer.lr".to_string(), Kind::Number), - ("optimizer.schedule.warmup".to_string(), Kind::Integer), - ("optimizer.weight-decay".to_string(), Kind::Number), - ("pair".to_string(), Kind::Json), - ("resumeFrom".to_string(), Kind::Json), - ("seed-value".to_string(), Kind::Integer), - ("shuffle".to_string(), Kind::Bool), - ("tag".to_string(), Kind::String), - ("tagged".to_string(), Kind::Json), - ("untagged".to_string(), Kind::Json), - ("workers".to_string(), Kind::Integer), + "optimizer.lr", + "optimizer.schedule.warmup", + "optimizer.weight-decay" ] ); - let weight_decay = flag(&flags, "optimizer.weight-decay"); - assert_eq!(weight_decay.path, ["optimizer", "weight_decay"]); - assert_eq!(weight_decay.id(), "optimizer.weight_decay"); } #[test] @@ -596,62 +573,67 @@ mod tests { assert_eq!(flag(&flags_of(schema, None), "resume").kind, Kind::Json); } - /// How to train. #[derive(JsonSchema)] #[expect(dead_code)] struct Documented { - /// Passes over the data. + #[schemars(description = "Passes over the data.")] epochs: u32, - /// Where to train. - device: Device, backend: Backend, + #[schemars(description = "The backend to compare with.")] + baseline: Backend, fallback: Option, - /// The backend to compare with. - baseline: Option, - optimizer: Nested, + seed: u64, + } + + #[derive(JsonSchema)] + #[schemars(description = "The tensor backend.")] + #[expect(dead_code)] + enum Backend { + Wgpu, + Cuda, } #[derive(JsonSchema)] - #[serde(rename_all = "snake_case")] #[expect(dead_code)] enum Device { - /// The CPU. + #[schemars(description = "The CPU.")] Cpu, - /// The first GPU. + #[schemars(description = "The first GPU.")] Gpu, } - /// The tensor backend. - #[derive(JsonSchema)] - #[expect(dead_code)] - enum Backend { - Wgpu, - Cuda, + #[rstest] + #[case::its_own_description("epochs", Some("Passes over the data."))] + #[case::the_description_of_its_type("backend", Some("The tensor backend."))] + #[case::its_own_over_its_types("baseline", Some("The backend to compare with."))] + #[case::that_of_the_type_it_makes_optional("fallback", Some("The tensor backend."))] + #[case::none_when_nothing_describes_it("seed", None)] + fn a_flag_is_helped_by_its_fields_description_or_else_its_types( + #[case] name: &str, + #[case] help: Option<&str>, + ) { + assert_eq!(flag(&flags_for::(), name).help.as_deref(), help); } - #[test] - fn doc_comments_are_help_and_unit_variants_are_choices() { - let flags = flags_for::(); + #[rstest] + #[case::described( + kind_of::(), + &[("Cpu", Some("The CPU.")), ("Gpu", Some("The first GPU."))] + )] + #[case::undescribed(kind_of::(), &[("Wgpu", None), ("Cuda", None)])] + fn the_unit_variants_of_an_enum_are_choices_with_their_descriptions( + #[case] kind: Kind, + #[case] expected: &[(&str, Option<&str>)], + ) { + let Kind::Choice(choices) = &kind else { + panic!("{kind:?} is not a choice"); + }; + let choices: Vec<(&str, Option<&str>)> = choices + .iter() + .map(|choice| (choice.text.as_str(), choice.help.as_deref())) + .collect(); - assert_eq!( - flag(&flags, "device").kind, - choices(&[("cpu", Some("The CPU.")), ("gpu", Some("The first GPU."))]) - ); - assert_eq!( - flag(&flags, "backend").kind, - choices(&[("Wgpu", None), ("Cuda", None)]) - ); - assert_eq!(flag(&flags, "fallback").kind, flag(&flags, "backend").kind); - for (name, help) in [ - ("epochs", Some("Passes over the data.")), - ("device", Some("Where to train.")), - ("backend", Some("The tensor backend.")), - ("fallback", Some("The tensor backend.")), - ("baseline", Some("The backend to compare with.")), - ("optimizer.lr", None), - ] { - assert_eq!(flag(&flags, name).help.as_deref(), help, "--{name}"); - } + assert_eq!(choices, expected); } #[rstest] @@ -673,17 +655,14 @@ mod tests { assert_eq!(flag(&flags_of(schema, None), "field").kind, Kind::Json); } - #[derive(Serialize, JsonSchema)] + #[derive(JsonSchema)] + #[expect(dead_code)] struct Defaulted { epochs: u32, - #[serde(default)] - workers: u32, #[serde(default = "seed")] seed: u64, #[serde(default)] schedule: Scheduled, - optimizer: Nested, - tag: Option, } #[derive(Serialize, JsonSchema)] @@ -706,102 +685,110 @@ mod tests { 42 } - #[test] - fn defaults_come_from_the_default_input_or_else_the_schema() { - let schema = schemars::schema_for!(Defaulted).to_value(); - let default = |flags: &[Flag], name| flag(flags, name).default.clone(); - - let without = flags_of(schema.clone(), None); - let with = flags_of( - schema, - Some(json!({"epochs": 10, "seed": 7, "schedule": {"warmup": 1}, "optimizer": {}})), + #[rstest] + #[case::without_a_schema_default("epochs")] + #[case::over_the_schema_default("seed")] + fn a_flag_defaults_to_its_fields_value_in_the_default_input(#[case] name: &str) { + let default_input = json!({"epochs": 10, "seed": 7}); + + let flags = flags_of( + schemars::schema_for!(Defaulted).to_value(), + Some(default_input.clone()), ); - assert_eq!(default(&without, "epochs"), None); - assert_eq!(default(&without, "workers"), Some(json!(0))); - assert_eq!(default(&without, "seed"), Some(json!(42))); - assert_eq!(default(&without, "schedule.warmup"), Some(json!(5))); - assert_eq!(default(&without, "schedule.cooldown"), Some(json!(2))); - assert_eq!(default(&with, "epochs"), Some(json!(10))); - assert_eq!(default(&with, "seed"), Some(json!(7))); - assert_eq!(default(&with, "schedule.warmup"), Some(json!(1))); - assert_eq!(default(&with, "schedule.cooldown"), Some(json!(2))); + assert_eq!(flag(&flags, name).default.as_ref(), default_input.get(name)); } - #[test] - fn a_field_the_schema_requires_is_required_unless_a_default_gives_it() { - let schema = schemars::schema_for!(Defaulted).to_value(); - let required = |flags: Vec| -> Vec { - flags - .iter() - .filter(|flag| flag.required) - .map(Flag::name) - .collect() - }; + #[rstest] + #[case::of_a_field("seed", None, json!(seed()))] + #[case::of_a_nested_field("schedule.warmup", None, json!(Scheduled::default().warmup))] + #[case::of_a_nested_field_the_default_input_leaves_out( + "schedule.cooldown", + Some(json!({"schedule": {"warmup": 1}})), + json!(Scheduled::default().cooldown) + )] + fn a_flag_the_default_input_does_not_give_defaults_to_the_schema_default( + #[case] name: &str, + #[case] default_input: Option, + #[case] default: Value, + ) { + let flags = flags_of(schemars::schema_for!(Defaulted).to_value(), default_input); - assert_eq!( - required(flags_of(schema.clone(), None)), - [ - "epochs", - "optimizer.lr", - "optimizer.schedule.warmup", - "optimizer.weight-decay" - ] - ); - assert_eq!( - required(flags_of( - schema, - Some(json!({"epochs": 10, "optimizer": {"lr": 0.1, "schedule": {"warmup": 1}}})) - )), - ["optimizer.weight-decay"] - ); + assert_eq!(flag(&flags, name).default, Some(default)); } - #[derive(JsonSchema)] - #[expect(dead_code)] - struct Node { - value: u32, - next: Option>, - children: Vec, + #[fixture] + fn requirements() -> Value { + json!({ + "type": "object", + "required": ["epochs", "seed", "optimizer"], + "properties": { + "epochs": {"type": "integer"}, + "seed": {"type": "integer", "default": 42}, + "tag": {"type": "string"}, + "optimizer": { + "type": "object", + "required": ["lr"], + "properties": {"lr": {"type": "number"}} + }, + "resume": { + "type": "object", + "required": ["epoch"], + "properties": {"epoch": {"type": "integer"}} + } + } + }) + } + + #[rstest] + #[case::required("epochs", true)] + #[case::required_in_a_required_object("optimizer.lr", true)] + #[case::required_with_a_default("seed", false)] + #[case::optional("tag", false)] + #[case::required_in_an_optional_object("resume.epoch", false)] + fn a_flag_is_required_when_the_schema_requires_its_field_and_gives_no_default( + requirements: Value, + #[case] name: &str, + #[case] required: bool, + ) { + assert_eq!(flag(&flags_of(requirements, None), name).required, required); + } + + #[rstest] + fn a_required_field_the_default_input_gives_is_not_a_required_flag(requirements: Value) { + let flags = flags_of(requirements, Some(json!({"epochs": 10}))); + + assert!(!flag(&flags, "epochs").required); } #[test] - fn an_input_that_is_not_an_object_with_fields_has_no_flags() { - assert!(flags_for::().is_empty()); - assert!(flags_for::<()>().is_empty()); - assert!(flags_for::>().is_empty()); - assert!(flags_for::().is_empty()); - assert!(flags_for::>().is_empty()); - assert!(flags_for::().is_empty()); - assert!(flags_of(json!({"type": "object", "properties": {}}), None).is_empty()); - assert!( - flags(&JobDefinition { - name: "train".to_string(), - description: None, - input_schema: None, - input_example: Some(json!({"epochs": 10})), - }) - .is_empty() - ); + fn a_job_without_an_input_schema_has_no_flags_whatever_its_default_input() { + let job = JobDefinition { + name: "train".to_string(), + description: None, + input_schema: None, + input_example: Some(json!({"epochs": 10})), + }; + + assert!(flags(&job).is_empty()); + } + + #[rstest] + #[case::of_an_integer(schemars::schema_for!(u32).to_value())] + #[case::of_a_list(schemars::schema_for!(Vec).to_value())] + #[case::of_a_unit_enum(schemars::schema_for!(Backend).to_value())] + #[case::of_an_optional_object(schemars::schema_for!(Option).to_value())] + #[case::of_any_json(schemars::schema_for!(Value).to_value())] + #[case::of_an_object_without_fields(json!({"type": "object", "properties": {}}))] + fn an_input_that_is_not_an_object_with_fields_has_no_flags(#[case] schema: Value) { + assert!(flags_of(schema, None).is_empty()); } #[test] fn a_field_that_refers_back_to_an_object_it_is_in_takes_json() { - assert_eq!( - kinds(&flags_for::()), - [ - ("children".to_string(), Kind::Json), - ("next".to_string(), Kind::Json), - ("value".to_string(), Kind::Integer), - ] - ); - assert_eq!( - kinds(&flags_of( - json!({"type": "object", "properties": {"node": {"$ref": "#"}}}), - None - )), - [("node".to_string(), Kind::Json)] - ); + let schema = json!({"type": "object", "properties": {"node": {"$ref": "#"}}}); + + assert_eq!(flag(&flags_of(schema, None), "node").kind, Kind::Json); } #[rstest] @@ -840,115 +827,123 @@ mod tests { assert_eq!(flag(&flags_of(schema, None), "epochs").kind, Kind::Integer); } - #[derive(JsonSchema)] - #[expect(dead_code)] - struct Escaped { - odd: Odd, - slash: Slash, + #[rstest] + #[case::into_defs("$defs")] + #[case::into_definitions("definitions")] + fn a_field_takes_the_values_of_the_schema_its_reference_resolves_to(#[case] definitions: &str) { + let schema = json!({ + "type": "object", + "properties": {"epochs": {"$ref": format!("#/{definitions}/Epochs")}}, + definitions: {"Epochs": {"type": "integer"}} + }); + + assert_eq!(flag(&flags_of(schema, None), "epochs").kind, Kind::Integer); } - #[derive(JsonSchema)] - #[schemars(rename = "Odd name%")] - #[expect(dead_code)] - struct Odd { - a: u32, + #[rstest] + #[case::reserved_for_help("help")] + #[case::reserved_for_the_config_file("config")] + #[case::with_a_dot("a.b")] + #[case::with_a_space("a b")] + #[case::starting_with_a_dash("-x")] + #[case::empty("")] + fn a_field_whose_key_cannot_name_a_flag_has_none(#[case] key: &str) { + let schema = json!({"type": "object", "properties": {key: {"type": "integer"}}}); + + assert!(flags_of(schema, None).is_empty()); } - #[derive(JsonSchema)] - #[schemars(rename = "a/b~c")] - #[expect(dead_code)] - struct Slash { - b: bool, + #[rstest] + #[case::as_the_object("config", "path")] + #[case::as_the_field("optimizer", "config")] + fn a_reserved_name_is_reserved_only_as_a_whole_flag_name( + #[case] object: &str, + #[case] field: &str, + ) { + let schema = json!({ + "type": "object", + "properties": {object: {"type": "object", "properties": {field: {"type": "integer"}}}} + }); + + let names: Vec = flags_of(schema, None).iter().map(Flag::name).collect(); + + assert_eq!(names, [format!("{object}.{field}")]); } #[test] - fn references_resolve_as_schemars_escapes_them() { - let schema = schemars::schema_for!(Escaped).to_value(); - assert_eq!(schema["properties"]["odd"]["$ref"], "#/$defs/Odd%20name%25"); - assert_eq!(schema["properties"]["slash"]["$ref"], "#/$defs/a~1b~0c"); + fn a_flag_name_an_earlier_field_took_is_not_given_again() { + let schema = json!({ + "type": "object", + "properties": {"batch-size": {"type": "integer"}, "batch_size": {"type": "integer"}} + }); - assert_eq!( - kinds(&flags_of(schema, None)), - [ - ("odd.a".to_string(), Kind::Integer), - ("slash.b".to_string(), Kind::Bool), - ] - ); + assert_eq!(flags_of(schema, None).len(), 1); } - #[test] - fn reserved_unusable_and_taken_names_have_no_flag() { - let integer = json!({"type": "integer"}); - let flags = flags_of( - json!({ - "type": "object", - "properties": { - "help": {"type": "boolean"}, - "version": {"type": "string"}, - "config": {"type": "object", "properties": {"path": {"type": "string"}}}, - "optimizer": {"type": "object", "properties": {"config": integer}}, - "a.b": integer, - "a b": integer, - "-x": integer, - "": integer, - "batch-size": integer, - "batch_size": integer - } - }), - None, - ); + #[rstest] + #[case::a_negative_integer(Kind::Integer, "-3", json!(-3))] + #[case::a_number_in_exponent_form(Kind::Number, "3e-4", json!(3e-4))] + #[case::a_whole_number_kept_whole(Kind::Number, "2", json!(2))] + #[case::a_number_as_rust_reads_it(Kind::Number, ".5", json!(0.5))] + #[case::a_bool(Kind::Bool, "false", json!(false))] + #[case::digits_as_a_string(Kind::String, "12", json!("12"))] + #[case::json(Kind::Json, "[1, 2]", json!([1, 2]))] + #[case::a_choice_as_its_value(one_of(&[json!("adam"), json!(2)]), "2", json!(2))] + fn an_argument_parses_as_the_value_its_kind_reads( + #[case] kind: Kind, + #[case] argument: &str, + #[case] value: Value, + ) { + assert_eq!(kind.parse(argument), Ok(value)); + } - assert_eq!( - kinds(&flags), - [ - ("batch-size".to_string(), Kind::Integer), - ("config.path".to_string(), Kind::String), - ("optimizer.config".to_string(), Kind::Integer), - ("version".to_string(), Kind::String), - ] - ); + #[rstest] + #[case::a_fraction_as_an_integer(Kind::Integer, "1.5")] + #[case::a_word_as_an_integer(Kind::Integer, "ten")] + #[case::infinity_as_a_number(Kind::Number, "inf")] + #[case::yes_as_a_bool(Kind::Bool, "yes")] + #[case::json_cut_short(Kind::Json, "[1,")] + #[case::a_value_no_choice_gives(one_of(&[json!("adam")]), "sgd")] + fn an_argument_its_kind_does_not_read_is_rejected(#[case] kind: Kind, #[case] argument: &str) { + assert!(kind.parse(argument).is_err()); } - #[test] - fn arguments_parse_as_the_kind_of_their_field() { - assert_eq!(Kind::Integer.parse("-3"), Ok(json!(-3))); - assert!(Kind::Integer.parse("1.5").is_err()); - assert!(Kind::Integer.parse("ten").is_err()); - assert_eq!(Kind::Number.parse("3e-4"), Ok(json!(3e-4))); - assert_eq!(Kind::Number.parse("2"), Ok(json!(2))); - assert_eq!(Kind::Number.parse(".5"), Ok(json!(0.5))); - assert!(Kind::Number.parse("inf").is_err()); - assert_eq!(Kind::Bool.parse("false"), Ok(json!(false))); - assert!(Kind::Bool.parse("yes").is_err()); - assert_eq!(Kind::String.parse("12"), Ok(json!("12"))); - assert_eq!(Kind::Json.parse("[1, 2]"), Ok(json!([1, 2]))); - assert!(Kind::Json.parse("[1,").is_err()); - let choice = Kind::Choice(vec![ - Choice::new(&json!("adam"), None).unwrap(), - Choice::new(&json!(2), None).unwrap(), - ]); - assert_eq!(choice.parse("2"), Ok(json!(2))); - assert!(choice.parse("sgd").is_err()); + #[rstest] + #[case::a_number(Kind::Number, json!(0.001))] + #[case::a_whole_number(Kind::Number, json!(10))] + #[case::an_integer(Kind::Integer, json!(64))] + #[case::a_string(Kind::String, json!("baseline"))] + #[case::a_bool(Kind::Bool, json!(true))] + #[case::json(Kind::Json, json!({"a": [1]}))] + #[case::a_choice(one_of(&[json!("adam")]), json!("adam"))] + fn a_default_renders_as_an_argument_that_parses_back_to_it( + #[case] kind: Kind, + #[case] default: Value, + ) { + let argument = kind.render(&default).unwrap(); + + assert_eq!(kind.parse(&argument), Ok(default)); } #[test] - fn defaults_render_as_arguments_their_flag_takes() { - assert_eq!( - Kind::Number.render(&json!(5e-5f32)).as_deref(), - Some("0.00005") - ); - assert_eq!(Kind::Number.render(&json!(0.001)).as_deref(), Some("0.001")); - assert_eq!(Kind::Number.render(&json!(10)).as_deref(), Some("10")); - assert_eq!(Kind::Integer.render(&json!(64)).as_deref(), Some("64")); + fn a_default_widened_from_an_f32_renders_as_the_f32() { + let single = 5e-5f32; + assert_eq!( - Kind::Json.render(&json!({"a": [1]})).as_deref(), - Some(r#"{"a":[1]}"#) + Kind::Number.render(&json!(single)), + Some(single.to_string()) ); - assert_eq!(Kind::Integer.render(&json!(1.5)), None); - assert_eq!(Kind::String.render(&json!(3)), None); - assert_eq!(Kind::Number.render(&Value::Null), None); - let choice = Kind::Choice(vec![Choice::new(&json!("adam"), None).unwrap()]); - assert_eq!(choice.render(&json!("adam")).as_deref(), Some("adam")); - assert_eq!(choice.render(&json!("sgd")), None); + } + + #[rstest] + #[case::a_fraction_as_an_integer(Kind::Integer, json!(1.5))] + #[case::a_number_as_a_string(Kind::String, json!(3))] + #[case::null_as_a_number(Kind::Number, Value::Null)] + #[case::a_value_no_choice_gives(one_of(&[json!("adam")]), json!("sgd"))] + fn a_default_its_flag_does_not_take_renders_as_nothing( + #[case] kind: Kind, + #[case] default: Value, + ) { + assert_eq!(kind.render(&default), None); } } From 38b64786cd2163666f43cf79934bd42aeda48ea4 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:29:36 -0400 Subject: [PATCH 59/90] test: target tests assert the target or error each variable gives The target tests checked several values in one test through loops and matched words of the error messages. Each rule is now one rstest test with a case per value: the default runs directory, an unknown target, the station URL and an offline target serving no registry, each asserting the Target or TargetError variant and its fields, with the defaults taken from DEFAULT_RUNS_DIR and DEFAULT_STATION_URL. The station URL case that relied on how the url crate reads localhost:8000 goes. --- crates/tracel/src/target.rs | 161 ++++++++++++++---------------------- 1 file changed, 61 insertions(+), 100 deletions(-) diff --git a/crates/tracel/src/target.rs b/crates/tracel/src/target.rs index 70256c86..692a61e6 100644 --- a/crates/tracel/src/target.rs +++ b/crates/tracel/src/target.rs @@ -257,6 +257,7 @@ pub enum TargetError { #[cfg(test)] mod tests { + use rstest::rstest; use tracel_console::ProjectRef; use super::*; @@ -274,111 +275,76 @@ mod tests { ) } - fn offline_dir(target: Result) -> PathBuf { - match target { - Ok(Target::Offline { dir }) => dir, - other => panic!("expected an offline target, got {other:?}"), - } - } - - #[test] - fn an_unset_or_empty_target_records_offline_under_runs() { - assert_eq!(offline_dir(target(&[])), Path::new("./runs")); - assert_eq!( - offline_dir(target(&[(TRACEL_TARGET, "")])), - Path::new("./runs") - ); - assert_eq!( - offline_dir(target(&[(TRACEL_TARGET, "offline")])), - Path::new("./runs") - ); + #[rstest] + #[case::unset(&[])] + #[case::empty(&[(TRACEL_TARGET, "")])] + #[case::offline(&[(TRACEL_TARGET, "offline")])] + #[case::runs_dir_empty(&[(TRACEL_RUNS_DIR, "")])] + fn an_offline_target_records_under_the_default_runs_dir(#[case] vars: &[(&str, &str)]) { + assert!(matches!( + target(vars), + Ok(Target::Offline { dir }) if dir == Path::new(DEFAULT_RUNS_DIR) + )); } #[test] fn tracel_runs_dir_moves_offline_runs() { - assert_eq!( - offline_dir(target(&[(TRACEL_RUNS_DIR, "/data/runs")])), - Path::new("/data/runs") - ); - assert_eq!( - offline_dir(target(&[(TRACEL_RUNS_DIR, "")])), - Path::new("./runs") - ); + let runs = "/data/runs"; + + assert!(matches!( + target(&[(TRACEL_RUNS_DIR, runs)]), + Ok(Target::Offline { dir }) if dir == Path::new(runs) + )); } - #[test] - fn an_unknown_target_names_the_value_and_the_accepted_ones() { - for value in ["cloud", "Offline", "CONSOLE", " console"] { - let error = target(&[(TRACEL_TARGET, value)]).unwrap_err(); - - assert!( - matches!( - &error, - TargetError::UnknownTarget { value: held, .. } if held == value - ), - "{value}: {error:?}" - ); - let message = error.to_string(); - assert!(message.contains("TRACEL_TARGET"), "{message}"); - assert!(message.contains(value), "{message}"); - assert!(message.contains("offline"), "{message}"); - assert!(message.contains("console"), "{message}"); - } + #[rstest] + #[case::unknown("cloud")] + #[case::capitalized("Offline")] + #[case::upper_case("CONSOLE")] + #[case::padded(" console")] + fn an_unknown_target_is_an_error_naming_it(#[case] value: &str) { + assert!(matches!( + target(&[(TRACEL_TARGET, value)]), + Err(TargetError::UnknownTarget { value: held, .. }) if held == value + )); } #[cfg(not(feature = "station"))] #[test] - fn a_station_target_says_it_needs_the_station_feature() { - let error = target(&[(TRACEL_TARGET, "station")]).unwrap_err(); - + fn a_station_target_needs_the_station_feature() { assert!(matches!( - error, - TargetError::FeatureRequired { + target(&[(TRACEL_TARGET, "station")]), + Err(TargetError::FeatureRequired { feature: "station", .. - } + }) )); - assert!(error.to_string().contains("`station` feature"), "{error}"); } #[cfg(feature = "station")] - #[test] - fn a_station_target_reads_tracel_station_url() { - let station_url = |vars: &[(&str, &str)]| match target(vars) { - Ok(Target::Station { url }) => url.to_string(), - other => panic!("expected a station target, got {other:?}"), - }; - - assert_eq!( - station_url(&[(TRACEL_TARGET, "station")]), - "http://localhost:8000/" - ); - assert_eq!( - station_url(&[ - (TRACEL_TARGET, "station"), - (TRACEL_STATION_URL, "https://station.example.com") - ]), - "https://station.example.com/" - ); + #[rstest] + #[case::unset(&[(TRACEL_TARGET, "station")], DEFAULT_STATION_URL)] + #[case::set( + &[(TRACEL_TARGET, "station"), (TRACEL_STATION_URL, "https://station.example.com")], + "https://station.example.com" + )] + fn tracel_station_url_names_the_station(#[case] vars: &[(&str, &str)], #[case] url: &str) { + assert!(matches!( + target(vars), + Ok(Target::Station { url: held }) if held == Url::parse(url).unwrap() + )); } #[cfg(feature = "station")] - #[test] - fn an_invalid_station_url_names_the_variable() { - for value in ["not a url", "localhost:8000", "ftp://localhost:8000"] { - let error = - target(&[(TRACEL_TARGET, "station"), (TRACEL_STATION_URL, value)]).unwrap_err(); - - assert!( - matches!( - &error, - TargetError::InvalidSetting { variable: "TRACEL_STATION_URL", value: held, .. } - if held == value - ), - "{value}: {error:?}" - ); - assert!(error.to_string().contains("TRACEL_STATION_URL"), "{error}"); - } + #[rstest] + #[case::not_a_url("not a url")] + #[case::not_http("ftp://localhost:8000")] + fn a_station_url_that_is_not_http_is_an_invalid_setting_naming_it(#[case] value: &str) { + assert!(matches!( + target(&[(TRACEL_TARGET, "station"), (TRACEL_STATION_URL, value)]), + Err(TargetError::InvalidSetting { variable: TRACEL_STATION_URL, value: held, .. }) + if held == value + )); } #[test] @@ -392,25 +358,20 @@ mod tests { assert!(!dir.exists()); } - #[test] - fn an_offline_target_does_not_serve_models_or_datasets() { + #[rstest] + #[case::models("models", |offline: &Target| offline.models().err())] + #[case::datasets("datasets", |offline: &Target| offline.datasets().err())] + fn an_offline_target_serves_no_registry( + #[case] service: &str, + #[case] build: fn(&Target) -> Option, + ) { let offline = Target::Offline { - dir: PathBuf::from("./runs"), + dir: PathBuf::from(DEFAULT_RUNS_DIR), }; assert!(matches!( - offline.models(), - Err(TargetError::Unsupported { - service: "models", - target: "offline" - }) - )); - assert!(matches!( - offline.datasets(), - Err(TargetError::Unsupported { - service: "datasets", - target: "offline" - }) + build(&offline), + Some(TargetError::Unsupported { service: held, target: "offline" }) if held == service )); } From 94a47e1b1c22e8862e127114d361b867e43bd663 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:30:10 -0400 Subject: [PATCH 60/90] fix: a station target's services share one station Target::Station held the Station's URL, and each experiments(), models() or datasets() call built a new Station from it, each with its own client and transfer client. Target::Station now holds the Station that Target::from_env builds, as Target::Console holds its project, so every service reuses it. Reading TRACEL_STATION_URL moves into station_url, which the URL tests now call, since the Station does not give its URL back. --- crates/tracel/src/target.rs | 75 ++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/crates/tracel/src/target.rs b/crates/tracel/src/target.rs index 692a61e6..19976b80 100644 --- a/crates/tracel/src/target.rs +++ b/crates/tracel/src/target.rs @@ -49,11 +49,12 @@ pub enum Target { /// The project experiments, models and datasets belong to. project: ProjectHandle, }, - /// A Tracel Station (requires the `station` feature). + /// A Tracel Station (requires the `station` feature). The services the target builds share + /// its client. #[cfg(feature = "station")] Station { - /// The Station's base URL. - url: Url, + /// The Station experiments, models and datasets belong to. + station: Station, }, } @@ -112,22 +113,9 @@ impl Target { project: ProjectHandle::from_vars(&lookup, dir, sessions)?, }), #[cfg(feature = "station")] - Some("station") => { - let url = var(TRACEL_STATION_URL).map_or_else( - || DEFAULT_STATION_URL.to_string(), - |value| value.to_string_lossy().into_owned(), - ); - match Url::parse(&url) { - Ok(url) if matches!(url.scheme(), "http" | "https") => { - Ok(Target::Station { url }) - } - _ => Err(TargetError::InvalidSetting { - variable: TRACEL_STATION_URL, - value: url, - expected: "an http or https URL, such as http://localhost:8000", - }), - } - } + Some("station") => Ok(Target::Station { + station: Station::connect(station_url(var(TRACEL_STATION_URL))?), + }), #[cfg(not(feature = "station"))] Some("station") => Err(TargetError::FeatureRequired { target: "station", @@ -158,7 +146,7 @@ impl Target { Target::Offline { dir } => Experiments::new(Arc::new(LocalExperiments::new(dir))), Target::Console { project } => project.experiments(), #[cfg(feature = "station")] - Target::Station { url } => Station::connect(url.clone()).experiments(), + Target::Station { station } => station.experiments(), }; Ok(running_job::linked(experiments)) } @@ -174,7 +162,7 @@ impl Target { }), Target::Console { project } => Ok(project.models()), #[cfg(feature = "station")] - Target::Station { url } => Ok(Station::connect(url.clone()).models()), + Target::Station { station } => Ok(station.models()), } } @@ -189,7 +177,7 @@ impl Target { }), Target::Console { project } => Ok(project.datasets()), #[cfg(feature = "station")] - Target::Station { url } => Ok(Station::connect(url.clone()).datasets()), + Target::Station { station } => Ok(station.datasets()), } } @@ -208,6 +196,25 @@ impl Target { } } +/// The Station URL `value` names, `http://localhost:8000` when it is unset. +/// +/// Fails with [`TargetError::InvalidSetting`] when it is not an http or https URL. +#[cfg(feature = "station")] +fn station_url(value: Option) -> Result { + let url = value.map_or_else( + || DEFAULT_STATION_URL.to_string(), + |value| value.to_string_lossy().into_owned(), + ); + match Url::parse(&url) { + Ok(url) if matches!(url.scheme(), "http" | "https") => Ok(url), + _ => Err(TargetError::InvalidSetting { + variable: TRACEL_STATION_URL, + value: url, + expected: "an http or https URL, such as http://localhost:8000", + }), + } +} + fn noop_inference() -> InferenceModule { InferenceModule::new(Arc::new(NoopInferenceProvider::new())) } @@ -322,19 +329,25 @@ mod tests { } #[cfg(feature = "station")] - #[rstest] - #[case::unset(&[(TRACEL_TARGET, "station")], DEFAULT_STATION_URL)] - #[case::set( - &[(TRACEL_TARGET, "station"), (TRACEL_STATION_URL, "https://station.example.com")], - "https://station.example.com" - )] - fn tracel_station_url_names_the_station(#[case] vars: &[(&str, &str)], #[case] url: &str) { + #[test] + fn tracel_target_station_reaches_a_station() { assert!(matches!( - target(vars), - Ok(Target::Station { url: held }) if held == Url::parse(url).unwrap() + target(&[(TRACEL_TARGET, "station")]), + Ok(Target::Station { .. }) )); } + #[cfg(feature = "station")] + #[rstest] + #[case::unset(None, DEFAULT_STATION_URL)] + #[case::set(Some("https://station.example.com"), "https://station.example.com")] + fn tracel_station_url_names_the_station(#[case] value: Option<&str>, #[case] url: &str) { + assert_eq!( + station_url(value.map(OsString::from)).unwrap(), + Url::parse(url).unwrap() + ); + } + #[cfg(feature = "station")] #[rstest] #[case::not_a_url("not a url")] From ae1d380fdbbc4c1edb14bb0eeba5df76bfa481b1 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:31:00 -0400 Subject: [PATCH 61/90] test: console settings tests assert outcomes, not messages The project tests listed several cases in one test, matched words of the missing namespace and project errors, and checked that tracel.toml keys other than namespace and project are ignored, which is serde's doing. The TRACEL_ENV tests looped over values and matched words of the message. Each rule is now one rstest test with a case per value, asserting the ProjectRef, the Env or the ConsoleError variant and its fields, and an empty TRACEL_NAMESPACE is a case of tracel.toml filling in what the environment leaves unset. --- crates/tracel-console/src/env.rs | 171 +++++++++++++------------------ 1 file changed, 72 insertions(+), 99 deletions(-) diff --git a/crates/tracel-console/src/env.rs b/crates/tracel-console/src/env.rs index 238c90a4..c9cf7700 100644 --- a/crates/tracel-console/src/env.rs +++ b/crates/tracel-console/src/env.rs @@ -236,49 +236,58 @@ mod tests { ProjectRef::from_vars(lookup, dir.path()) } - #[test] - fn the_environment_names_the_project_over_tracel_toml() { - let file = "namespace = \"file-owner\"\nproject = \"file-project\""; + const FILE_NAMESPACE: &str = "file-owner"; + const FILE_PROJECT: &str = "file-project"; + #[fixture] + fn tracel_toml() -> String { + format!("namespace = \"{FILE_NAMESPACE}\"\nproject = \"{FILE_PROJECT}\"") + } + + #[rstest] + fn the_environment_names_the_project_over_tracel_toml(tracel_toml: String) { assert_eq!( - located(Some("env-owner"), Some("env-project"), file).unwrap(), + located(Some("env-owner"), Some("env-project"), &tracel_toml).unwrap(), ProjectRef::new("env-owner", "env-project") ); } - #[test] - fn tracel_toml_fills_in_what_the_environment_leaves_unset() { - let file = "namespace = \"file-owner\"\nproject = \"file-project\""; + #[rstest] + #[case::namespace_unset( + None, + Some("env-project"), + ProjectRef::new(FILE_NAMESPACE, "env-project") + )] + #[case::namespace_empty( + Some(""), + Some("env-project"), + ProjectRef::new(FILE_NAMESPACE, "env-project") + )] + #[case::project_unset(Some("env-owner"), None, ProjectRef::new("env-owner", FILE_PROJECT))] + #[case::both_unset(None, None, ProjectRef::new(FILE_NAMESPACE, FILE_PROJECT))] + fn tracel_toml_fills_in_what_the_environment_leaves_unset( + tracel_toml: String, + #[case] namespace: Option<&str>, + #[case] project: Option<&str>, + #[case] expected: ProjectRef, + ) { + assert_eq!(located(namespace, project, &tracel_toml).unwrap(), expected); + } - assert_eq!( - located(Some("env-owner"), None, file).unwrap(), - ProjectRef::new("env-owner", "file-project") - ); - assert_eq!( - located(None, Some("env-project"), file).unwrap(), - ProjectRef::new("file-owner", "env-project") - ); - assert_eq!( - located(None, None, file).unwrap(), - ProjectRef::new("file-owner", "file-project") - ); + #[test] + fn no_namespace_in_the_environment_or_tracel_toml_is_an_error() { + assert!(matches!( + located(None, Some("env-project"), ""), + Err(ConsoleError::NoNamespace) + )); } #[test] - fn tracel_toml_names_the_project_only_by_namespace_and_project() { - let file = "name = \"file-project\"\nowner = \"file-owner\""; - - let no_namespace = located(None, None, file).unwrap_err(); - let no_project = located(Some("env-owner"), None, file).unwrap_err(); - - assert!(matches!(no_namespace, ConsoleError::NoNamespace)); - assert!(matches!(no_project, ConsoleError::NoProject)); - for error in [no_namespace, no_project] { - assert!( - error.to_string().contains("`namespace` and `project`"), - "{error}" - ); - } + fn no_project_in_the_environment_or_tracel_toml_is_an_error() { + assert!(matches!( + located(Some("env-owner"), None, ""), + Err(ConsoleError::NoProject) + )); } #[test] @@ -313,19 +322,6 @@ mod tests { ); } - #[test] - fn a_missing_namespace_or_project_says_where_to_set_it() { - let no_namespace = located(None, Some("env-project"), "").unwrap_err(); - let no_project = located(Some("env-owner"), None, "").unwrap_err(); - - assert!(matches!(no_namespace, ConsoleError::NoNamespace)); - assert!(no_namespace.to_string().contains("TRACEL_NAMESPACE")); - assert!(no_namespace.to_string().contains("tracel.toml")); - assert!(matches!(no_project, ConsoleError::NoProject)); - assert!(no_project.to_string().contains("TRACEL_PROJECT")); - assert!(no_project.to_string().contains("tracel.toml")); - } - fn api_key(value: Option<&str>) -> CredentialSource { CredentialSource::from_vars(|name| match name { TRACEL_API_KEY => value.map(OsString::from), @@ -411,62 +407,39 @@ mod tests { )); } - #[test] - fn tracel_env_unset_or_empty_names_production() { - assert!(matches!(parse_env(None), Ok(Env::Production))); - assert!(matches!(parse_env(Some("")), Ok(Env::Production))); + #[rstest] + #[case::unset(None)] + #[case::empty(Some(""))] + fn an_unset_or_empty_tracel_env_names_production(#[case] value: Option<&str>) { + assert!(matches!(parse_env(value), Ok(Env::Production))); } - #[test] - fn tracel_env_accepts_the_spellings_of_the_tracel_cli() { - for value in ["Production", "production"] { - assert!( - matches!(parse_env(Some(value)), Ok(Env::Production)), - "{value}" - ); - } - for value in ["Development", "development"] { - assert!( - matches!(parse_env(Some(value)), Ok(Env::Development)), - "{value}" - ); - } - for value in ["Staging(2)", "staging-2"] { - assert!( - matches!(parse_env(Some(value)), Ok(Env::Staging(2))), - "{value}" - ); - } + #[rstest] + #[case::production("Production", Env::Production)] + #[case::production_as_the_cli_spells_it("production", Env::Production)] + #[case::development("Development", Env::Development)] + #[case::development_as_the_cli_spells_it("development", Env::Development)] + #[case::staging("Staging(2)", Env::Staging(2))] + #[case::staging_as_the_cli_spells_it("staging-2", Env::Staging(2))] + fn tracel_env_accepts_the_spellings_of_the_tracel_cli(#[case] value: &str, #[case] env: Env) { + assert_eq!(parse_env(Some(value)).unwrap().get_url(), env.get_url()); } - #[test] - fn an_unrecognized_tracel_env_is_an_error_naming_the_variable_and_value() { - for value in [ - "DEVELOPMENT", - "dev", - " production", - "Staging(x)", - "Staging()", - "Staging(1", - "Staging(256)", - "Staging(+2)", - "staging-", - "staging--1", - "staging-+2", - "staging-256", - ] { - let error = parse_env(Some(value)).unwrap_err(); - - assert!( - matches!( - &error, - ConsoleError::InvalidSetting { variable: "TRACEL_ENV", value: held, .. } - if held == value - ), - "{value}: {error:?}" - ); - assert!(error.to_string().contains("TRACEL_ENV"), "{error}"); - assert!(error.to_string().contains(value), "{error}"); - } + #[rstest] + #[case::wrong_case("DEVELOPMENT")] + #[case::abbreviated("dev")] + #[case::padded(" production")] + #[case::unclosed("Staging(1")] + #[case::without_a_number("staging-")] + #[case::not_a_number("Staging(x)")] + #[case::signed("staging-+2")] + #[case::signed_in_parentheses("Staging(+2)")] + #[case::past_255("Staging(256)")] + fn an_unrecognized_tracel_env_is_an_invalid_setting_naming_it(#[case] value: &str) { + assert!(matches!( + parse_env(Some(value)), + Err(ConsoleError::InvalidSetting { variable: TRACEL_ENV, value: held, .. }) + if held == value + )); } } From df6a50266a4d79d4c6c29d5cd33deb376a9583c6 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:31:20 -0400 Subject: [PATCH 62/90] test: console tests no longer read the connection's private state binding_and_scoping_send_no_request read the console's private OnceLock and compared Arc pointers, against Env::Development, the devstack's port, so a laziness bug would have reached a running devstack instead of failing. It goes, and the console keeps only the test that its handles can be shared across threads. --- crates/tracel-console/src/console.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/crates/tracel-console/src/console.rs b/crates/tracel-console/src/console.rs index 59cea5ee..6b655a80 100644 --- a/crates/tracel-console/src/console.rs +++ b/crates/tracel-console/src/console.rs @@ -407,23 +407,4 @@ mod tests { assert_send_sync::(); assert_send_sync::(); } - - #[test] - fn binding_and_scoping_send_no_request() { - let console = Console::connect( - Env::Development, - TracelCredentials::api_key("tcl_key_not_checked_until_the_first_request"), - ); - let project = console.project("owner", "project"); - let _ = ( - project.datasets(), - project.models(), - project.experiments(), - project.inference(), - ); - - assert!(console.inner.connected.get().is_none()); - assert!(Arc::ptr_eq(&project.console().inner, &console.inner)); - assert_eq!(console.base_url(), &Env::Development.get_url()); - } } From d8d24141444b2fcfa6ccb9718e99924ee8410fc4 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:32:56 -0400 Subject: [PATCH 63/90] test: dataset upload tests check what is sent, not the constants Three tests divided BATCH_BYTES by an item's estimate or subtracted the envelope's bytes by hand, restating the constants rather than checking a rule, and the refusal test never reached add_item. They go. The undercount test looped over every combination of payload, id and extras; it is now an rstest case per shape that stresses the estimate, a large payload among them, so measuring raw bytes still fails it. The contiguous run and paging tests become cases, and paging also covers a page shorter than the total says. --- crates/tracel-console/src/datasets.rs | 194 ++++++++------------------ 1 file changed, 58 insertions(+), 136 deletions(-) diff --git a/crates/tracel-console/src/datasets.rs b/crates/tracel-console/src/datasets.rs index 6065764f..82b5ea50 100644 --- a/crates/tracel-console/src/datasets.rs +++ b/crates/tracel-console/src/datasets.rs @@ -456,143 +456,69 @@ fn map_version_error( #[cfg(test)] mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + use super::*; - #[test] - fn one_batch_of_neighbours_is_one_run() { - assert_eq!(contiguous_runs(&[4, 5, 6]), vec![4..7]); + fn item( + payload: usize, + source_item_id: Option<&str>, + extra: Option, + ) -> DatasetVersionUploadItemRequest { + DatasetVersionUploadItemRequest { + source_item_id: source_item_id.map(str::to_string), + example_payload: vec![7; payload], + annotation: extra.clone(), + metadata: extra, + } } - #[test] - fn a_shuffled_batch_is_sorted_and_coalesced() { - assert_eq!(contiguous_runs(&[9, 1, 8, 0, 2]), vec![0..3, 8..10]); + fn wire_size(value: &impl serde::Serialize) -> usize { + serde_json::to_vec(value).unwrap().len() } - #[test] - fn a_repeated_index_is_asked_for_once() { - assert_eq!(contiguous_runs(&[3, 3, 3]), vec![3..4]); + #[rstest] + #[case::neighbours(&[4, 5, 6], vec![4..7])] + #[case::shuffled(&[9, 1, 8, 0, 2], vec![0..3, 8..10])] + #[case::repeated(&[3, 3, 3], vec![3..4])] + fn indexes_are_asked_for_in_ascending_contiguous_runs( + #[case] indexes: &[u64], + #[case] runs: Vec>, + ) { + assert_eq!(contiguous_runs(indexes), runs); } - #[test] - fn the_estimate_never_undercounts_what_is_actually_sent() { - let payloads = [0usize, 1, 2, 3, 4, 5, 1_000, 1_001]; - let ids = [ - None, - Some(String::new()), - Some("plain".to_string()), - // JSON escapes make the wire form longer than the Rust string. - Some("\"quoted\"".to_string()), - Some("back\\slash".to_string()), - Some("tab\there\nand\nnewlines".to_string()), - Some("\u{1}\u{2}\u{3} control".to_string()), - Some("emoji \u{1F600} and accents \u{e9}\u{e8}".to_string()), - Some("\"".repeat(64)), - ]; - let extras = [ - None, - Some(serde_json::json!(null)), - Some(serde_json::json!({"label": "cat", "nested": [1, 2, 3]})), - Some(serde_json::json!({"quote": "say \"hi\"", "uni": "\u{e9}"})), - ]; - - for payload in payloads { - for id in &ids { - for extra in &extras { - let item = DatasetVersionUploadItemRequest { - source_item_id: id.clone(), - example_payload: vec![7; payload], - annotation: extra.clone(), - metadata: extra.clone(), - }; - - let actual = serde_json::to_vec(&item).expect("an item serializes").len(); - let estimate = encoded_size(&item); - - assert!( - estimate >= actual, - "undercounted by {}: payload={payload} id={id:?} extra={extra:?}", - actual - estimate - ); - } - } - } + #[rstest] + #[case::empty(item(0, None, None))] + #[case::one_byte_past_a_base64_block(item(1, None, None))] + #[case::two_bytes_past_a_base64_block(item(2, None, None))] + #[case::a_whole_base64_block(item(3, None, None))] + #[case::a_large_payload(item(3_000, None, None))] + #[case::an_id_json_escapes(item(0, Some("\"quoted\" back\\slash\ttab\nline \u{1}"), None))] + #[case::an_id_beyond_ascii(item(0, Some("emoji \u{1F600} and accents \u{e9}"), None))] + #[case::null_extras(item(0, None, Some(json!(null))))] + #[case::nested_extras(item(0, None, Some(json!({"quote": "say \"hi\"", "nested": [1, 2, 3]}))))] + fn an_items_estimate_never_undercounts_what_is_sent( + #[case] item: DatasetVersionUploadItemRequest, + ) { + assert!(encoded_size(&item) >= wire_size(&item)); } #[test] fn the_budget_bounds_the_whole_request_body_not_just_the_items() { let items: Vec<_> = (0..500) - .map(|n| DatasetVersionUploadItemRequest { - source_item_id: Some(format!("item-\"{n}\"")), - example_payload: vec![n as u8; n * 7 % 1_000], - annotation: Some(serde_json::json!({ "label": n })), - metadata: None, + .map(|n| { + item( + n * 7 % 1_000, + Some(format!("item-\"{n}\"").as_str()), + Some(json!({ "label": n })), + ) }) .collect(); - let budgeted: usize = items.iter().map(encoded_size).sum(); - let request = AddDatasetVersionUploadItemsRequest { - items: items.clone(), - }; - let actual = serde_json::to_vec(&request) - .expect("a batch serializes") - .len(); - - assert!( - budgeted >= actual, - "a batch of {} items budgeted {budgeted} but sends {actual}", - items.len() - ); - } - #[test] - fn a_payload_is_measured_base64_encoded_not_raw() { - let item = DatasetVersionUploadItemRequest { - source_item_id: None, - example_payload: vec![0; 3_000], - annotation: None, - metadata: None, - }; - - // 3 raw bytes become 4 on the wire, so a batch budgeted on raw sizes overshoots. - assert_eq!(encoded_size(&item) - 96 - 4 - 4 - 4, 4_000); - } - - #[test] - fn a_payload_that_alone_exceeds_the_budget_is_refused() { - let raw = BATCH_BYTES / 4 * 3 + 1; - let item = DatasetVersionUploadItemRequest { - source_item_id: None, - example_payload: vec![0; raw], - annotation: None, - metadata: None, - }; - - assert!(encoded_size(&item) > BATCH_BYTES); - } - - #[test] - fn a_megabyte_item_fills_the_budget_long_before_a_large_count_does() { - let item = DatasetVersionUploadItemRequest { - source_item_id: None, - example_payload: vec![0; 1024 * 1024], - annotation: None, - metadata: None, - }; - - assert!(BATCH_BYTES / encoded_size(&item) < 64); - } - - #[test] - fn small_items_are_not_held_back_by_a_count() { - let item = DatasetVersionUploadItemRequest { - source_item_id: None, - example_payload: vec![0; 1024], - annotation: None, - metadata: None, - }; - - // Bytes alone decide, so a batch holds far more than the 256 it used to. - assert!(BATCH_BYTES / encoded_size(&item) > 10_000); + assert!(budgeted >= wire_size(&AddDatasetVersionUploadItemsRequest { items })); } #[test] @@ -611,22 +537,18 @@ mod tests { ); } - #[test] - fn queries_every_page_needed_to_reach_the_total() { - let mut fetched = Vec::new(); - let mut pages = vec![ - ((0..PAGE_SIZE).collect::>(), u64::from(PAGE_SIZE) + 1), - (vec![PAGE_SIZE], u64::from(PAGE_SIZE) + 1), - ] - .into_iter(); - - let values = collect_pages(|page| { - fetched.push(page); - Ok(pages.next().expect("the test provided this page")) - }) - .unwrap(); + #[rstest] + #[case::one_more_than_a_page(u64::from(PAGE_SIZE) + 1, u64::from(PAGE_SIZE) + 1)] + #[case::fewer_than_the_total_says(u64::from(PAGE_SIZE) + 1, 10 * u64::from(PAGE_SIZE))] + fn a_query_reads_every_item_across_its_pages(#[case] count: u64, #[case] total: u64) { + let all: Vec = (0..count).collect(); + let pages: Vec<&[u64]> = all.chunks(PAGE_SIZE as usize).collect(); + + let read = collect_pages(|page| { + let items = pages.get(page as usize).copied().unwrap_or_default(); + Ok((items.to_vec(), total)) + }); - assert_eq!(fetched, [0, 1]); - assert_eq!(values.len(), PAGE_SIZE as usize + 1); + assert_eq!(read.unwrap(), all); } } From a91beb4bd883af42d1a29205d8112e815b468915 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:33:25 -0400 Subject: [PATCH 64/90] test: inference telemetry tests check what reaches the console The batch test asserted four counts at once, one of them len, which only adds the other three, and the rfc3339 and wire format tests checked what serde writes. The batch test now shows only that a metric descriptor is sent once per name. --- crates/tracel-console/src/inference.rs | 71 +++++++------------------- 1 file changed, 18 insertions(+), 53 deletions(-) diff --git a/crates/tracel-console/src/inference.rs b/crates/tracel-console/src/inference.rs index 76c7d551..109a739f 100644 --- a/crates/tracel-console/src/inference.rs +++ b/crates/tracel-console/src/inference.rs @@ -299,63 +299,28 @@ fn rfc3339(timestamp_ms: i64) -> String { mod tests { use super::*; - #[test] - fn rfc3339_formats_epoch_millis() { - assert_eq!(rfc3339(1000), "1970-01-01T00:00:01.000Z"); + fn descriptor(name: &str) -> TelemetryMsg { + TelemetryMsg::Descriptor(MetricDescriptor { + name: name.to_string(), + kind: MetricKind::Distribution, + unit: Some("ms".to_string()), + description: None, + }) } #[test] - fn converts_metric_sample_to_backend_contract() { - let sample = MetricSample { - name: "inference_duration_ms".to_string(), - timestamp_ms: 1000, - metadata: serde_json::json!({ "request_id": "wordtok/0" }), - data: MetricData::Distribution { value: 5.0 }, - }; - - let json = serde_json::to_value(convert_metric(sample)).unwrap(); - assert_eq!( - json, - serde_json::json!({ - "name": "inference_duration_ms", - "timestamp": "1970-01-01T00:00:01.000Z", - "metadata": { "request_id": "wordtok/0" }, - "kind": "distribution", - "value": 5.0 - }) - ); - } + fn a_metric_descriptor_is_sent_once_per_name() { + let mut batch = Batch::default(); - #[test] - fn batch_dedups_descriptors_and_counts_events() { - let descriptor = || { - TelemetryMsg::Descriptor(MetricDescriptor { - name: "latency".to_string(), - kind: MetricKind::Distribution, - unit: Some("ms".to_string()), - description: None, - }) - }; + batch.push(descriptor("latency")); + batch.push(descriptor("latency")); + batch.push(descriptor("throughput")); - let mut batch = Batch::default(); - batch.push(descriptor()); - batch.push(descriptor()); - batch.push(TelemetryMsg::Log(LogSample { - timestamp_ms: 0, - level: LogLevel::Info, - message: "hi".to_string(), - metadata: serde_json::json!({}), - })); - batch.push(TelemetryMsg::Metric(MetricSample { - name: "latency".to_string(), - timestamp_ms: 0, - metadata: serde_json::json!({}), - data: MetricData::Gauge { value: 1.0 }, - })); - - assert_eq!(batch.descriptors.len(), 1); - assert_eq!(batch.logs.len(), 1); - assert_eq!(batch.metrics.len(), 1); - assert_eq!(batch.len(), 3); + let sent: Vec<_> = batch + .descriptors + .iter() + .map(|descriptor| descriptor.name.as_str()) + .collect(); + assert_eq!(sent, ["latency", "throughput"]); } } From 7770378e5d3a969fcd0ec0f190ed4877089ab316 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:33:34 -0400 Subject: [PATCH 65/90] test: a job number's attribute value is tested by case The job number test asserted a number and a text in one test. Each is now an rstest case of the rule that a job number is recorded as a number when it is one. --- crates/tracel/src/running_job.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/tracel/src/running_job.rs b/crates/tracel/src/running_job.rs index fa612930..4c36a50a 100644 --- a/crates/tracel/src/running_job.rs +++ b/crates/tracel/src/running_job.rs @@ -205,10 +205,14 @@ mod tests { assert_eq!(recorder.recorded(), Recorded::default()); } - #[test] - fn a_job_number_is_recorded_as_a_number_when_it_is_one() { - assert_eq!(job_num("42"), json!(42)); - assert_eq!(job_num("job-42"), json!("job-42")); + #[rstest] + #[case::digits("42", json!(42))] + #[case::not_only_digits("job-42", json!("job-42"))] + fn a_job_number_is_recorded_as_a_number_when_it_is_one( + #[case] num: &str, + #[case] recorded: Value, + ) { + assert_eq!(job_num(num), recorded); } #[rstest] From 76a68f68c3c49eaf16422f0de82608eb3ee12630 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 66/90] docs: the runtime's docs say outside a job, not in a unit test The crate and scope docs said a thread has no job "in a plain binary or a unit test", and FinishSink said it was split out so the loop is testable with a fake. A public doc says what the code does: there is no job outside one, and the sink reports job outcomes to the station. --- crates/tracel-runtime/src/lib.rs | 5 ++--- crates/tracel-runtime/src/scope.rs | 4 ++-- crates/tracel-runtime/src/station/serve.rs | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/tracel-runtime/src/lib.rs b/crates/tracel-runtime/src/lib.rs index d3d0eb9c..de638226 100644 --- a/crates/tracel-runtime/src/lib.rs +++ b/crates/tracel-runtime/src/lib.rs @@ -73,9 +73,8 @@ //! function, while it runs, so jobs a Station runner runs at once each see their own. The //! functions read it: [`job_running`] says whether there is one, [`job_num`] gives its number, and //! [`job_input`] its input, as JSON. To read it on another thread, hold [`JobScope::current`]. The -//! `tracel` crate links the experiments a job starts to it through its scope. While a thread runs -//! no job, such as in a plain binary or a unit test, there is none: [`job_running`] gives `false`, -//! and [`job_num`] and [`job_input`] give `None`. +//! `tracel` crate links the experiments a job starts to it through its scope. Outside a job there +//! is none: [`job_running`] gives `false`, and [`job_num`] and [`job_input`] give `None`. mod execution; mod job; diff --git a/crates/tracel-runtime/src/scope.rs b/crates/tracel-runtime/src/scope.rs index 82a7484f..3635ccae 100644 --- a/crates/tracel-runtime/src/scope.rs +++ b/crates/tracel-runtime/src/scope.rs @@ -2,8 +2,8 @@ //! //! Each execution of a job has a scope of its own, current on the thread that runs the job's //! function, while it runs. Code in the job reads it through the functions below, or holds it, -//! through [`JobScope::current`], to read it on another thread. While a thread runs no job, such as -//! in a plain binary or a unit test, there is none, and the functions say so. +//! through [`JobScope::current`], to read it on another thread. Outside a job there is none, and the +//! functions say so. use std::cell::RefCell; use std::sync::Arc; diff --git a/crates/tracel-runtime/src/station/serve.rs b/crates/tracel-runtime/src/station/serve.rs index 70c678c1..20e3bb16 100644 --- a/crates/tracel-runtime/src/station/serve.rs +++ b/crates/tracel-runtime/src/station/serve.rs @@ -15,7 +15,7 @@ use crate::station::infrastructure::protocol::{ }; use crate::{Execution, Jobs}; -/// Reports job outcomes back to the station. Split out so the loop is testable with a fake. +/// Reports job outcomes back to the station. pub trait FinishSink: Send + Sync + 'static { fn finish_job(&self, job_id: Uuid, finish: &FinishJob); } From 200b71956c9ba8cdd27604f495daacaeacf10231 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 67/90] test: the panic tests assert the failure, not its message The panic test compared each failure with the exact message catch_panic words, all in one test. A panic now fails the run, a returned error is the run's failure as it was returned, and the panic message is the text the payload carries, a &str or a String, as rstest cases. --- crates/tracel-runtime/src/panics.rs | 35 ++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/tracel-runtime/src/panics.rs b/crates/tracel-runtime/src/panics.rs index be037f88..04531454 100644 --- a/crates/tracel-runtime/src/panics.rs +++ b/crates/tracel-runtime/src/panics.rs @@ -24,21 +24,30 @@ fn panic_message(panic: &(dyn Any + Send)) -> &str { #[cfg(test)] mod tests { + use std::io; + + use rstest::rstest; + use super::*; + const MESSAGE: &str = "the kernel exploded"; + #[test] - fn a_panic_is_the_failure_the_job_ends_with() { - let failure = |run: fn() -> Result<(), BoxError>| catch_panic(run).unwrap_err().to_string(); - - assert_eq!( - failure(|| panic!("kernel exploded")), - "the job panicked: kernel exploded" - ); - assert_eq!( - failure(|| panic!("{} exploded", "kernel")), - "the job panicked: kernel exploded" - ); - assert_eq!(failure(|| Err("boom".into())), "boom"); - assert!(catch_panic(|| Ok(())).is_ok()); + fn a_panic_fails_the_run() { + assert!(catch_panic(|| panic!("{MESSAGE}")).is_err()); + } + + #[test] + fn an_error_the_run_returns_is_its_failure() { + let failure = catch_panic(|| Err(io::Error::other(MESSAGE).into())).unwrap_err(); + + assert!(failure.is::()); + } + + #[rstest] + #[case::str(Box::new(MESSAGE))] + #[case::string(Box::new(MESSAGE.to_string()))] + fn the_panic_message_is_the_text_the_panic_carries(#[case] payload: Box) { + assert_eq!(panic_message(payload.as_ref()), MESSAGE); } } From c8872d532e6e2c2586bad2c04307c957a660e55a Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 68/90] test: each scope test checks one rule against the scope it entered One test checked what the functions give both while a scope is entered and once its guard drops; they are now two tests, the first comparing with the scope itself rather than numbers written out again. The two threads entering their own scopes now meet at a Barrier while both are entered, so a scope shared between threads would fail the test, and the held scope is compared with the scope it holds. --- crates/tracel-runtime/src/scope.rs | 71 +++++++++++++----------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/crates/tracel-runtime/src/scope.rs b/crates/tracel-runtime/src/scope.rs index 3635ccae..2a04bd68 100644 --- a/crates/tracel-runtime/src/scope.rs +++ b/crates/tracel-runtime/src/scope.rs @@ -91,6 +91,9 @@ pub fn job_input() -> Option { #[cfg(test)] mod tests { + use std::sync::Barrier; + use std::thread; + use super::*; fn scope(num: &str) -> JobScope { @@ -98,19 +101,20 @@ mod tests { } #[test] - fn a_scope_is_current_on_its_thread_until_its_guard_drops() { - let entered = enter(scope("12")); - let given = (job_running(), job_num(), job_input()); - drop(entered); + fn the_entered_scope_gives_the_number_and_input_of_its_job() { + let job = scope("12"); + let _entered = enter(job.clone()); assert_eq!( - given, - ( - true, - Some("12".to_string()), - Some(serde_json::json!({"num": "12"})) - ) + (job_num().as_deref(), job_input().as_ref()), + (job.num(), Some(job.input())) ); + } + + #[test] + fn once_its_guard_drops_the_thread_runs_no_job() { + drop(enter(scope("12"))); + assert_eq!((job_running(), job_num(), job_input()), (false, None, None)); } @@ -125,46 +129,33 @@ mod tests { } #[test] - fn each_thread_sees_the_scope_of_the_job_it_runs() { - let run = |num: &'static str| { - let scope = scope(num); - std::thread::spawn(move || { - let _entered = enter(scope); - (job_num(), job_input()) + fn jobs_running_at_once_each_see_their_own_scope() { + let both_entered = Arc::new(Barrier::new(2)); + let run = |job: JobScope| { + let both_entered = both_entered.clone(); + thread::spawn(move || { + let _entered = enter(job); + both_entered.wait(); + job_num() }) }; + let jobs = [scope("1"), scope("2")]; - let one = run("1"); - let two = run("2"); + let seen = jobs.clone().map(run).map(|running| running.join().unwrap()); - let input = |num: &str| Some(serde_json::json!({ "num": num })); - assert_eq!(one.join().unwrap(), (Some("1".to_string()), input("1"))); - assert_eq!(two.join().unwrap(), (Some("2".to_string()), input("2"))); - assert!(!job_running()); + assert_eq!(seen, jobs.map(|job| job.num().map(str::to_string))); } #[test] fn a_held_scope_reads_the_same_job_on_another_thread() { - let _entered = enter(scope("12")); + let job = scope("12"); + let _entered = enter(job.clone()); let held = JobScope::current().unwrap(); - let seen = std::thread::spawn(move || { - ( - job_running(), - held.num().map(str::to_string), - held.input().clone(), - ) - }) - .join() - .unwrap(); + let read = thread::spawn(move || (held.num().map(str::to_string), held.input().clone())) + .join() + .unwrap(); - assert_eq!( - seen, - ( - false, - Some("12".to_string()), - serde_json::json!({"num": "12"}) - ) - ); + assert_eq!(read, (job.num().map(str::to_string), job.input().clone())); } } From 6d2e05269a06e4c9218e225dde41e7e839195411 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 69/90] test: how a job ends is one rstest case per outcome The test of how a job ends ran three jobs and asserted six things about them, among them the failed job's exact message. A job that completes, fails or panics is now an rstest case of one fixture, each checking the status of its outcome; the error a job fails with is job.rs's to test. --- crates/tracel-runtime/src/execution.rs | 49 +++++++++++--------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/crates/tracel-runtime/src/execution.rs b/crates/tracel-runtime/src/execution.rs index b5a378dc..3fd51f0e 100644 --- a/crates/tracel-runtime/src/execution.rs +++ b/crates/tracel-runtime/src/execution.rs @@ -108,25 +108,22 @@ impl Outcome { mod tests { use std::sync::{Arc, OnceLock}; - use rstest::rstest; + use rstest::{fixture, rstest}; use serde_json::json; use super::*; - use crate::{Job, Jobs, job_input, job_num}; + use crate::{Jobs, job_input, job_num}; - /// `train`, which takes a number of epochs, three by default, and fails on zero; and `panic`, - /// which panics. + #[fixture] fn jobs() -> Jobs { Jobs::new() - .job( - "train", - Job::new(|epochs: u32| match epochs { - 0 => Err::<(), BoxError>("no epochs".into()), - _ => Ok(()), - }) - .with_default(3), - ) - .job("panic", |()| -> Result<(), BoxError> { panic!("kaboom") }) + .job("completes", |()| Ok::<_, BoxError>(())) + .job("fails", |()| { + Err::<(), BoxError>("no epochs to train".into()) + }) + .job("panics", |()| -> Result<(), BoxError> { + panic!("the kernel exploded") + }) } fn reading(read: fn() -> T) -> (Jobs, Arc>) @@ -144,21 +141,17 @@ mod tests { (jobs, seen) } - #[test] - fn a_job_completes_or_fails_by_what_it_returns_and_a_panic_fails_it() { - let run = - |name: &str, input: Value| jobs().prepare(name, input).unwrap().run(Execution::new()); - - let completed = run("train", Value::Null); - let failed = run("train", json!(0)); - let panicked = run("panic", Value::Null); - - assert_eq!(completed.status(), JobStatus::Completed); - assert!(completed.error().is_none()); - assert!(completed.into_result().is_ok()); - assert_eq!(failed.status(), JobStatus::Failed); - assert_eq!(failed.error().unwrap().to_string(), "no epochs"); - assert_eq!(panicked.status(), JobStatus::Failed); + #[rstest] + #[case::completes("completes", JobStatus::Completed)] + #[case::fails("fails", JobStatus::Failed)] + #[case::panics("panics", JobStatus::Failed)] + fn a_job_ends_as_its_function_does(jobs: Jobs, #[case] name: &str, #[case] status: JobStatus) { + let outcome = jobs + .prepare(name, Value::Null) + .unwrap() + .run(Execution::new()); + + assert_eq!(outcome.status(), status); } #[rstest] From b2379f5167bc93d29501d07de3c355b7a6b3fd04 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 70/90] test: the job tests build their expectations from the fixture The definition test spelled out the default as JSON and also restated that a bare function has no description or default; it now compares with the fixture's own default, and the bare case is gone. The merge test builds the expected input from the default it was merged onto, the failed job is checked by the error's type rather than its message, and the schema is compared with the input type's own. The repeated runs compare what each run was given instead of asserting in a loop, and the tests that only sent an input through serde and back are gone. --- crates/tracel-runtime/src/job.rs | 79 +++++++++----------------------- 1 file changed, 21 insertions(+), 58 deletions(-) diff --git a/crates/tracel-runtime/src/job.rs b/crates/tracel-runtime/src/job.rs index 615b3bb1..9b3966ee 100644 --- a/crates/tracel-runtime/src/job.rs +++ b/crates/tracel-runtime/src/job.rs @@ -181,6 +181,7 @@ fn resolve(default: Option, input: Value) -> Value { #[cfg(test)] mod tests { + use std::io; use std::sync::Mutex; use rstest::rstest; @@ -215,7 +216,6 @@ mod tests { } } - /// A job that keeps the input it runs with. fn recording() -> (Job, Arc>>) where I: DeserializeOwned + Send + 'static, @@ -231,7 +231,6 @@ mod tests { (job, ran_with) } - /// Runs `job`, registered as `train`, with `input`. fn run(job: Job, input: Value) -> Result<(), BoxError> where I: DeserializeOwned + Send + 'static, @@ -243,47 +242,21 @@ mod tests { #[test] fn a_job_is_defined_by_its_name_description_and_default() { + let (name, description) = ("train", "Train the model"); let (job, _) = recording::(); let job = job - .with_description("Train the model") + .with_description(description) .with_default(default_config()); - let (bare, _) = recording::(); assert_eq!( - Registered::new("train".to_string(), job).definition(), + Registered::new(name.to_string(), job).definition(), &JobDefinition { - name: "train".to_string(), - description: Some("Train the model".to_string()), + name: name.to_string(), + description: Some(description.to_string()), input_schema: None, - input_example: Some(json!({ - "epochs": 10, - "optimizer": {"lr": 0.001, "weight_decay": 0.0}, - "tag": "baseline" - })), + input_example: Some(serde_json::to_value(default_config()).unwrap()), } ); - assert_eq!( - Registered::new("evaluate".to_string(), bare).definition(), - &JobDefinition { - name: "evaluate".to_string(), - description: None, - input_schema: None, - input_example: None, - } - ); - } - - #[test] - fn without_a_default_the_input_decodes_as_given() { - let optimizer = Optimizer { - lr: 0.1, - weight_decay: 0.01, - }; - let (job, ran_with) = recording(); - - run(job, serde_json::to_value(&optimizer).unwrap()).unwrap(); - - assert_eq!(ran_with.lock().unwrap().take(), Some(optimizer)); } #[test] @@ -295,19 +268,6 @@ mod tests { assert_eq!(ran_with.lock().unwrap().take(), Some(default_config())); } - #[test] - fn the_prepared_input_is_the_json_the_job_runs_with() { - let (job, ran_with) = recording::(); - let prepared = Registered::new("train".to_string(), job.with_default(default_config())) - .prepare(json!({"epochs": 2, "optimizer": {"lr": 0.1}})) - .unwrap(); - - (prepared.run)().unwrap(); - - let ran = ran_with.lock().unwrap().take().unwrap(); - assert_eq!(prepared.input, serde_json::to_value(ran).unwrap()); - } - #[test] fn an_input_is_merge_patched_onto_the_default() { let (job, ran_with) = recording(); @@ -324,7 +284,7 @@ mod tests { epochs: 2, optimizer: Optimizer { lr: 0.1, - weight_decay: 0.0, + ..default_config().optimizer }, tag: None, }) @@ -347,20 +307,23 @@ mod tests { fn a_registered_job_runs_each_time_it_is_prepared() { let (job, ran_with) = recording::(); let registered = Registered::new("train".to_string(), job); + let epochs = [1, 2]; - for epochs in [1, 2] { + let ran = epochs.map(|epochs| { (registered.prepare(json!(epochs)).unwrap().run)().unwrap(); - assert_eq!(ran_with.lock().unwrap().take(), Some(epochs)); - } + ran_with.lock().unwrap().take() + }); + + assert_eq!(ran, epochs.map(Some)); } #[test] fn a_job_fails_with_the_error_it_returns() { - let job = Job::new(|_: ()| Err::<(), _>(std::io::Error::other("disk full"))); + let job = Job::new(|_: ()| Err::<(), _>(io::Error::other("disk full"))); - let failed = run(job, Value::Null); + let failed = run(job, Value::Null).unwrap_err(); - assert_eq!(failed.unwrap_err().to_string(), "disk full"); + assert!(failed.is::()); } #[cfg(feature = "schema")] @@ -370,9 +333,9 @@ mod tests { let registered = Registered::new("train".to_string(), job.with_schema()); - let schema = registered.definition().input_schema.clone().unwrap(); - assert_eq!(schema["title"], "Config"); - assert!(schema["properties"]["epochs"].is_object()); - assert!(schema["properties"]["optimizer"].is_object()); + assert_eq!( + registered.definition().input_schema, + Some(schemars::schema_for!(Config).to_value()) + ); } } From 590a1702c85aca08e6e5f27d158c5b83f1239098 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 71/90] test: each write_atomically test checks one rule The two write tests each checked the contents and that no temporary file was left, the second by guessing the temporary file's name. The contents written and the replaced file are now one test each, and no temporary file is left is its own test, listing the directory as the failed-rename test does. --- crates/tracel-runtime/src/process/file.rs | 26 ++++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/tracel-runtime/src/process/file.rs b/crates/tracel-runtime/src/process/file.rs index d6eda0c0..1767b8ed 100644 --- a/crates/tracel-runtime/src/process/file.rs +++ b/crates/tracel-runtime/src/process/file.rs @@ -23,6 +23,8 @@ mod tests { use super::*; + const CONTENTS: &str = "{}\n"; + fn entries(dir: &Path) -> Vec { fs::read_dir(dir) .unwrap() @@ -31,26 +33,34 @@ mod tests { } #[test] - fn the_file_holds_the_contents_and_no_temporary_file_is_left() { + fn the_file_holds_the_contents_written() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("jobs.json"); - write_atomically(&path, "{}\n").unwrap(); + write_atomically(&path, CONTENTS).unwrap(); - assert_eq!(fs::read_to_string(&path).unwrap(), "{}\n"); - assert_eq!(fs::read_dir(dir.path()).unwrap().count(), 1); + assert_eq!(fs::read_to_string(&path).unwrap(), CONTENTS); } #[test] - fn the_file_replaces_an_existing_one_through_a_temporary_file() { + fn the_file_replaces_an_existing_one() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("jobs.json"); fs::write(&path, "stale").unwrap(); - write_atomically(&path, "fresh").unwrap(); + write_atomically(&path, CONTENTS).unwrap(); + + assert_eq!(fs::read_to_string(&path).unwrap(), CONTENTS); + } - assert_eq!(fs::read_to_string(&path).unwrap(), "fresh"); - assert!(!dir.path().join("jobs.json.tmp").exists()); + #[test] + fn a_written_file_leaves_no_temporary_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("jobs.json"); + + write_atomically(&path, CONTENTS).unwrap(); + + assert_eq!(entries(dir.path()), [path.file_name().unwrap()]); } #[test] From feb9849d62ed7ef016af9651c825b6648095be2f Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 72/90] test: the process test helpers carry no doc comments parse kept the one doc comment left among the process test helpers; its name and signature say what it reads. --- crates/tracel-runtime/src/process/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/tracel-runtime/src/process/mod.rs b/crates/tracel-runtime/src/process/mod.rs index 771f28db..53547cf6 100644 --- a/crates/tracel-runtime/src/process/mod.rs +++ b/crates/tracel-runtime/src/process/mod.rs @@ -215,7 +215,6 @@ mod tests { .job("fail", |()| Err::<(), BoxError>("the job failed".into())) } - /// What the command line `program ` asks of `jobs`. fn parse(jobs: &Jobs, args: &[&str]) -> Result { let mut command = command_line(jobs, "program"); read_request(jobs, &mut command, ["program"].iter().chain(args)) From 93641a8c20679284d811b071ad21e6139759eacf Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 73/90] test: describe is not tested for the file serde writes The definitions test compared what describe writes with the file's JSON, which serde decides. The process adapter's tests already show that a launcher asking for the definitions gets them. --- crates/tracel-runtime/src/process/describe.rs | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/crates/tracel-runtime/src/process/describe.rs b/crates/tracel-runtime/src/process/describe.rs index ab8550d5..4809e90f 100644 --- a/crates/tracel-runtime/src/process/describe.rs +++ b/crates/tracel-runtime/src/process/describe.rs @@ -33,7 +33,7 @@ pub struct DescribeError { #[cfg(test)] mod tests { - use serde_json::{Value, json}; + use serde_json::json; use super::*; @@ -54,37 +54,6 @@ mod tests { ] } - #[test] - fn the_definitions_file_lists_every_job() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("jobs.json"); - - describe(&definitions(), &path).unwrap(); - - let file: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); - assert_eq!( - file, - json!({ - "protocol": 1, - "sdk_version": env!("CARGO_PKG_VERSION"), - "jobs": [ - { - "name": "evaluate", - "description": null, - "input_schema": null, - "input_example": null - }, - { - "name": "train", - "description": "Train the model", - "input_schema": null, - "input_example": {"epochs": 10, "optimizer": {"lr": 0.001}} - } - ] - }) - ); - } - #[test] fn a_failed_write_names_the_path() { let dir = tempfile::tempdir().unwrap(); From 2e67871757bc6231e00eb17ee69576b0bd293b1e Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 74/90] test: the cancelled job test compares the statuses reported The test counted the finishes, then looked each one up in a loop. It now compares the statuses reported with both jobs completed, as the other serve tests do, and takes the setup from the fixture. --- crates/tracel-runtime/src/station/serve.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/crates/tracel-runtime/src/station/serve.rs b/crates/tracel-runtime/src/station/serve.rs index 20e3bb16..565a4cbf 100644 --- a/crates/tracel-runtime/src/station/serve.rs +++ b/crates/tracel-runtime/src/station/serve.rs @@ -399,9 +399,8 @@ mod tests { assert_eq!(statuses(&finishes), completed(&waiting)); } - #[test] - fn a_cancelled_job_runs_to_completion_and_the_stream_goes_on() { - let setup = setup(); + #[rstest] + fn a_cancelled_job_runs_to_completion_and_the_stream_goes_on(setup: Setup) { let (cancelled, next) = (dispatched("train"), dispatched("train")); serve_stream( @@ -419,14 +418,7 @@ mod tests { ); setup.executor.wait_idle(); - let finishes = setup.sink.finishes.lock().unwrap().clone(); - assert_eq!(finishes.len(), 2); - for job in [&cancelled, &next] { - assert!( - finishes.contains(&(job.id, setup.runner_id, FinishStatus::Completed, None)), - "{finishes:?}" - ); - } + assert_eq!(statuses(&finishes(&setup)), completed([&cancelled, &next])); } #[rstest] From e88a5bd138eb1679e88cbdcad65e14347f28dadd Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:25 -0400 Subject: [PATCH 75/90] test: the SSE tests compare whole frames Two parser tests checked a frame field by field, one of them after counting the frames. They now compare the whole frame the parser gives. --- .../src/station/infrastructure/sse.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/tracel-runtime/src/station/infrastructure/sse.rs b/crates/tracel-runtime/src/station/infrastructure/sse.rs index 421824d5..6921f8c3 100644 --- a/crates/tracel-runtime/src/station/infrastructure/sse.rs +++ b/crates/tracel-runtime/src/station/infrastructure/sse.rs @@ -99,16 +99,25 @@ mod tests { fn a_value_needs_no_leading_space() { let frames = parse(&["event:job", "data:x", ""]); - assert_eq!(frames[0].event, "job"); - assert_eq!(frames[0].data, "x"); + assert_eq!( + frames, + [SseFrame { + event: "job".to_string(), + data: "x".to_string(), + }] + ); } #[test] fn each_frame_starts_afresh() { let frames = parse(&["event: a", "data: 1", "", "data: 2", ""]); - assert_eq!(frames.len(), 2); - assert_eq!(frames[1].event, "message"); - assert_eq!(frames[1].data, "2"); + assert_eq!( + frames[1], + SseFrame { + event: "message".to_string(), + data: "2".to_string(), + } + ); } } From 9a68d9ffd0538a918d97e2d0d63aaee8df3c9eb4 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:39:52 -0400 Subject: [PATCH 76/90] test: the run numbering test checks only the numbers runs_are_numbered_from_one_under_the_experiment_name also checked each run's directory, which the location test pins, so it now checks the numbers alone. The test of a runs directory that already exists takes its run from the local_run fixture. --- crates/tracel-experiment/src/local/runs.rs | 23 ++++++---------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/crates/tracel-experiment/src/local/runs.rs b/crates/tracel-experiment/src/local/runs.rs index f5f9160c..9c14343b 100644 --- a/crates/tracel-experiment/src/local/runs.rs +++ b/crates/tracel-experiment/src/local/runs.rs @@ -76,7 +76,7 @@ mod tests { use rstest::rstest; use super::*; - use crate::local::test_support::run; + use crate::local::test_support::{LocalRun, local_run, run}; #[test] fn a_runs_directory_it_creates_ignores_everything_in_it() { @@ -92,13 +92,9 @@ mod tests { ); } - #[test] - fn a_runs_directory_that_exists_is_left_as_it_is() { - let dir = tempfile::tempdir().unwrap(); - - let _run = run(dir.path(), "mnist"); - - assert!(!dir.path().join(".gitignore").exists()); + #[rstest] + fn a_runs_directory_that_exists_is_left_as_it_is(local_run: LocalRun) { + assert!(!local_run.runs.path().join(".gitignore").exists()); } #[rstest] @@ -124,16 +120,9 @@ mod tests { fn runs_are_numbered_from_one_under_the_experiment_name() { let dir = tempfile::tempdir().unwrap(); - let first = run(dir.path(), "mnist"); - let second = run(dir.path(), "mnist"); - let other = run(dir.path(), "other"); + let ids = ["mnist", "mnist", "other"].map(|name| run(dir.path(), name).id().to_string()); - assert_eq!(first.id().as_str(), "1"); - assert_eq!(second.id().as_str(), "2"); - assert_eq!(other.id().as_str(), "1"); - assert!(dir.path().join("mnist/1").is_dir()); - assert!(dir.path().join("mnist/2").is_dir()); - assert!(dir.path().join("other/1").is_dir()); + assert_eq!(ids, ["1", "2", "1"]); } #[test] From 9ab32034b52f4043ac865012d2d5ab35fae230d2 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:40:22 -0400 Subject: [PATCH 77/90] test: the status tests check how a run ended, not its JSON the_status_follows_the_run_from_the_start_to_the_end pinned the file twice, checked its timestamps against chrono's format, and checked a temporary file that the rename never leaves. Keeping the start and stamping the end are now checked on the status itself, and the store writing a finished run as completed is tested with the session that writes it. --- crates/tracel-experiment/src/local/mod.rs | 8 +++ crates/tracel-experiment/src/local/status.rs | 54 +++----------------- 2 files changed, 16 insertions(+), 46 deletions(-) diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs index 8b2c6d7d..c1d1f427 100644 --- a/crates/tracel-experiment/src/local/mod.rs +++ b/crates/tracel-experiment/src/local/mod.rs @@ -321,6 +321,14 @@ mod tests { assert_eq!(read_lines(&local_run.dir.join(EVENTS_FILE)).len(), 1); } + #[rstest] + fn a_finished_run_is_completed_in_its_status(local_run: LocalRun) { + local_run.run.finish().unwrap(); + + let status = read_json(&local_run.dir.join(STATUS_FILE)); + assert_eq!(status["status"], json!(StatusKind::Completed)); + } + #[derive(Default)] struct FailsFirstWrite { failed: bool, diff --git a/crates/tracel-experiment/src/local/status.rs b/crates/tracel-experiment/src/local/status.rs index 1923b9d3..7ff83cd8 100644 --- a/crates/tracel-experiment/src/local/status.rs +++ b/crates/tracel-experiment/src/local/status.rs @@ -78,10 +78,9 @@ fn now() -> String { #[cfg(test)] mod tests { use rstest::rstest; - use serde_json::{Value, json}; use super::*; - use crate::local::test_support::{EXPERIMENT, read_json, run}; + use crate::local::test_support::EXPERIMENT; const REASON: &str = "loss is NaN"; @@ -89,53 +88,16 @@ mod tests { Status::running(EXPERIMENT.to_string(), 1) } - fn is_utc_seconds(timestamp: &Value) -> bool { - timestamp.as_str().is_some_and(|timestamp| { - timestamp.ends_with('Z') - && !timestamp.contains('.') - && chrono::DateTime::parse_from_rfc3339(timestamp).is_ok() - }) - } - #[test] - fn the_status_follows_the_run_from_the_start_to_the_end() { - let dir = tempfile::tempdir().unwrap(); - let experiment = run(dir.path(), "mnist"); - let run_dir = dir.path().join("mnist/1"); - - let mut running = read_json(&run_dir.join("status.json")); - experiment.finish().unwrap(); - let mut completed = read_json(&run_dir.join("status.json")); - - assert!(is_utc_seconds(&running["started_at"]), "{running}"); - assert!(is_utc_seconds(&completed["finished_at"]), "{completed}"); - assert_eq!(completed["started_at"], running["started_at"]); - running["started_at"] = Value::Null; - completed["started_at"] = Value::Null; - completed["finished_at"] = Value::Null; - assert_eq!( - running, - json!({ - "name": "mnist", - "num": 1, - "status": "running", - "started_at": null, - "finished_at": null, - "error": null - }) - ); + fn an_ended_status_keeps_when_the_run_started_and_says_when_it_ended() { + let running = running(); + + let ended = running.ended(ExperimentCompletion::Success); + assert_eq!( - completed, - json!({ - "name": "mnist", - "num": 1, - "status": "completed", - "started_at": null, - "finished_at": null, - "error": null - }) + (ended.started_at, ended.finished_at.is_some()), + (running.started_at, true) ); - assert!(!run_dir.join("status.json.tmp").exists()); } #[rstest] From 9b906a6987c60231516162be2b9cad89e8453916 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 13:34:27 -0400 Subject: [PATCH 78/90] test: the events lines are not tested for the JSON serde writes a_run_records_each_event_as_a_line_of_json drove one run through every kind of event and compared each line it read back, which is what serde writes, so it is gone. The attribute lines, which the session writes as the run starts, are pinned with the session, in order of their keys. --- crates/tracel-experiment/src/local/events.rs | 146 ------------------- crates/tracel-experiment/src/local/mod.rs | 28 +++- 2 files changed, 27 insertions(+), 147 deletions(-) diff --git a/crates/tracel-experiment/src/local/events.rs b/crates/tracel-experiment/src/local/events.rs index d9e48e5f..14c87dc0 100644 --- a/crates/tracel-experiment/src/local/events.rs +++ b/crates/tracel-experiment/src/local/events.rs @@ -182,149 +182,3 @@ pub fn to_jsonl(lines: impl IntoIterator) -> serde_json::Result Date: Thu, 15 Oct 2026 12:41:59 -0400 Subject: [PATCH 79/90] test: a saved artifact loads back from a run of the fixture's store The test made its own store and the first run in it: it now saves from the local_run fixture's run and loads from a later run of the same store. The one-file artifact type loses its comment. --- .../tracel-experiment/src/local/artifacts.rs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/tracel-experiment/src/local/artifacts.rs b/crates/tracel-experiment/src/local/artifacts.rs index 7ae470d1..f1bf3feb 100644 --- a/crates/tracel-experiment/src/local/artifacts.rs +++ b/crates/tracel-experiment/src/local/artifacts.rs @@ -159,7 +159,6 @@ mod tests { use crate::local::test_support::{EXPERIMENT, LocalRun, local_run, run}; use crate::{ArtifactKind, ExperimentRun}; - /// A one-file artifact. #[derive(Debug, PartialEq)] struct Note(String); @@ -251,20 +250,22 @@ mod tests { names } - #[test] - fn a_saved_artifact_loads_back_from_a_later_run() { - let dir = tempfile::tempdir().unwrap(); - let first = run(dir.path(), "mnist"); - first - .save_artifact("note", ArtifactKind::Other, Note("kept".to_string()), &()) + #[rstest] + fn a_saved_artifact_loads_back_from_a_later_run(local_run: LocalRun) { + let saved = || Note("kept".to_string()); + local_run + .run + .save_artifact("note", ArtifactKind::Other, saved(), &()) .unwrap(); - let first_id = first.id().clone(); - first.finish().unwrap(); + let saved_by = local_run.run.id().clone(); + local_run.run.finish().unwrap(); - let second = run(dir.path(), "mnist"); - let note: Note = second.use_artifact(first_id, "note", &()).unwrap(); + let later = run(local_run.runs.path(), EXPERIMENT); - assert_eq!(note, Note("kept".to_string())); + assert_eq!( + later.use_artifact::(saved_by, "note", &()).unwrap(), + saved() + ); } #[rstest] From b735bbd34c0e0dfcb2961c817d31605327c7588c Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:43:56 -0400 Subject: [PATCH 80/90] test: activity endings and progress are tested beside the activity The activity tests checked each ending in a test of its own and reached into events by position. Endings, of a guard and of a run, are now rstest cases compared as status and message, which ActivityStatus now derives PartialEq for. Progress updates are one rule tested with and without a meter, and each test asserts what it is named for. The tests of a guard failing with the panic that unwinds it and of running an activity, once in lib.rs and the tracing tests, move here beside the code that decides them; the ambient scope a run installs is checked through the current experiment, without tracing. --- crates/tracel-experiment/src/activity.rs | 306 ++++++++---------- .../src/integration/tracing/mod.rs | 26 -- crates/tracel-experiment/src/lib.rs | 81 ----- crates/tracel-experiment/src/test_support.rs | 24 +- 4 files changed, 161 insertions(+), 276 deletions(-) diff --git a/crates/tracel-experiment/src/activity.rs b/crates/tracel-experiment/src/activity.rs index b6200121..cb39b6b5 100644 --- a/crates/tracel-experiment/src/activity.rs +++ b/crates/tracel-experiment/src/activity.rs @@ -68,7 +68,7 @@ pub struct ActivitySpec { } /// Terminal state for an activity. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum ActivityStatus { /// The activity completed successfully. Success, @@ -644,229 +644,199 @@ impl std::ops::Deref for ActivityGuard { #[cfg(test)] mod tests { - use std::sync::Arc; + use std::convert::Infallible; + use rstest::rstest; use serde_json::json; - use crate::test_support::{MockSession, create_run, create_run_with_control}; + use crate::ExperimentRun; + use crate::context::ExperimentGlobalExt; + use crate::test_support::{ + MockRun, MockSession, create_run, create_run_with_control, mock_run, + }; use super::*; - fn setup_run() -> (Arc, crate::ExperimentRun) { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - (session, run) - } - - #[test] - fn start_reports_configured_activity() { - let (session, run) = setup_run(); - - let _guard = run - .activity("load") - .meter(12, "items") - .attr("split", "train") - .start(); - - let events = session.activity_events(); - let ActivityEvent::Started { activity } = &events[0] else { - panic!("unexpected event: {:?}", events[0]); - }; - assert_eq!(activity.name, "load"); - let meter = activity.meter.as_ref().expect("expected activity meter"); - assert_eq!(meter.total, Some(12)); - assert_eq!(meter.unit.as_deref(), Some("items")); - assert_eq!(activity.attributes.get("split"), Some(&json!("train"))); - } - - #[test] - fn inc_reports_updated_progress() { - let (session, run) = setup_run(); - let guard = run.activity("items").meter(8, "items").start(); - - guard.inc(3); + const REASON: &str = "fold failed"; + const PANIC: &str = "fusion stream out of order"; - let events = session.activity_events(); - let ActivityEvent::Updated { current, .. } = &events[1] else { - panic!("unexpected event: {:?}", events[1]); - }; - assert_eq!(*current, 3); + fn updates(session: &MockSession) -> Vec { + session + .activity_events() + .into_iter() + .filter_map(|event| match event { + ActivityEvent::Updated { current, .. } => Some(current), + _ => None, + }) + .collect() } - #[test] - fn cloned_activity_reports_accumulated_progress_from_another_thread() { - let (session, run) = setup_run(); - let guard = run.activity("items").meter(8, "items").start(); - let activity = guard.share(); - - activity.inc(2); - let worker_activity = activity.clone(); - std::thread::spawn(move || worker_activity.inc(3)) - .join() - .unwrap(); - - let updates = session + fn endings(session: &MockSession) -> Vec<(ActivityStatus, Option)> { + session .activity_events() .into_iter() .filter_map(|event| match event { - ActivityEvent::Updated { current, .. } => Some(current), + ActivityEvent::Finished { + status, message, .. + } => Some((status, message)), _ => None, }) - .collect::>(); - assert_eq!(updates, vec![2, 5]); + .collect() } - #[test] - fn meterless_activity_reports_updates_as_open_ended_counts() { - let (session, run) = setup_run(); - let guard = run.activity("items").start(); + #[rstest] + fn starting_an_activity_reports_how_it_was_built(mock_run: MockRun) { + let _guard = mock_run + .run + .activity("load") + .meter(12, "items") + .attr("split", "train") + .start(); + + let started = mock_run.session.started_activities().remove(0); + let meter = started.meter.unwrap(); + assert_eq!( + ( + started.name.as_str(), + meter.total, + meter.unit.as_deref(), + started.attributes.get("split") + ), + ("load", Some(12), Some("items"), Some(&json!("train"))) + ); + } + + #[rstest] + #[case::metered(|run: &ExperimentRun| run.activity("items").meter(8, "items").start())] + #[case::open_ended(|run: &ExperimentRun| run.activity("items").start())] + fn increments_from_any_thread_report_the_accumulated_progress( + mock_run: MockRun, + #[case] start: fn(&ExperimentRun) -> ActivityGuard, + ) { + let guard = start(&mock_run.run); let activity = guard.share(); - activity.inc(4); + activity.inc(2); + let worker = activity.clone(); + std::thread::spawn(move || worker.inc(3)).join().unwrap(); - let events = session.activity_events(); - assert!(matches!( - &events[0], - ActivityEvent::Started { activity } if activity.meter.is_none() - )); - assert!(matches!( - &events[1], - ActivityEvent::Updated { current: 4, .. } - )); + assert_eq!(updates(&mock_run.session), [2, 5]); } - #[test] - fn message_reports_activity_message() { - let (session, run) = setup_run(); - let guard = run.activity("items").start(); + #[rstest] + fn a_message_is_reported_for_its_activity(mock_run: MockRun) { + let guard = mock_run.run.activity("items").start(); guard.message("halfway"); - let events = session.activity_events(); assert!(matches!( - &events[1], - ActivityEvent::Message { id, message } - if *id == guard.id() && message == "halfway" + mock_run.session.activity_events().last(), + Some(ActivityEvent::Message { id, message }) if *id == guard.id() && message == "halfway" )); } - #[test] - fn finish_reports_one_success_completion() { - let (session, run) = setup_run(); - - run.activity("node").start().finish(); + #[rstest] + #[case::finished(ActivityGuard::finish, ActivityStatus::Success, None)] + #[case::failed(|guard: ActivityGuard| guard.fail(REASON), ActivityStatus::Failed, Some(REASON))] + #[case::abandoned(ActivityGuard::abandon, ActivityStatus::Abandoned, None)] + #[case::dropped(drop, ActivityStatus::Abandoned, None)] + #[case::dropped_once_cancelled( + |guard: ActivityGuard| { + guard.cancel_token().cancel(); + drop(guard); + }, + ActivityStatus::Abandoned, + None + )] + fn an_activity_reports_once_how_it_ended( + mock_run: MockRun, + #[case] end: fn(ActivityGuard), + #[case] status: ActivityStatus, + #[case] message: Option<&str>, + ) { + end(mock_run.run.activity("node").start()); - let events = session.activity_events(); - let finished: Vec<_> = events - .iter() - .filter_map(|event| match event { - ActivityEvent::Finished { status, .. } => Some(status), - _ => None, - }) - .collect(); - assert!(matches!(finished.as_slice(), [ActivityStatus::Success])); + assert_eq!( + endings(&mock_run.session), + [(status, message.map(String::from))] + ); } #[test] - fn fail_reports_failed_completion_with_its_message() { - let (session, run) = setup_run(); + fn a_guard_unwound_by_a_panic_fails_its_activity_with_the_panic() { + let session = Arc::new(MockSession::default()); + let run_session = session.clone(); - run.activity("node").start().fail("kernel panicked"); + std::thread::spawn(move || { + let run = create_run(run_session); + let _stage = run.activity("stage").start(); + panic!("{PANIC}"); + }) + .join() + .unwrap_err(); - let events = session.activity_events(); assert!(matches!( - events.last(), - Some(ActivityEvent::Finished { - status: ActivityStatus::Failed, - message: Some(message), - .. - }) if message == "kernel panicked" + endings(&session).as_slice(), + [(ActivityStatus::Failed, Some(reason))] if reason.ends_with(PANIC) )); } - #[test] - fn abandon_reports_abandoned_completion() { - let (session, run) = setup_run(); - - run.activity("node").start().abandon(); - - let events = session.activity_events(); - assert!(matches!( - events.last(), - Some(ActivityEvent::Finished { - status: ActivityStatus::Abandoned, - .. + #[rstest] + #[case::returned_ok(Ok(()), ActivityStatus::Success, None)] + #[case::returned_an_error(Err(REASON), ActivityStatus::Failed, Some(REASON))] + fn running_an_activity_ends_it_by_what_its_code_returns( + mock_run: MockRun, + #[case] returned: Result<(), &'static str>, + #[case] status: ActivityStatus, + #[case] message: Option<&str>, + ) { + let _ = mock_run.run.activity("fold").run(|_| returned); + + assert_eq!( + endings(&mock_run.session), + [(status, message.map(String::from))] + ); + } + + #[rstest] + fn running_an_activity_makes_it_the_current_scope(mock_run: MockRun) { + let fold = mock_run + .run + .activity("fold") + .run(|activity| { + ExperimentRun::current().unwrap().log_info("in the fold"); + Ok::<_, Infallible>(activity.id()) }) - )); - } - - #[test] - fn drop_reports_abandoned_completion() { - let (session, run) = setup_run(); - - drop(run.activity("node").start()); + .unwrap(); - let events = session.activity_events(); - assert!(matches!( - events.last(), - Some(ActivityEvent::Finished { - status: ActivityStatus::Abandoned, - .. - }) - )); + assert_eq!(mock_run.session.log_activities(), [Some(fold)]); } - #[test] - fn cancellable_activity_reports_control_metadata() { - let (session, run) = setup_run(); - - let _guard = run.activity("node").cancellable().start(); + #[rstest] + fn a_cancellable_activity_says_so_when_it_starts(mock_run: MockRun) { + let _guard = mock_run.run.activity("node").cancellable().start(); - let events = session.activity_events(); - let ActivityEvent::Started { activity } = &events[0] else { - panic!("unexpected event: {:?}", events[0]); - }; - assert!(activity.cancellable); + assert!(mock_run.session.started_activities()[0].cancellable); } #[test] - fn cancellable_activity_registers_with_control() { - let session = Arc::new(MockSession::default()); + fn a_cancellable_activity_is_cancelled_through_its_run_control() { let control = ExperimentRunControl::default(); - let run = create_run_with_control(session, control.clone()); + let run = create_run_with_control(Default::default(), control.clone()); let guard = run.activity("node").cancellable().start(); - assert!(control.cancel_activity(guard.id())); + control.cancel_activity(guard.id()); + assert!(guard.is_cancel_requested()); } - #[test] - fn child_activity_cancel_request_is_linked_to_parent_activity_token() { - let (_session, run) = setup_run(); - let parent = run.activity("parent").start(); + #[rstest] + fn cancelling_an_activity_requests_the_cancel_of_its_children(mock_run: MockRun) { + let parent = mock_run.run.activity("parent").start(); let child = parent.activity("child").start(); parent.cancel_token().cancel(); - assert!(parent.is_cancel_requested()); assert!(child.is_cancel_requested()); } - - #[test] - fn drop_reports_abandoned_even_when_cancel_was_requested() { - let (session, run) = setup_run(); - let guard = run.activity("node").start(); - - guard.cancel_token().cancel(); - drop(guard); - - let events = session.activity_events(); - assert!(matches!( - events.last(), - Some(ActivityEvent::Finished { - status: ActivityStatus::Abandoned, - .. - }) - )); - } } diff --git a/crates/tracel-experiment/src/integration/tracing/mod.rs b/crates/tracel-experiment/src/integration/tracing/mod.rs index d9369bff..74ffa367 100644 --- a/crates/tracel-experiment/src/integration/tracing/mod.rs +++ b/crates/tracel-experiment/src/integration/tracing/mod.rs @@ -159,32 +159,6 @@ mod tests { } } - #[test] - fn activity_run_installs_its_scope_for_tracing_events() { - let session = Arc::new(MockSession::default()); - let run = create_run_with_id("trace-test-activity", session.clone()); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); - - let activity_id = tracing::subscriber::with_default(subscriber, || { - run.activity("fold") - .run(|activity| { - tracing::info!(fold = 1u64, "training fold"); - Ok::<_, std::convert::Infallible>(activity.id()) - }) - .unwrap() - }); - - let events = session.events.lock().unwrap(); - let activity = events - .iter() - .find_map(|event| match event { - Event::Log { activity, .. } => *activity, - _ => None, - }) - .expect("tracing log should carry an activity scope"); - assert_eq!(activity, activity_id); - } - #[test] fn activity_span_routes_logs_to_its_activity() { let session = Arc::new(MockSession::default()); diff --git a/crates/tracel-experiment/src/lib.rs b/crates/tracel-experiment/src/lib.rs index e8d068ab..9022ad07 100644 --- a/crates/tracel-experiment/src/lib.rs +++ b/crates/tracel-experiment/src/lib.rs @@ -887,45 +887,6 @@ mod tests { use super::*; - #[test] - fn panic_unwinding_a_guard_fails_the_activity_with_the_panic() { - let session = Arc::new(MockSession::default()); - let run_session = session.clone(); - - let outcome = std::thread::Builder::new() - .name("stage-runner".into()) - .spawn(move || { - let run = create_run(run_session); - let _stage = run.activity("Train fold 3 of 6").cancellable().start(); - panic!("fusion stream out of order"); - }) - .unwrap() - .join(); - assert!(outcome.is_err()); - - let finished = session - .activity_events() - .into_iter() - .find_map(|event| match event { - ActivityEvent::Finished { - status, message, .. - } => Some((status, message)), - _ => None, - }) - .expect("the stage reported an ending"); - assert!(matches!(finished.0, ActivityStatus::Failed)); - let message = finished.1.expect("the ending carried the panic"); - assert!(message.contains("fusion stream out of order"), "{message}"); - - let completions = session.completions.lock().unwrap(); - match completions.as_slice() { - [ExperimentCompletion::Failed(reason)] => { - assert!(reason.contains("fusion stream out of order"), "{reason}"); - } - other => panic!("unexpected completions: {other:?}"), - } - } - #[test] fn panicking_run_reports_the_panic_as_its_failure_reason() { let session = Arc::new(MockSession::default()); @@ -1034,48 +995,6 @@ mod tests { assert_eq!(event_activity(&events[5]), None); } - #[test] - fn activity_run_finishes_fails_and_propagates_panics() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - - let value = run - .activity("ok") - .run(|_| Ok::<_, &'static str>(7)) - .unwrap(); - assert_eq!(value, 7); - - let error = run - .activity("error") - .run(|_| Err::<(), _>("fold failed")) - .unwrap_err(); - assert_eq!(error, "fold failed"); - - let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _: Result<(), &'static str> = run.activity("panic").run(|_| panic!("boom")); - })); - assert!(panic.is_err()); - - let events = session.events.lock().unwrap(); - let completions = events - .iter() - .filter_map(|event| match event { - Event::Activity(ActivityEvent::Finished { - status, message, .. - }) => Some((status, message.as_deref())), - _ => None, - }) - .collect::>(); - match completions.as_slice() { - [ - (ActivityStatus::Success, None), - (ActivityStatus::Failed, Some("fold failed")), - (ActivityStatus::Failed, Some(panicked)), - ] => assert!(panicked.contains("boom"), "{panicked}"), - other => panic!("unexpected completions: {other:?}"), - } - } - #[test] fn child_activity_from_reference_uses_reference_as_parent() { let session = Arc::new(MockSession::default()); diff --git a/crates/tracel-experiment/src/test_support.rs b/crates/tracel-experiment/src/test_support.rs index 8a85a25f..a5353dc4 100644 --- a/crates/tracel-experiment/src/test_support.rs +++ b/crates/tracel-experiment/src/test_support.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex}; use rstest::fixture; -use crate::activity::ActivityEvent; +use crate::activity::{ActivityEvent, ActivityId, ActivitySpec}; use crate::error::ExperimentError; use crate::reader::{ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact}; use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; @@ -28,6 +28,16 @@ impl MockSession { .collect() } + pub fn started_activities(&self) -> Vec { + self.activity_events() + .into_iter() + .filter_map(|event| match event { + ActivityEvent::Started { activity } => Some(activity), + _ => None, + }) + .collect() + } + pub fn logs(&self) -> Vec { self.events .lock() @@ -40,6 +50,18 @@ impl MockSession { .collect() } + pub fn log_activities(&self) -> Vec> { + self.events + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + Event::Log { activity, .. } => Some(*activity), + _ => None, + }) + .collect() + } + pub fn log_messages(&self) -> Vec { self.logs() .into_iter() From 2228530711b80134a2248f6d1a81d9dbff037318 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:44:20 -0400 Subject: [PATCH 81/90] test: the panic watch is tested in its own module watched_run_logs_panics_from_other_threads lived in lib.rs, checked the panic hook's line format through the thread name, and asserted both the error log and the flush, the second under a comment. Two tests in panic_watch.rs now check one each: a watched run logs a panic on another thread as an error ending with its message, and flushes what it logged. --- crates/tracel-experiment/src/lib.rs | 33 ------------------ crates/tracel-experiment/src/panic_watch.rs | 38 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/crates/tracel-experiment/src/lib.rs b/crates/tracel-experiment/src/lib.rs index 9022ad07..8d4334a0 100644 --- a/crates/tracel-experiment/src/lib.rs +++ b/crates/tracel-experiment/src/lib.rs @@ -913,39 +913,6 @@ mod tests { } } - #[test] - fn watched_run_logs_panics_from_other_threads() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let _watch = run.capture_panics(); - - let outcome = std::thread::Builder::new() - .name("device-runner".into()) - .spawn(|| panic!("unsupported op: plane.elect")) - .unwrap() - .join(); - assert!(outcome.is_err()); - - let events = session.events.lock().unwrap(); - let logged = events.iter().any(|event| match event { - Event::Log { record, .. } => { - record.level == LogLevel::Error - && record.message.contains("unsupported op: plane.elect") - && record.message.contains("device-runner") - } - _ => false, - }); - assert!(logged, "no error log carried the panic: {events:?}"); - drop(events); - - // The hook pushes what it just recorded out of the process while it - // still can; an aborting teardown must not be able to eat it. - assert!( - *session.flushes.lock().unwrap() > 0, - "the hook never flushed" - ); - } - #[test] fn run_context_records_events() { let session = Arc::new(MockSession::default()); diff --git a/crates/tracel-experiment/src/panic_watch.rs b/crates/tracel-experiment/src/panic_watch.rs index bd627c0a..383bceec 100644 --- a/crates/tracel-experiment/src/panic_watch.rs +++ b/crates/tracel-experiment/src/panic_watch.rs @@ -109,3 +109,41 @@ fn describe(info: &PanicHookInfo<'_>) -> String { None => format!("thread '{name}' panicked: {payload}"), } } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::LogLevel; + use crate::test_support::{MockRun, mock_run}; + + const PANIC: &str = "unsupported op: plane.elect"; + + fn panic_on_another_thread() { + std::thread::spawn(|| panic!("{PANIC}")).join().unwrap_err(); + } + + #[rstest] + fn a_watched_run_logs_a_panic_on_another_thread_as_an_error(mock_run: MockRun) { + let _watch = mock_run.run.capture_panics(); + + panic_on_another_thread(); + + assert!( + mock_run + .session + .logs() + .iter() + .any(|record| record.level == LogLevel::Error && record.message.ends_with(PANIC)) + ); + } + + #[rstest] + fn a_watched_run_flushes_the_panic_it_logged(mock_run: MockRun) { + let _watch = mock_run.run.capture_panics(); + + panic_on_another_thread(); + + assert!(*mock_run.session.flushes.lock().unwrap() > 0); + } +} From f4bea28259025d7af12565f2c798894f94bf23a2 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:44:45 -0400 Subject: [PATCH 82/90] test: a record's own attribute winning is tested on the record scoped_handle_inherits_attributes_and_call_site_wins asserted the level, the message, an inherited attribute, an overridden one and a third, two of them under comments. Which attribute wins is LogRecord::inherit_attrs's rule, now tested in log.rs, and the lib.rs test checks only that a log through a scoped handle inherits its attributes. --- crates/tracel-experiment/src/lib.rs | 44 ++++++----------------------- crates/tracel-experiment/src/log.rs | 16 +++++++++++ 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/crates/tracel-experiment/src/lib.rs b/crates/tracel-experiment/src/lib.rs index 8d4334a0..54c65691 100644 --- a/crates/tracel-experiment/src/lib.rs +++ b/crates/tracel-experiment/src/lib.rs @@ -1025,44 +1025,16 @@ mod tests { assert_eq!(session.events.lock().unwrap().len(), recorded); } - #[test] - fn scoped_handle_inherits_attributes_and_call_site_wins() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); + #[rstest] + fn a_log_through_a_scoped_handle_inherits_its_attributes(mock_run: MockRun) { + let scoped = mock_run.run.handle().with_attr("phase", "train"); - let scoped = run - .handle() - .with_attr("phase", "train") - .with_attr("shard", 1i64); - scoped.log( - LogRecord::warn("slow step") - .with("shard", 2i64) - .with("elapsed_ms", 900i64), - ); + scoped.log_warn("slow step"); - let events = session.events.lock().unwrap(); - assert_eq!(events.len(), 1); - match &events[0] { - Event::Log { record, .. } => { - assert_eq!(record.level, LogLevel::Warn); - assert_eq!(record.message, "slow step"); - // Inherited from scope. - assert_eq!( - record.attributes.get("phase").and_then(|v| v.as_str()), - Some("train") - ); - // Call-site value overrides the inherited scope value. - assert_eq!( - record.attributes.get("shard").and_then(|v| v.as_i64()), - Some(2) - ); - assert_eq!( - record.attributes.get("elapsed_ms").and_then(|v| v.as_i64()), - Some(900) - ); - } - event => panic!("unexpected event: {event:?}"), - } + assert_eq!( + mock_run.session.logs()[0].attributes.get("phase"), + Some(&serde_json::json!("train")) + ); } #[test] diff --git a/crates/tracel-experiment/src/log.rs b/crates/tracel-experiment/src/log.rs index e661abdf..4f73f038 100644 --- a/crates/tracel-experiment/src/log.rs +++ b/crates/tracel-experiment/src/log.rs @@ -104,3 +104,19 @@ impl LogRecord { } } } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn an_attribute_of_the_record_wins_over_an_inherited_one() { + let mut record = LogRecord::warn("slow step").with("shard", 2); + + record.inherit_attrs(&Map::from_iter([("shard".to_string(), json!(1))])); + + assert_eq!(record.attributes.get("shard"), Some(&json!(2))); + } +} From baa41a2cd2546d7091c6717d3f6975e04f62f2ea Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:46:02 -0400 Subject: [PATCH 83/90] test: run tests assert one outcome each, with rstest cases The run tests reached into events by position, matched panic reasons on the hook's format and thread name, and asserted several outcomes at once through a helper that returned each log's level, message, attribute count and activity. Each test now checks one outcome on the mock_run fixture: telemetry through an activity is one case per kind, a handle of a finished or dropped run is inactive and discards logs, a cancelled run ends as it is ended, and a stop request is logged once, as a message. run_context_records_events goes: every test that reads a log back shows it. --- crates/tracel-experiment/src/lib.rs | 335 +++++++++++----------------- 1 file changed, 132 insertions(+), 203 deletions(-) diff --git a/crates/tracel-experiment/src/lib.rs b/crates/tracel-experiment/src/lib.rs index 54c65691..71e00ee1 100644 --- a/crates/tracel-experiment/src/lib.rs +++ b/crates/tracel-experiment/src/lib.rs @@ -879,150 +879,94 @@ impl Drop for ExperimentRun { #[cfg(test)] mod tests { use rstest::rstest; + use serde_json::json; - use crate::activity::ActivityEvent; use crate::test_support::{ MockRun, MockSession, create_run, create_run_with_control, mock_run, }; use super::*; - #[test] - fn panicking_run_reports_the_panic_as_its_failure_reason() { - let session = Arc::new(MockSession::default()); - let run_session = session.clone(); + const PANIC: &str = "kernel exploded mid-batch"; + const REASON: &str = "interrupted"; - let outcome = std::thread::Builder::new() - .name("fold-runner".into()) - .spawn(move || { - let _run = create_run(run_session); - panic!("kernel exploded mid-batch"); - }) - .unwrap() - .join(); - assert!(outcome.is_err()); - - let completions = session.completions.lock().unwrap(); - match completions.as_slice() { - [ExperimentCompletion::Failed(reason)] => { - assert!(reason.contains("kernel exploded mid-batch"), "{reason}"); - assert!(reason.contains("panicked at"), "{reason}"); - assert!(reason.contains("fold-runner"), "{reason}"); - } - other => panic!("unexpected completions: {other:?}"), + fn metrics() -> Vec { + vec![MetricValue { + name: "mean_score".to_string(), + value: 0.8, + }] + } + + fn recorded_activity(event: &Event) -> Option { + match event { + Event::Log { activity, .. } + | Event::Metrics { activity, .. } + | Event::EpochSummary { activity, .. } + | Event::Summary { activity, .. } => *activity, + event => panic!("{event:?} is not telemetry"), } } #[test] - fn run_context_records_events() { + fn a_run_unwound_by_a_panic_fails_with_the_panic() { let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); + let run_session = session.clone(); - run.log_info("hello"); + std::thread::spawn(move || { + let _run = create_run(run_session); + panic!("{PANIC}"); + }) + .join() + .unwrap_err(); - let events = session.events.lock().unwrap(); - assert_eq!(events.len(), 1); - match &events[0] { - Event::Log { record, .. } => assert_eq!(record.message, "hello"), - event => panic!("unexpected event: {event:?}"), - } + assert!(matches!( + session.completions.lock().unwrap().as_slice(), + [ExperimentCompletion::Failed(reason)] if reason.ends_with(PANIC) + )); } - #[test] - fn telemetry_records_the_emitting_activity_scope() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let activity = run.activity("cross-validation").start(); - let items = || { - vec![MetricValue { - name: "mean_score".to_string(), - value: 0.8, - }] - }; + #[rstest] + #[case::log(|activity: &Activity| activity.log_info("in scope"))] + #[case::metric(|activity: &Activity| activity.log_metric(1, "train", 1, metrics()))] + #[case::epoch_summary(|activity: &Activity| activity.log_epoch_summary(1, "train", metrics()))] + #[case::summary(|activity: &Activity| activity.log_summary(metrics()))] + fn telemetry_through_an_activity_records_the_activity( + mock_run: MockRun, + #[case] emit: fn(&Activity), + ) { + let guard = mock_run.run.activity("cross-validation").start(); - activity.log_info("in scope"); - activity.log_metric(1, "train", 1, items()); - activity.log_epoch_summary(1, "train", items()); - activity.log_summary(items()); - run.log_info("out of scope"); + emit(&guard); - let events = session.events.lock().unwrap(); - let event_activity = |event: &Event| match event { - Event::Log { activity, .. } - | Event::Metrics { activity, .. } - | Event::EpochSummary { activity, .. } - | Event::Summary { activity, .. } => *activity, - event => panic!("unexpected event: {event:?}"), - }; - assert!( - events[1..=4] - .iter() - .all(|event| event_activity(event) == Some(activity.id())) - ); - assert_eq!(event_activity(&events[5]), None); + let events = mock_run.session.events.lock().unwrap(); + assert_eq!(recorded_activity(events.last().unwrap()), Some(guard.id())); } - #[test] - fn child_activity_from_reference_uses_reference_as_parent() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let parent = run.activity("parent").start(); - let activity = parent.share(); + #[rstest] + fn an_activity_started_from_another_is_its_child(mock_run: MockRun) { + let parent = mock_run.run.activity("parent").start(); - let _child = activity.activity("child").start(); + let _child = parent.share().activity("child").start(); - let events = session.events.lock().unwrap(); - let Event::Activity(ActivityEvent::Started { activity: child }) = &events[1] else { - panic!("unexpected event: {:?}", events[1]); - }; - assert_eq!(child.parent, Some(parent.id())); + let started = mock_run.session.started_activities(); + assert_eq!(started.last().unwrap().parent, Some(parent.id())); } - #[test] - fn activity_reference_preserves_inherited_log_attributes() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let guard = run + #[rstest] + fn an_activity_keeps_the_attributes_of_the_handle_it_started_from(mock_run: MockRun) { + let guard = mock_run + .run .handle() .with_attr("pipeline", "cv") .activity("fold") .start(); - let activity = guard.share().with_attr("fold", 1i64); - activity.log(LogRecord::info("started").with("fold", 2i64)); + guard.share().log_info("started"); - let events = session.events.lock().unwrap(); - let Event::Log { - record, - activity: Some(id), - } = &events[1] - else { - panic!("unexpected event: {:?}", events[1]); - }; - assert_eq!(*id, guard.id()); assert_eq!( - record.attributes.get("pipeline"), - Some(&serde_json::json!("cv")) + mock_run.session.logs()[0].attributes.get("pipeline"), + Some(&json!("cv")) ); - assert_eq!(record.attributes.get("fold"), Some(&serde_json::json!(2))); - } - - #[test] - fn activity_reference_discards_telemetry_after_the_run_is_dropped() { - let session = Arc::new(MockSession::default()); - let activity = { - let run = create_run(session.clone()); - let guard = run.activity("fold").start(); - let activity = guard.share(); - guard.finish(); - activity - }; - let recorded = session.events.lock().unwrap().len(); - - assert!(!activity.is_active()); - activity.log_info("late"); - - assert_eq!(session.events.lock().unwrap().len(), recorded); } #[rstest] @@ -1033,79 +977,82 @@ mod tests { assert_eq!( mock_run.session.logs()[0].attributes.get("phase"), - Some(&serde_json::json!("train")) + Some(&json!("train")) ); } - #[test] - fn finish_marks_handle_inactive_and_discards_later_telemetry() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let handle = run.handle(); + #[rstest] + #[case::finished(|run: ExperimentRun| run.finish().unwrap())] + #[case::dropped(drop)] + fn a_handle_of_a_run_that_ended_is_inactive(mock_run: MockRun, #[case] end: fn(ExperimentRun)) { + let handle = mock_run.run.handle(); - run.finish().unwrap(); + end(mock_run.run); assert!(!handle.is_active()); - handle.log_info("after-finish"); - assert!(session.events.lock().unwrap().is_empty()); } - #[test] - fn a_run_dropped_before_it_finishes_fails() { - let session = Arc::new(MockSession::default()); - let finished = create_run(session.clone()); + #[rstest] + #[case::finished(|run: ExperimentRun| run.finish().unwrap())] + #[case::dropped(drop)] + fn a_log_through_a_handle_of_a_run_that_ended_is_discarded( + mock_run: MockRun, + #[case] end: fn(ExperimentRun), + ) { + let handle = mock_run.run.handle(); + end(mock_run.run); - drop(create_run(session.clone())); - finished.finish().unwrap(); + handle.log_info("late"); - let completions = session.completions.lock().unwrap(); - assert_eq!( - completions.as_slice(), - &[ - ExperimentCompletion::Failed(DROPPED_UNFINISHED.to_string()), - ExperimentCompletion::Success, - ] - ); + assert!(mock_run.session.logs().is_empty()); } - #[test] - fn a_cancelled_run_ends_by_how_it_is_finished() { - let session = Arc::new(MockSession::default()); + #[rstest] + fn an_activity_started_from_a_handle_of_a_dropped_run_records_nothing(mock_run: MockRun) { + let handle = mock_run.run.handle(); + drop(mock_run.run); + + let _late = handle.activity("late").meter(1, "items").start(); + + assert!(mock_run.session.events.lock().unwrap().is_empty()); + } - let completed = create_run(session.clone()); - completed.cancel().unwrap(); - completed.finish().unwrap(); - let failed = create_run(session.clone()); - failed.cancel().unwrap(); - failed.fail("interrupted").unwrap(); + #[rstest] + fn a_run_dropped_before_it_finishes_fails(mock_run: MockRun) { + drop(mock_run.run); + + assert!(matches!( + mock_run.session.completions.lock().unwrap().as_slice(), + [ExperimentCompletion::Failed(_)] + )); + } + + #[rstest] + fn a_finished_run_ends_once_as_completed(mock_run: MockRun) { + mock_run.run.finish().unwrap(); - let completions = session.completions.lock().unwrap(); assert_eq!( - completions.as_slice(), - &[ - ExperimentCompletion::Success, - ExperimentCompletion::Failed("interrupted".to_string()), - ] + *mock_run.session.completions.lock().unwrap(), + [ExperimentCompletion::Success] ); } - /// The level, message, attributes and activity of each log `session` recorded. - fn logs(session: &MockSession) -> Vec<(LogLevel, String, usize, Option)> { - session - .events - .lock() - .unwrap() - .iter() - .filter_map(|event| match event { - Event::Log { record, activity } => Some(( - record.level, - record.message.clone(), - record.attributes.len(), - *activity, - )), - _ => None, - }) - .collect() + #[rstest] + #[case::finished(|run: ExperimentRun| run.finish().unwrap(), ExperimentCompletion::Success)] + #[case::failed( + |run: ExperimentRun| run.fail(REASON).unwrap(), + ExperimentCompletion::Failed(REASON.to_string()) + )] + fn a_cancelled_run_ends_as_it_is_ended( + mock_run: MockRun, + #[case] end: fn(ExperimentRun), + #[case] completion: ExperimentCompletion, + ) { + mock_run.run.cancel().unwrap(); + + end(mock_run.run); + + assert_eq!(*mock_run.session.completions.lock().unwrap(), [completion]); } #[rstest] @@ -1120,22 +1067,22 @@ mod tests { ); } - #[test] - fn a_stop_request_is_logged_once_while_the_run_is_active() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let activity = run.activity("epoch 1").cancellable().start(); + #[rstest] + fn cancelling_an_activity_logs_no_stop_request(mock_run: MockRun) { + let activity = mock_run.run.activity("epoch 1").cancellable().start(); activity.cancel_token().cancel(); - assert!(logs(&session).is_empty()); - run.cancel_token().cancel(); - run.cancel().unwrap(); - run.cancel_token().cancel(); - assert_eq!( - logs(&session), - [(LogLevel::Warn, STOP_REQUESTED.to_string(), 0, None)] - ); + assert!(mock_run.session.logs().is_empty()); + } + + #[rstest] + fn a_stop_request_is_logged_once_however_often_the_run_is_cancelled(mock_run: MockRun) { + mock_run.run.cancel_token().cancel(); + mock_run.run.cancel().unwrap(); + mock_run.run.cancel_token().cancel(); + + assert_eq!(mock_run.session.log_messages(), [STOP_REQUESTED]); } #[test] @@ -1146,34 +1093,16 @@ mod tests { let _run = create_run_with_control(session.clone(), control); - assert_eq!( - logs(&session), - [(LogLevel::Warn, STOP_REQUESTED.to_string(), 0, None)] - ); + assert_eq!(session.log_messages(), [STOP_REQUESTED]); } - #[test] - fn a_stop_requested_once_the_run_has_finished_is_not_logged() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let cancel_token = run.cancel_token(); + #[rstest] + fn a_stop_requested_once_the_run_has_finished_is_not_logged(mock_run: MockRun) { + let cancel_token = mock_run.run.cancel_token(); + mock_run.run.finish().unwrap(); - run.finish().unwrap(); cancel_token.cancel(); - assert!(logs(&session).is_empty()); - } - - #[test] - fn dropped_run_handle_activity_records_no_event() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let handle = run.handle(); - drop(run); - - let _progress = handle.activity("late").meter(1, "items").start(); - - let events = session.events.lock().unwrap(); - assert!(events.is_empty()); + assert!(mock_run.session.logs().is_empty()); } } From ba37671a7445f2df1c16980075002cd635f5f072 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:47:01 -0400 Subject: [PATCH 84/90] test: provider tests take an rstest fixture and assert one outcome The tests built their fixture by hand with Fixture::new: it is now an rstest fixture. run_ends_the_experiment_by_what_its_code_returns is one case per outcome and checks only how the experiment ended, not that the output and the error pass through. The test that run runs nothing when the experiment does not start no longer matches the backend's message, and the current experiment test leaves the end of the scope to the context tests. --- crates/tracel-experiment/src/provider.rs | 86 ++++++++++-------------- 1 file changed, 35 insertions(+), 51 deletions(-) diff --git a/crates/tracel-experiment/src/provider.rs b/crates/tracel-experiment/src/provider.rs index cdb8e187..60dab194 100644 --- a/crates/tracel-experiment/src/provider.rs +++ b/crates/tracel-experiment/src/provider.rs @@ -162,6 +162,7 @@ impl Experiments { mod tests { use std::sync::Mutex; + use rstest::{fixture, rstest}; use serde_json::json; use super::*; @@ -170,6 +171,8 @@ mod tests { type BoxError = Box; + const REASON: &str = "diverged"; + #[derive(Default)] struct MockProvider { session: Arc, @@ -197,19 +200,10 @@ mod tests { } impl Fixture { - fn new() -> Self { - let provider = Arc::new(MockProvider::default()); - Self { - experiments: Experiments::new(provider.clone()), - provider, - } - } - fn completions(&self) -> Vec { self.provider.session.completions.lock().unwrap().clone() } - /// The arguments the runs recorded, in order. fn arguments(&self) -> Vec { self.provider .session @@ -225,32 +219,30 @@ mod tests { } } - #[test] - fn run_ends_the_experiment_by_what_its_code_returns() { - let fixture = Fixture::new(); - - let output = fixture - .experiments - .run("train", |_run| Ok::<_, BoxError>(3)); - let failed = fixture - .experiments - .run("train", |_run| Err::<(), BoxError>("diverged".into())); - - assert_eq!(output.unwrap(), 3); - assert_eq!(failed.unwrap_err().to_string(), "diverged"); - assert_eq!( - fixture.completions(), - [ - ExperimentCompletion::Success, - ExperimentCompletion::Failed("diverged".to_string()) - ] - ); + #[fixture] + fn fixture() -> Fixture { + let provider = Arc::new(MockProvider::default()); + Fixture { + experiments: Experiments::new(provider.clone()), + provider, + } } - #[test] - fn a_panic_in_run_fails_the_experiment_with_its_message() { - let fixture = Fixture::new(); + #[rstest] + #[case::ok(Ok(()), ExperimentCompletion::Success)] + #[case::err(Err(REASON.into()), ExperimentCompletion::Failed(REASON.to_string()))] + fn run_ends_the_experiment_by_what_its_code_returns( + fixture: Fixture, + #[case] returned: Result<(), BoxError>, + #[case] completion: ExperimentCompletion, + ) { + let _ = fixture.experiments.run("train", |_run| returned); + + assert_eq!(fixture.completions(), [completion]); + } + #[rstest] + fn a_panic_in_run_fails_the_experiment_with_its_message(fixture: Fixture) { let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { fixture .experiments @@ -268,17 +260,14 @@ mod tests { } } - #[test] - fn run_makes_the_run_the_current_experiment() { - let fixture = Fixture::new(); - + #[rstest] + fn run_makes_the_run_the_current_experiment(fixture: Fixture) { let current = fixture.experiments.run("train", |run| { let current = ExperimentRun::current().map(|current| current.id().clone()); Ok::<_, BoxError>(current == Some(run.id().clone())) }); assert!(current.unwrap()); - assert!(ExperimentRun::current().is_none()); } #[test] @@ -299,20 +288,18 @@ mod tests { } let mut ran = false; - let error = Experiments::new(Arc::new(Unreachable)) + Experiments::new(Arc::new(Unreachable)) .run("train", |_run| -> Result<(), BoxError> { ran = true; Ok(()) }) .unwrap_err(); - assert_eq!(error.to_string(), "the backend is unreachable"); assert!(!ran); } - #[test] - fn a_started_experiment_records_the_attributes() { - let fixture = Fixture::new(); + #[rstest] + fn a_started_experiment_records_the_attributes(fixture: Fixture) { let experiments = fixture .experiments .clone() @@ -331,9 +318,8 @@ mod tests { ); } - #[test] - fn only_experiments_given_arguments_record_them_when_a_run_starts() { - let fixture = Fixture::new(); + #[rstest] + fn only_experiments_given_arguments_record_them_when_a_run_starts(fixture: Fixture) { let experiments = fixture.experiments.clone().arguments(json!({"epochs": 2})); experiments.start("train").unwrap().finish().unwrap(); @@ -349,9 +335,8 @@ mod tests { assert_eq!(fixture.arguments(), recorded); } - #[test] - fn arguments_logged_on_a_run_are_recorded_after_those_it_started_with() { - let fixture = Fixture::new(); + #[rstest] + fn arguments_logged_on_a_run_are_recorded_after_those_it_started_with(fixture: Fixture) { let experiments = fixture.experiments.clone().arguments(json!({"epochs": 2})); let run = experiments.start("train").unwrap(); @@ -363,9 +348,8 @@ mod tests { ); } - #[test] - fn each_hook_is_called_with_each_run_started() { - let fixture = Fixture::new(); + #[rstest] + fn each_hook_is_called_with_each_run_started(fixture: Fixture) { let called = Arc::new(Mutex::new(Vec::new())); let hook = |tag: &'static str| { let called = called.clone(); From 7d28cf938d2f88b51243a760a0befefc1ae15b60 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:47:36 -0400 Subject: [PATCH 85/90] test: each ambient scope rule has a test of its own The context tests asserted the whole stack of scopes in one test, the current experiment before and after every await, and that none was left on the test thread, each behind an expect message. Each test now checks one rule through the current experiment's id: an inner scope overrides the outer one, leaving it restores the outer one, leaving the outer one first keeps the inner one, a handle enters its run on another thread, and instrumented tasks keep their own experiment across an await, as does one spawned in the current experiment. --- crates/tracel-experiment/src/context.rs | 175 +++++++++--------------- 1 file changed, 61 insertions(+), 114 deletions(-) diff --git a/crates/tracel-experiment/src/context.rs b/crates/tracel-experiment/src/context.rs index 33ba3d7d..cc7a7d14 100644 --- a/crates/tracel-experiment/src/context.rs +++ b/crates/tracel-experiment/src/context.rs @@ -214,9 +214,9 @@ impl Drop for CurrentExperimentGuard { #[cfg(test)] mod tests { - use crate::ExperimentRun; use crate::context::ExperimentGlobalExt as _; use crate::test_support::create_run_with_id; + use crate::{ExperimentId, ExperimentRun}; use super::ExperimentInstrument; @@ -224,144 +224,91 @@ mod tests { create_run_with_id(id, Default::default()) } - #[test] - fn nested_experiment_scopes_restore_the_previous_experiment() { - let run_a = create_run("ambient-test-a"); - let run_b = create_run("ambient-test-b"); - - assert!(ExperimentRun::current().is_none()); - { - let _outer = run_a.enter(); - let current = ExperimentRun::current().expect("current experiment should be set"); - assert_eq!(current.id(), run_a.id()); - - { - let _inner = run_b.enter(); - let current = - ExperimentRun::current().expect("inner experiment should override outer scope"); - assert_eq!(current.id(), run_b.id()); - } - - let current = ExperimentRun::current().expect("outer experiment should be restored"); - assert_eq!(current.id(), run_a.id()); - } - assert!(ExperimentRun::current().is_none()); + fn current_id() -> Option { + ExperimentRun::current().map(|current| current.id().clone()) } - #[test] - fn dropping_an_outer_scope_does_not_remove_the_inner_scope() { - let run_a = create_run("ambient-test-out-of-order-a"); - let run_b = create_run("ambient-test-out-of-order-b"); - let outer = run_a.enter(); - let inner = run_b.enter(); - - drop(outer); - - let current = ExperimentRun::current().expect("inner experiment should remain current"); - assert_eq!(current.id(), run_b.id()); - drop(inner); - assert!(ExperimentRun::current().is_none()); + async fn current_id_after_an_await() -> Option { + tokio::task::yield_now().await; + current_id() } #[test] - fn handle_with_current_can_be_used_in_spawned_threads() { - let run = create_run("ambient-test-thread"); - let handle = run.handle(); + fn an_inner_scope_overrides_the_outer_one() { + let outer = create_run("outer"); + let inner = create_run("inner"); + let _outer = outer.enter(); - std::thread::spawn(move || { - handle.in_scope(|| { - let current = ExperimentRun::current().expect("current experiment should be set"); - assert_eq!(current.id().as_str(), "ambient-test-thread"); - }) - }) - .join() - .expect("thread should complete successfully"); + let _inner = inner.enter(); - assert!(ExperimentRun::current().is_none()); + assert_eq!(current_id(), Some(inner.id().clone())); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn instrumented_tokio_tasks_keep_multiple_experiments_isolated() { - let run_a = create_run("ambient-test-tokio-a"); - let run_b = create_run("ambient-test-tokio-b"); - - let task_a = tokio::spawn( - async { - let current = ExperimentRun::current().expect("current experiment should be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-a"); + #[test] + fn leaving_an_inner_scope_restores_the_outer_one() { + let outer = create_run("outer"); + let inner = create_run("inner"); + let _outer = outer.enter(); - tokio::task::yield_now().await; + drop(inner.enter()); - let current = - ExperimentRun::current().expect("current experiment should still be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-a"); - } - .in_experiment(&run_a), - ); + assert_eq!(current_id(), Some(outer.id().clone())); + } - let task_b = tokio::spawn( - async { - let current = ExperimentRun::current().expect("current experiment should be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-b"); + #[test] + fn leaving_an_outer_scope_first_keeps_the_inner_one() { + let outer = create_run("outer"); + let inner = create_run("inner"); + let outer_scope = outer.enter(); + let _inner_scope = inner.enter(); - tokio::task::yield_now().await; + drop(outer_scope); - let current = - ExperimentRun::current().expect("current experiment should still be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-b"); - } - .in_experiment(&run_b), - ); - - task_a - .await - .expect("first tokio task should complete successfully"); - task_b - .await - .expect("second tokio task should complete successfully"); - assert!(ExperimentRun::current().is_none()); + assert_eq!(current_id(), Some(inner.id().clone())); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn nested_tokio_spawn_can_use_in_current_experiment() { - let run = create_run("ambient-test-tokio-nested"); - - let outer = tokio::spawn( - async { - let current = ExperimentRun::current().expect("current experiment should be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-nested"); + #[test] + fn a_handle_enters_its_run_on_another_thread() { + let run = create_run("run"); + let handle = run.handle(); - let inner = tokio::spawn( - async { - let current = - ExperimentRun::current().expect("current experiment should be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-nested"); + let current = std::thread::spawn(move || handle.in_scope(current_id)) + .join() + .unwrap(); - tokio::task::yield_now().await; + assert_eq!(current, Some(run.id().clone())); + } - let current = ExperimentRun::current() - .expect("current experiment should still be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-nested"); - } - .in_current_experiment(), - ); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn instrumented_tasks_each_keep_their_own_experiment_across_an_await() { + let first = create_run("first"); + let second = create_run("second"); + + let tasks = [&first, &second] + .map(|run| tokio::spawn(current_id_after_an_await().in_experiment(run))); + let mut seen = Vec::new(); + for task in tasks { + seen.push(task.await.unwrap()); + } - tokio::task::yield_now().await; + assert_eq!(seen, [Some(first.id().clone()), Some(second.id().clone())]); + } - let current = - ExperimentRun::current().expect("outer current experiment should still be set"); - assert_eq!(current.id().as_str(), "ambient-test-tokio-nested"); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_task_spawned_in_the_current_experiment_keeps_it() { + let run = create_run("run"); - inner + let seen = tokio::spawn( + async { + tokio::spawn(current_id_after_an_await().in_current_experiment()) .await - .expect("nested tokio task should complete successfully"); + .unwrap() } .in_experiment(&run), - ); + ) + .await + .unwrap(); - outer - .await - .expect("outer tokio task should complete successfully"); - assert!(ExperimentRun::current().is_none()); + assert_eq!(seen, Some(run.id().clone())); } } From 5165701c00760d2b6a0e06534fc4abbe794dca44 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:48:29 -0400 Subject: [PATCH 86/90] test: each run control test checks one outcome The run control tests asserted both the run's token and an activity's linked one, and a registered activity's cancel both returned true and cancelled its token. Each outcome is now its own test, and whether cancel_activity finds an activity is one rule with a case registered and one unknown. --- crates/tracel-experiment/src/control.rs | 32 ++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/crates/tracel-experiment/src/control.rs b/crates/tracel-experiment/src/control.rs index f3cbd643..cf4195c7 100644 --- a/crates/tracel-experiment/src/control.rs +++ b/crates/tracel-experiment/src/control.rs @@ -86,39 +86,43 @@ impl Default for ExperimentRunControl { mod tests { use std::num::NonZeroU64; + use rstest::rstest; + use super::*; - fn activity_id(value: u64) -> ActivityId { - ActivityId::new(NonZeroU64::new(value).unwrap()) + fn activity_id() -> ActivityId { + ActivityId::new(NonZeroU64::new(1).unwrap()) } #[test] - fn cancel_run_cancels_linked_activity_tokens() { + fn cancelling_the_run_cancels_its_token() { let control = ExperimentRunControl::default(); - let activity_token = control.cancel_token().linked(CancelToken::new()); - control.register_activity_cancellation(activity_id(1), activity_token.clone()); control.cancel_run(); assert!(control.cancel_token().is_cancelled()); - assert!(activity_token.is_cancelled()); } #[test] - fn cancel_activity_cancels_registered_activity_token() { + fn cancelling_a_registered_activity_cancels_its_token() { let control = ExperimentRunControl::default(); - let activity_token = CancelToken::new(); - control.register_activity_cancellation(activity_id(1), activity_token.clone()); + let token = CancelToken::new(); + control.register_activity_cancellation(activity_id(), token.clone()); - assert!(control.cancel_activity(activity_id(1))); + control.cancel_activity(activity_id()); - assert!(activity_token.is_cancelled()); + assert!(token.is_cancelled()); } - #[test] - fn cancel_unknown_activity_returns_false() { + #[rstest] + #[case::registered(true)] + #[case::unknown(false)] + fn cancelling_an_activity_reports_whether_it_was_registered(#[case] registered: bool) { let control = ExperimentRunControl::default(); + if registered { + control.register_activity_cancellation(activity_id(), CancelToken::new()); + } - assert!(!control.cancel_activity(activity_id(99))); + assert_eq!(control.cancel_activity(activity_id()), registered); } } From cbdd1a45937687a4e448e1be89325e14a59aa5bc Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:48:29 -0400 Subject: [PATCH 87/90] test: a token cancels each child once, counted by the child The test child guarded its own cancel, so it counted at most one cancel whatever the token did, and the concurrent test slept so its two cancels would race. The child now counts every cancel it is sent, and the two threads cancel together behind a barrier. The tests are named for the outcome, and the one restating that a cancelled token is cancelled goes. --- crates/tracel-experiment/src/cancellation.rs | 93 ++++++++------------ 1 file changed, 35 insertions(+), 58 deletions(-) diff --git a/crates/tracel-experiment/src/cancellation.rs b/crates/tracel-experiment/src/cancellation.rs index 38a93610..dc934dd7 100644 --- a/crates/tracel-experiment/src/cancellation.rs +++ b/crates/tracel-experiment/src/cancellation.rs @@ -136,105 +136,82 @@ impl Cancellable for Arc { #[cfg(test)] mod tests { - use super::*; - use std::sync::Arc; + use std::sync::Barrier; use std::sync::atomic::AtomicUsize; use std::thread; - use std::time::Duration; - struct TestCancellable { - cancelled: AtomicBool, - cancel_count: AtomicUsize, - } + use super::*; - impl TestCancellable { - fn new() -> Self { - Self { - cancelled: AtomicBool::new(false), - cancel_count: AtomicUsize::new(0), - } - } + #[derive(Default)] + struct CountsCancels(AtomicUsize); - fn cancel_count(&self) -> usize { - self.cancel_count.load(Ordering::Relaxed) + impl CountsCancels { + fn count(&self) -> usize { + self.0.load(Ordering::Acquire) } } - impl Cancellable for TestCancellable { + impl Cancellable for CountsCancels { fn cancel(&self) { - if !self.cancelled.swap(true, Ordering::AcqRel) { - self.cancel_count.fetch_add(1, Ordering::AcqRel); - } + self.0.fetch_add(1, Ordering::AcqRel); } fn is_cancelled(&self) -> bool { - self.cancelled.load(Ordering::Acquire) + self.count() > 0 } } - #[test] - fn test_cancel_token() { - let token = CancelToken::new(); - - token.cancel(); - - assert!(token.is_cancelled()); + fn linked_child(token: &CancelToken) -> Arc { + let child = Arc::new(CountsCancels::default()); + token.link(child.clone()); + child } #[test] - fn test_cancel_children() { + fn cancelling_a_token_cancels_every_child_linked_to_it() { let token = CancelToken::new(); - let child1 = Arc::new(TestCancellable::new()); - let child2 = Arc::new(TestCancellable::new()); - token.link(child1.clone()); - token.link(child2.clone()); + let children = [linked_child(&token), linked_child(&token)]; token.cancel(); - assert!(child1.is_cancelled()); - assert!(child2.is_cancelled()); + assert!(children.iter().all(|child| child.is_cancelled())); } #[test] - fn test_idempotent_cancel() { + fn cancelling_a_token_twice_cancels_its_child_once() { let token = CancelToken::new(); - let child = Arc::new(TestCancellable::new()); - token.link(child.clone()); - token.cancel(); + let child = linked_child(&token); + token.cancel(); token.cancel(); - assert!(token.is_cancelled()); - assert!(child.is_cancelled()); - assert_eq!(child.cancel_count(), 1); + assert_eq!(child.count(), 1); } #[test] - fn test_concurrent_cancel() { + fn cancelling_a_token_from_two_threads_at_once_cancels_its_child_once() { let token = CancelToken::new(); - let child = Arc::new(TestCancellable::new()); - token.link(child.clone()); - let token_clone = token.clone(); - let handle = thread::spawn(move || { - thread::sleep(Duration::from_millis(50)); - token_clone.cancel(); + let child = linked_child(&token); + let barrier = Barrier::new(2); + + thread::scope(|scope| { + for _ in 0..2 { + scope.spawn(|| { + barrier.wait(); + token.cancel(); + }); + } }); - token.cancel(); - - handle.join().unwrap(); - assert!(token.is_cancelled()); - assert!(child.is_cancelled()); - assert_eq!(child.cancel_count(), 1); // Should only be cancelled once + assert_eq!(child.count(), 1); } #[test] - fn test_link_after_cancel() { + fn a_child_linked_to_a_cancelled_token_is_cancelled() { let token = CancelToken::new(); token.cancel(); - let child = Arc::new(TestCancellable::new()); - token.link(child.clone()); + let child = linked_child(&token); assert!(child.is_cancelled()); } From b0fb0d995a0b468bebab2fa6793a75ddd4c69717 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:49:36 -0400 Subject: [PATCH 88/90] test: the tracing layer's rules are tested in its own module The tracing tests lived beside the span helpers, matched each log's message, and checked a span field and an event field overriding it in one test under a comment. The layer's rules, routing to the current experiment or to one a span names, skipping events outside any, and turning event and span fields into attributes with the event's winning, are now tested in layer.rs, one rule each, counting the logs routed. The span helpers keep a test each, through the public tracing_span of a run and of an activity. --- .../src/integration/tracing/layer.rs | 82 ++++++++++ .../src/integration/tracing/mod.rs | 148 ++---------------- crates/tracel-experiment/src/test_support.rs | 6 + 3 files changed, 100 insertions(+), 136 deletions(-) diff --git a/crates/tracel-experiment/src/integration/tracing/layer.rs b/crates/tracel-experiment/src/integration/tracing/layer.rs index a497ab6e..0c609619 100644 --- a/crates/tracel-experiment/src/integration/tracing/layer.rs +++ b/crates/tracel-experiment/src/integration/tracing/layer.rs @@ -224,3 +224,85 @@ impl Visit for JsonFieldVisitor { self.record(field, Value::String(format!("{value:?}"))); } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use rstest::rstest; + use serde_json::json; + + use crate::ExperimentGlobalExt; + use crate::test_support::{MockRun, MockSession, create_run_with_id, mock_run, traced}; + + #[rstest] + fn an_event_in_the_current_experiment_is_logged_to_it(mock_run: MockRun) { + traced(|| mock_run.run.in_scope(|| tracing::info!("epoch completed"))); + + assert_eq!(mock_run.session.logs().len(), 1); + } + + #[test] + fn an_event_in_a_span_naming_an_experiment_is_logged_to_it() { + let session = Arc::new(MockSession::default()); + let run = create_run_with_id("layer-span", session.clone()); + + traced(|| { + tracing::info_span!("experiment", experiment_id = %run.id()) + .in_scope(|| tracing::info!("epoch completed")) + }); + + assert_eq!(session.logs().len(), 1); + } + + #[rstest] + fn an_event_outside_any_experiment_is_not_logged(mock_run: MockRun) { + traced(|| tracing::info!("outside any experiment")); + + assert!(mock_run.session.logs().is_empty()); + } + + #[rstest] + fn an_event_field_is_an_attribute_of_its_log(mock_run: MockRun) { + traced(|| { + mock_run + .run + .in_scope(|| tracing::info!(step = 3u64, "epoch completed")) + }); + + assert_eq!( + mock_run.session.logs()[0].attributes.get("step"), + Some(&json!(3)) + ); + } + + #[rstest] + fn a_span_field_is_an_attribute_of_the_logs_inside_it(mock_run: MockRun) { + traced(|| { + mock_run.run.in_scope(|| { + tracing::info_span!("phase", stage = "train") + .in_scope(|| tracing::info!("step done")) + }) + }); + + assert_eq!( + mock_run.session.logs()[0].attributes.get("stage"), + Some(&json!("train")) + ); + } + + #[rstest] + fn an_event_field_wins_over_a_span_field_of_the_same_name(mock_run: MockRun) { + traced(|| { + mock_run.run.in_scope(|| { + tracing::info_span!("phase", shard = 2u64) + .in_scope(|| tracing::info!(shard = 5u64, "step done")) + }) + }); + + assert_eq!( + mock_run.session.logs()[0].attributes.get("shard"), + Some(&json!(5)) + ); + } +} diff --git a/crates/tracel-experiment/src/integration/tracing/mod.rs b/crates/tracel-experiment/src/integration/tracing/mod.rs index 74ffa367..05ab674f 100644 --- a/crates/tracel-experiment/src/integration/tracing/mod.rs +++ b/crates/tracel-experiment/src/integration/tracing/mod.rs @@ -125,158 +125,34 @@ impl ActivityTracingExt for Activity { mod tests { use std::sync::Arc; - use tracing_subscriber::layer::SubscriberExt; - - use crate::context::ExperimentGlobalExt; - use crate::session::Event; - use crate::test_support::{MockSession, create_run_with_id}; - use super::*; + use crate::test_support::{MockSession, create_run_with_id, traced}; #[test] - fn tracing_layer_forwards_events_to_current_experiment() { + fn an_event_in_an_experiment_tracing_span_is_logged_to_that_experiment() { let session = Arc::new(MockSession::default()); - let run = create_run_with_id("trace-test-1", session.clone()); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); + let run = create_run_with_id("tracing-experiment-span", session.clone()); - tracing::subscriber::with_default(subscriber, || { - run.in_scope(|| { - tracing::info!(step = 3u64, "epoch completed"); - }) + traced(|| { + run.tracing_span() + .in_scope(|| tracing::info!("epoch completed")) }); - let events = session.events.lock().unwrap(); - assert_eq!(events.len(), 1); - match &events[0] { - Event::Log { record, .. } => { - assert!(record.message.contains("epoch completed")); - assert_eq!( - record.attributes.get("step").and_then(|v| v.as_u64()), - Some(3) - ); - } - event => panic!("unexpected event: {event:?}"), - } + assert_eq!(session.logs().len(), 1); } #[test] - fn activity_span_routes_logs_to_its_activity() { + fn an_event_in_an_activity_tracing_span_is_logged_to_that_activity() { let session = Arc::new(MockSession::default()); - let run = create_run_with_id("trace-test-activity-span", session.clone()); + let run = create_run_with_id("tracing-activity-span", session.clone()); let activity = run.activity("fold").start(); - let activity_id = activity.id(); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); - tracing::subscriber::with_default(subscriber, || { + traced(|| { activity .tracing_span() - .in_scope(|| tracing::info!("training fold")); - }); - - let events = session.events.lock().unwrap(); - assert!(events.iter().any(|event| matches!( - event, - Event::Log { - activity: Some(id), - .. - } if *id == activity_id - ))); - } - - #[test] - fn tracing_layer_routes_from_span_experiment_id_without_ambient_scope() { - let session = Arc::new(MockSession::default()); - let run = create_run_with_id("trace-test-span", session.clone()); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); - - tracing::subscriber::with_default(subscriber, || { - let span = tracing::info_span!("experiment", experiment_id = %run.id()); - let _guard = span.enter(); - tracing::info!(step = 7u64, "span-routed event"); - }); - - let events = session.events.lock().unwrap(); - assert_eq!(events.len(), 1); - match &events[0] { - Event::Log { record, .. } => { - assert!(record.message.contains("span-routed event")); - assert_eq!( - record.attributes.get("step").and_then(|v| v.as_u64()), - Some(7) - ); - } - event => panic!("unexpected event: {event:?}"), - } - } - - #[test] - fn tracing_layer_routes_from_experiment_span_helper_without_ambient_scope() { - let session = Arc::new(MockSession::default()); - let run = create_run_with_id("trace-test-helper-span", session.clone()); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); - - tracing::subscriber::with_default(subscriber, || { - let span = experiment_span(&run); - let _guard = span.enter(); - tracing::info!("helper-span-routed event"); - }); - - let events = session.events.lock().unwrap(); - assert_eq!(events.len(), 1); - match &events[0] { - Event::Log { record, .. } => { - assert!(record.message.contains("helper-span-routed event")); - } - event => panic!("unexpected event: {event:?}"), - } - } - - #[test] - fn tracing_layer_skips_events_without_experiment_scope() { - let session = Arc::new(MockSession::default()); - let _run = create_run_with_id("trace-test-2", session.clone()); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); - - tracing::subscriber::with_default(subscriber, || { - tracing::info!("outside experiment scope"); - }); - - let events = session.events.lock().unwrap(); - assert!(events.is_empty()); - } - - #[test] - fn tracing_layer_inherits_span_field_attributes() { - let session = Arc::new(MockSession::default()); - let run = create_run_with_id("trace-test-scope", session.clone()); - let subscriber = tracing_subscriber::registry().with(tracing_log_layer()); - - tracing::subscriber::with_default(subscriber, || { - run.in_scope(|| { - let span = tracing::info_span!("phase", stage = "train", shard = 2u64); - span.in_scope(|| { - // The event's own field overrides the inherited `shard` from the span. - tracing::info!(shard = 5u64, "step done"); - }); - }) + .in_scope(|| tracing::info!("training fold")) }); - let events = session.events.lock().unwrap(); - let log = events - .iter() - .find_map(|event| match event { - Event::Log { record, .. } => Some(record), - _ => None, - }) - .expect("a log event should have been recorded"); - assert_eq!(log.message, "step done"); - assert_eq!( - log.attributes.get("stage").and_then(|v| v.as_str()), - Some("train") - ); - assert_eq!( - log.attributes.get("shard").and_then(|v| v.as_u64()), - Some(5) - ); + assert_eq!(session.log_activities(), [Some(activity.id())]); } } diff --git a/crates/tracel-experiment/src/test_support.rs b/crates/tracel-experiment/src/test_support.rs index a5353dc4..b427ac8e 100644 --- a/crates/tracel-experiment/src/test_support.rs +++ b/crates/tracel-experiment/src/test_support.rs @@ -1,9 +1,11 @@ use std::sync::{Arc, Mutex}; use rstest::fixture; +use tracing_subscriber::layer::SubscriberExt; use crate::activity::{ActivityEvent, ActivityId, ActivitySpec}; use crate::error::ExperimentError; +use crate::integration::tracing::tracing_log_layer; use crate::reader::{ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact}; use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; use crate::{ArtifactKind, ExperimentId, ExperimentRun, ExperimentRunControl, LogRecord}; @@ -147,3 +149,7 @@ pub fn mock_run() -> MockRun { session, } } + +pub fn traced(f: impl FnOnce() -> T) -> T { + tracing::subscriber::with_default(tracing_subscriber::registry().with(tracing_log_layer()), f) +} From ef5a51466ec02716381feda7ea41ca6f063076ef Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 12:51:13 -0400 Subject: [PATCH 89/90] test: the progress loggers are tested one rule at a time The progress tests read the started activities by position and asserted every name, parent, meter and attribute at once, and the endings test listed four statuses under a comment each. Each rule now has its own test, finding activities by the names the test gives them: how the training and evaluation activities nest, what their meters count, the epoch an epoch activity numbers, and how a split and the training end, one case per outcome. --- .../src/integration/training/progress.rs | 280 +++++++++--------- 1 file changed, 146 insertions(+), 134 deletions(-) diff --git a/crates/tracel-experiment/src/integration/training/progress.rs b/crates/tracel-experiment/src/integration/training/progress.rs index 2a162cc6..9e05bcd1 100644 --- a/crates/tracel-experiment/src/integration/training/progress.rs +++ b/crates/tracel-experiment/src/integration/training/progress.rs @@ -243,166 +243,178 @@ impl EvaluationProgressLogger for ExperimentEvaluationProgressLogger { #[cfg(test)] mod tests { - use std::sync::Arc; - - use crate::{ - activity::ActivityEvent, - integration::training::ExperimentTrainingExt, - session::Event, - test_support::{MockSession, create_run}, - }; + use rstest::rstest; + use serde_json::json; use super::*; + use crate::integration::training::ExperimentTrainingExt; + use crate::test_support::{MockRun, MockSession, mock_run}; + use crate::{ActivityEvent, ActivitySpec, ActivityStatus}; + + const TRAINING: &str = "Pretraining"; + const EVALUATION: &str = "Holdout"; + const EPOCHS: usize = 2; + const TESTS: usize = 2; + const ITEMS: usize = 10; + const RESUMED_AT: usize = 3; + + fn started(session: &MockSession, name: &str) -> ActivitySpec { + session + .started_activities() + .into_iter() + .find(|activity| activity.name == name) + .unwrap() + } - #[test] - fn training_progress_groups_splits_under_supplied_scope() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let parent = run.activity("Fold 1").start(); - let parent_id = parent.id(); - let mut logger = parent.training_progress_logger(); - - logger.start(2, 1, None, None); - logger.start_split("train", 10); - logger.update_split(4); - logger.end_split(); - logger.start_split("valid", 5); - logger.end_split(); - logger.update_epoch(1); - logger.end(); + fn epoch_of(session: &MockSession, split: &str) -> ActivitySpec { + let split = started(session, split); + session + .started_activities() + .into_iter() + .find(|activity| Some(activity.id) == split.parent) + .unwrap() + } - let events = session.events.lock().unwrap(); - let started = events - .iter() - .filter_map(|event| match event { - Event::Activity(ActivityEvent::Started { activity }) => Some(activity), + fn ending(session: &MockSession, name: &str) -> Option { + let id = started(session, name).id; + session + .activity_events() + .into_iter() + .find_map(|event| match event { + ActivityEvent::Finished { + id: finished, + status, + .. + } if finished == id => Some(status), _ => None, }) - .collect::>(); + } + + fn total(session: &MockSession, name: &str) -> Option { + started(session, name).meter.and_then(|meter| meter.total) + } + + #[rstest] + fn the_training_activity_takes_the_name_of_its_logger(mock_run: MockRun) { + let mut logger = mock_run.run.training_progress_logger().with_name(TRAINING); - assert_eq!(started.len(), 5); - let training = started[1]; - let epoch = started[2]; - let train = started[3]; - let valid = started[4]; + logger.start(EPOCHS, 1, None, None); - assert_eq!(training.name, "Training"); - assert_eq!(training.parent, Some(parent_id)); + assert_eq!(mock_run.session.started_activities()[0].name, TRAINING); + } + + #[rstest] + fn a_split_nests_under_its_epoch_under_the_training_under_its_scope(mock_run: MockRun) { + let fold = mock_run.run.activity("Fold 1").start(); + let mut logger = fold.training_progress_logger().with_name(TRAINING); + + logger.start(EPOCHS, 1, None, None); + logger.start_split("train", ITEMS); + + let training = started(&mock_run.session, TRAINING); + let epoch = epoch_of(&mock_run.session, "train"); assert_eq!( - training.meter.as_ref().unwrap().unit.as_deref(), - Some("epochs") + (training.parent, epoch.parent), + (Some(fold.id()), Some(training.id)) ); - assert_eq!(training.meter.as_ref().unwrap().total, Some(2)); + } + + #[rstest] + fn the_training_is_metered_by_its_epochs_and_a_split_by_its_items(mock_run: MockRun) { + let mut logger = mock_run.run.training_progress_logger().with_name(TRAINING); + + logger.start(EPOCHS, 1, None, None); + logger.start_split("train", ITEMS); - assert_eq!(epoch.name, "Epoch 1"); - assert_eq!(epoch.parent, Some(training.id)); - assert!(epoch.meter.is_none()); assert_eq!( - epoch.attributes.get("activity_type"), - Some(&serde_json::json!("epoch")) + ( + total(&mock_run.session, TRAINING), + total(&mock_run.session, "train") + ), + (Some(EPOCHS as u64), Some(ITEMS as u64)) ); - assert_eq!(epoch.attributes.get("epoch"), Some(&serde_json::json!(1))); + } + + #[rstest] + fn an_epoch_activity_numbers_its_epoch_from_where_the_training_started(mock_run: MockRun) { + let mut logger = mock_run.run.training_progress_logger().with_name(TRAINING); - assert_eq!(train.name, "train"); - assert_eq!(train.parent, Some(epoch.id)); - assert_eq!(train.meter.as_ref().unwrap().unit.as_deref(), Some("steps")); - assert_eq!(train.meter.as_ref().unwrap().total, Some(10)); + logger.start(EPOCHS, RESUMED_AT, None, None); + logger.start_split("train", ITEMS); - assert_eq!(valid.name, "valid"); - assert_eq!(valid.parent, Some(epoch.id)); - assert_eq!(valid.meter.as_ref().unwrap().unit.as_deref(), Some("steps")); - assert_eq!(valid.meter.as_ref().unwrap().total, Some(5)); + assert_eq!( + epoch_of(&mock_run.session, "train").attributes.get("epoch"), + Some(&json!(RESUMED_AT)) + ); } - #[test] - fn training_progress_uses_supplied_name() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let mut logger = run.training_progress_logger().with_name("Pretraining"); + #[rstest] + #[case::through_its_items(ITEMS, ActivityStatus::Success)] + #[case::short_of_its_items(ITEMS - 1, ActivityStatus::Abandoned)] + fn a_split_ends_by_how_far_its_meter_got( + mock_run: MockRun, + #[case] processed: usize, + #[case] status: ActivityStatus, + ) { + let mut logger = mock_run.run.training_progress_logger().with_name(TRAINING); + logger.start(EPOCHS, 1, None, None); + logger.start_split("train", ITEMS); - logger.start(1, 1, None, None); - logger.end(); + logger.update_split(processed); + logger.end_split(); - let names = session - .activity_events() - .into_iter() - .filter_map(|event| match event { - ActivityEvent::Started { activity } => Some(activity.name), - _ => None, - }) - .collect::>(); - assert_eq!(names, ["Pretraining"]); + assert_eq!(ending(&mock_run.session, "train"), Some(status)); } - #[test] - fn endings_follow_the_meter() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let parent = run.activity("Fold 6").start(); - let mut logger = parent.training_progress_logger(); + #[rstest] + #[case::every_epoch(EPOCHS, ActivityStatus::Success)] + #[case::short_of_its_epochs(EPOCHS - 1, ActivityStatus::Abandoned)] + fn the_training_ends_by_how_many_epochs_it_completed( + mock_run: MockRun, + #[case] completed: usize, + #[case] status: ActivityStatus, + ) { + let mut logger = mock_run.run.training_progress_logger().with_name(TRAINING); + logger.start(EPOCHS, 1, None, None); - logger.start(2, 1, None, None); - logger.start_split("train", 10); - logger.update_split(10); - logger.end_split(); - logger.start_split("valid", 10); - logger.update_split(3); - logger.end_split(); + logger.update_epoch(completed); logger.end(); - let completions = session - .activity_events() - .into_iter() - .filter_map(|event| match event { - ActivityEvent::Finished { id, status, .. } => Some((id, status)), - _ => None, - }) - .collect::>(); - - let statuses: Vec<_> = completions.iter().map(|(_, s)| s.clone()).collect(); - assert!( - matches!( - statuses.as_slice(), - [ - crate::ActivityStatus::Success, // the train split ran dry - crate::ActivityStatus::Abandoned, // the valid split stopped at 3 of 10 - crate::ActivityStatus::Abandoned, // the epoch never completed - crate::ActivityStatus::Abandoned, // 0 of 2 epochs is not a finished training - ] - ), - "unexpected endings: {statuses:?}" + assert_eq!(ending(&mock_run.session, TRAINING), Some(status)); + } + + #[rstest] + fn a_test_nests_under_the_evaluation_under_its_scope(mock_run: MockRun) { + let fold = mock_run.run.activity("Fold 1").start(); + let mut logger = fold.evaluation_progress_logger().with_name(EVALUATION); + + logger.start_global_progress(TESTS); + logger.start_test("test", ITEMS); + + let evaluation = started(&mock_run.session, EVALUATION); + let test = started(&mock_run.session, "test"); + assert_eq!( + (evaluation.parent, test.parent), + (Some(fold.id()), Some(evaluation.id)) ); } - #[test] - fn evaluation_progress_groups_tests_under_supplied_scope() { - let session = Arc::new(MockSession::default()); - let run = create_run(session.clone()); - let parent = run.activity("Fold 1").start(); - let mut logger = parent.evaluation_progress_logger().with_name("Holdout"); - - logger.start_global_progress(2); - logger.start_test("test", 4); - logger.update_test_progress(2); - logger.end_test(); - logger.end_global_progress(); - - let events = session.events.lock().unwrap(); - let started = events - .iter() - .filter_map(|event| match event { - Event::Activity(ActivityEvent::Started { activity }) => Some(activity), - _ => None, - }) - .collect::>(); - - assert_eq!(started.len(), 3); - let evaluation = started[1]; - let test = started[2]; - assert_eq!(evaluation.name, "Holdout"); - assert_eq!(evaluation.parent, Some(parent.id())); - assert_eq!(evaluation.meter.as_ref().unwrap().total, Some(2)); - assert_eq!(test.parent, Some(evaluation.id)); - assert_eq!(test.meter.as_ref().unwrap().total, Some(4)); + #[rstest] + fn the_evaluation_is_metered_by_its_tests_and_a_test_by_its_items(mock_run: MockRun) { + let mut logger = mock_run + .run + .evaluation_progress_logger() + .with_name(EVALUATION); + + logger.start_global_progress(TESTS); + logger.start_test("test", ITEMS); + + assert_eq!( + ( + total(&mock_run.session, EVALUATION), + total(&mock_run.session, "test") + ), + (Some(TESTS as u64), Some(ITEMS as u64)) + ); } } From eac7fad92e791a2da7371e47f493d7e29b7d5d99 Mon Sep 17 00:00:00 2001 From: Jonathan Richard Date: Thu, 15 Oct 2026 13:55:19 -0400 Subject: [PATCH 90/90] refactor: an experiment run no longer has a location or announces itself A run carried a location, its console page or its offline directory, so that a job's experiment could print where it recorded as it started. The console page was guessed by taking `api/` off the API URL and appending the web app's routes, after asking the console whether the project's owner is a user or an organization, and the offline directory is one the user chose. Nothing read the line, and tracel run gives its own note when it falls back to recording offline. A run no longer says where its backend keeps it, and ExperimentLocation, ProjectScope::owner_kind and Experiments::on_start, which only the line used, are gone. --- README.md | 6 +- crates/tracel-console/src/console.rs | 13 --- crates/tracel-console/src/experiment.rs | 109 +----------------- crates/tracel-experiment/src/lib.rs | 25 ---- crates/tracel-experiment/src/local/mod.rs | 24 +--- .../src/local/test_support.rs | 11 +- crates/tracel-experiment/src/provider.rs | 59 +--------- crates/tracel/src/lib.rs | 10 +- crates/tracel/src/running_job.rs | 49 +------- crates/tracel/src/target.rs | 9 +- 10 files changed, 31 insertions(+), 284 deletions(-) diff --git a/README.md b/README.md index f9e168e6..9e789834 100644 --- a/README.md +++ b/README.md @@ -91,10 +91,8 @@ input, merged onto its default, as its arguments, which the console lists as its The experiments `Target::experiments` builds in a job, on the thread that runs it, are linked to that job, even for runs started on other threads: each one started from them records the job's -number and input, and prints where it records to stderr as it starts, such as -`Recording experiment 42: `, which `tracel run` and a remote job's logs show. Experiments -built another way, such as `ProjectHandle::experiments`, are not linked unless the code links -them, through `Experiments::attribute`, `arguments` and `on_start`. +number and input. Experiments built another way, such as `ProjectHandle::experiments`, are not +linked unless the code links them, through `Experiments::attribute` and `arguments`. The [`cli`](examples/basics/examples/cli.rs) example in [`examples/basics`](examples/basics) is a complete, runnable version. diff --git a/crates/tracel-console/src/console.rs b/crates/tracel-console/src/console.rs index 6b655a80..1b2f7ac5 100644 --- a/crates/tracel-console/src/console.rs +++ b/crates/tracel-console/src/console.rs @@ -97,8 +97,6 @@ pub struct ProjectScope { console: Console, pub owner: String, pub project: String, - /// The kind of namespace that owns the project, once read from the console. - owner_kind: OnceLock, } impl ProjectScope { @@ -116,19 +114,9 @@ impl ProjectScope { pub fn fetch(&self) -> Result { let project = self.client()?.get_project(&self.owner, &self.project)?; let project = Project::try_from(project)?; - let _ = self.owner_kind.set(project.namespace.kind); Ok(project) } - /// Returns the kind of namespace that owns the project, fetching the project on the first - /// call. - pub fn owner_kind(&self) -> Result { - match self.owner_kind.get() { - Some(kind) => Ok(*kind), - None => self.fetch().map(|project| project.namespace.kind), - } - } - /// Returns the client that moves files to and from presigned URLs. pub fn transfer_client(&self) -> &ReqwestTransferClient { &self.console.inner.transfer_client @@ -241,7 +229,6 @@ impl Console { console: self.clone(), owner: owner.into(), project: project.into(), - owner_kind: OnceLock::new(), }), } } diff --git a/crates/tracel-console/src/experiment.rs b/crates/tracel-console/src/experiment.rs index 2675ac0f..bb46087f 100644 --- a/crates/tracel-console/src/experiment.rs +++ b/crates/tracel-console/src/experiment.rs @@ -21,15 +21,13 @@ use tracel_experiment::reader::{ ArtifactRef, ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact, }; use tracel_experiment::{ - ArtifactKind, CancelToken, ExperimentId, ExperimentLocation, ExperimentProvider, ExperimentRun, + ArtifactKind, CancelToken, ExperimentId, ExperimentProvider, ExperimentRun, ExperimentRunControl, }; use tracel_experiment_remote::{ArtifactUploadError, ArtifactUploader, RemoteExperimentSession}; -use url::Url; use crate::console::ProjectScope; use crate::wire::console_timestamp; -use crate::{Namespace, NamespaceKind}; /// Experiment provider backed by the console's experiment run protocol. pub struct ConsoleExperimentProvider { @@ -94,50 +92,7 @@ fn create_run( let id = ExperimentId::from(format!("{experiment_num}")); let run = ExperimentRun::new_with_control(id, session, reader, control); - Ok(match page_of(scope, experiment_num) { - Some(page) => run.with_location(ExperimentLocation::Url(page.into())), - None => run, - }) -} - -/// The console page of experiment `num` of the project `scope` names, when the project's owner -/// can be read. -fn page_of(scope: &ProjectScope, num: i32) -> Option { - let kind = match scope.owner_kind() { - Ok(kind) => kind, - Err(error) => { - tracing::warn!("Could not find the console page of experiment {num}: {error}"); - return None; - } - }; - let owner = Namespace { - name: scope.owner.clone(), - kind, - }; - experiment_page(scope.console().base_url(), &owner, &scope.project, num) -} - -/// The page of experiment `num` of `project`, owned by `owner`, on the console whose API is at -/// `api_url`. -/// -/// The console serves its API under `api/` of its web address, and an experiment's page at -/// `users//projects//experiments/` of it, or under `orgs/` for a project -/// an organization owns. `None` when the API is not served under `api/`. -fn experiment_page(api_url: &Url, owner: &Namespace, project: &str, num: i32) -> Option { - let mut page = Url::parse(api_url.as_str().strip_suffix("api/")?).ok()?; - let owners = match owner.kind { - NamespaceKind::User => "users", - NamespaceKind::Organization => "orgs", - }; - page.path_segments_mut().ok()?.pop_if_empty().extend([ - owners, - &owner.name, - "projects", - project, - "experiments", - &num.to_string(), - ]); - Some(page) + Ok(run) } /// A scope for artifact operations within a specific experiment. @@ -381,7 +336,6 @@ impl ArtifactUploader for ConsoleArtifactUploader { #[cfg(test)] mod tests { use rstest::{fixture, rstest}; - use tracel_client::console::Env; use tracel_client::console::artifact::response::ArtifactSourceResponse; use super::*; @@ -441,63 +395,4 @@ mod tests { assert_eq!(newest_named(listed, "model").unwrap().id, newest.id); } - - fn page(api_url: &str, owner: Namespace, project: &str, num: i32) -> Option { - experiment_page(&Url::parse(api_url).unwrap(), &owner, project, num).map(String::from) - } - - #[test] - fn a_user_project_experiment_is_under_users() { - assert_eq!( - page( - Env::Production.get_url().as_str(), - Namespace::user("alice"), - "mnist", - 42 - ) - .as_deref(), - Some("https://console.tracel.ai/users/alice/projects/mnist/experiments/42") - ); - } - - #[test] - fn an_organization_project_experiment_is_under_orgs() { - assert_eq!( - page( - "https://console.example.com/api/", - Namespace::organization("tracel"), - "vision", - 7 - ) - .as_deref(), - Some("https://console.example.com/orgs/tracel/projects/vision/experiments/7") - ); - } - - #[test] - fn names_are_escaped_as_path_segments() { - assert_eq!( - page( - "https://console.example.com/api/", - Namespace::user("a b"), - "x/y", - 1 - ) - .as_deref(), - Some("https://console.example.com/users/a%20b/projects/x%2Fy/experiments/1") - ); - } - - #[test] - fn an_api_not_served_under_api_has_no_pages() { - assert_eq!( - page( - Env::Development.get_url().as_str(), - Namespace::user("alice"), - "mnist", - 1 - ), - None - ); - } } diff --git a/crates/tracel-experiment/src/lib.rs b/crates/tracel-experiment/src/lib.rs index 71e00ee1..1cb00013 100644 --- a/crates/tracel-experiment/src/lib.rs +++ b/crates/tracel-experiment/src/lib.rs @@ -28,7 +28,6 @@ //! feature, `local::LocalExperiments` records experiments on this machine. use std::fmt; -use std::path::PathBuf; use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, Weak}; @@ -133,15 +132,6 @@ impl From for ExperimentId { } } -/// Where an experiment run can be looked at. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ExperimentLocation { - /// The experiment's page on the console. - Url(String), - /// The directory the run is recorded in on this machine. - Dir(PathBuf), -} - /// Artifact category associated with an experiment run. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ArtifactKind { @@ -198,7 +188,6 @@ struct ExperimentMetadata { pub struct ExperimentRun { inner: Arc, handle: ExperimentRunHandle, - location: Option, _tracing_registration: TracingRegistration, } @@ -316,24 +305,10 @@ impl ExperimentRun { Self { inner, handle, - location: None, _tracing_registration: tracing_registration, } } - /// Records where the run can be looked at, such as its page on the console. - /// - /// A provider sets it when it creates the run. - pub fn with_location(mut self, location: ExperimentLocation) -> Self { - self.location = Some(location); - self - } - - /// Where the run can be looked at, when its provider says. - pub fn location(&self) -> Option<&ExperimentLocation> { - self.location.as_ref() - } - /// Stream every panic on any thread into this run's log until the guard /// drops. The failure reason needs no guard: an unwinding run reads the /// panicking thread's record on its own. diff --git a/crates/tracel-experiment/src/local/mod.rs b/crates/tracel-experiment/src/local/mod.rs index 7851ac1b..f1063f8b 100644 --- a/crates/tracel-experiment/src/local/mod.rs +++ b/crates/tracel-experiment/src/local/mod.rs @@ -18,7 +18,7 @@ use serde_json::Value; use crate::error::{ExperimentError, ExperimentErrorKind}; use crate::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; -use crate::{ArtifactKind, CancelToken, ExperimentLocation, ExperimentProvider, ExperimentRun}; +use crate::{ArtifactKind, CancelToken, ExperimentProvider, ExperimentRun}; use artifacts::{LocalExperimentReader, save_artifact}; use events::{EVENTS_FILE, Line, lines, to_jsonl}; use runs::{child_dir, create_local_run_dir, create_runs_dir}; @@ -120,12 +120,12 @@ impl ExperimentProvider for LocalExperiments { .map_err(internal("Failed to create local experiment directory"))?; let (num, run_root) = create_local_run_dir(&root) .map_err(internal("Failed to create local experiment run directory"))?; - let session = LocalExperimentSession::start(run_root.clone(), name, num, attributes) + let session = LocalExperimentSession::start(run_root, name, num, attributes) .map_err(internal("Failed to initialize local experiment session"))?; let reader = LocalExperimentReader { root }; let run = ExperimentRun::new(num.to_string(), session, reader, CancelToken::new()); - Ok(run.with_location(ExperimentLocation::Dir(run_root))) + Ok(run) } } @@ -281,21 +281,7 @@ mod tests { use super::*; use crate::MetricValue; use status::StatusKind; - use test_support::{EXPERIMENT, LocalRun, local_run, read_json, read_lines, run, run_dir}; - - #[test] - fn a_run_is_located_in_its_directory() { - let dir = tempfile::tempdir().unwrap(); - - let experiment = run(dir.path(), "mnist"); - - assert_eq!( - experiment.location(), - Some(&ExperimentLocation::Dir( - dir.path().canonicalize().unwrap().join("mnist/1") - )) - ); - } + use test_support::{EXPERIMENT, LocalRun, local_run, read_json, read_lines, run_dir}; #[test] fn an_experiment_name_that_is_not_one_directory_creates_nothing() { @@ -326,7 +312,7 @@ mod tests { experiment.handle().flush(); assert_eq!( - read_lines(&run_dir(&experiment).join(EVENTS_FILE)), + read_lines(&run_dir(dir.path(), &experiment).join(EVENTS_FILE)), [ json!({"type": "attribute", "key": "dataset", "value": "mnist"}), json!({"type": "attribute", "key": "kind", "value": "example"}), diff --git a/crates/tracel-experiment/src/local/test_support.rs b/crates/tracel-experiment/src/local/test_support.rs index 442fa5c4..f0e9283e 100644 --- a/crates/tracel-experiment/src/local/test_support.rs +++ b/crates/tracel-experiment/src/local/test_support.rs @@ -7,7 +7,7 @@ use serde_json::Value; use tempfile::TempDir; use super::LocalExperiments; -use crate::{ExperimentLocation, ExperimentProvider, ExperimentRun}; +use crate::{ExperimentProvider, ExperimentRun}; pub const EXPERIMENT: &str = "mnist"; @@ -22,7 +22,7 @@ pub fn local_run() -> LocalRun { let runs = tempfile::tempdir().unwrap(); let run = run(runs.path(), EXPERIMENT); LocalRun { - dir: run_dir(&run), + dir: run_dir(runs.path(), &run), run, runs, } @@ -34,11 +34,8 @@ pub fn run(dir: &Path, name: &str) -> ExperimentRun { .unwrap() } -pub fn run_dir(run: &ExperimentRun) -> PathBuf { - match run.location() { - Some(ExperimentLocation::Dir(dir)) => dir.clone(), - location => panic!("a local run is located in a directory, not {location:?}"), - } +pub fn run_dir(runs: &Path, run: &ExperimentRun) -> PathBuf { + runs.join(EXPERIMENT).join(run.id().as_str()) } pub fn read_lines(path: &Path) -> Vec { diff --git a/crates/tracel-experiment/src/provider.rs b/crates/tracel-experiment/src/provider.rs index 60dab194..a7b060e4 100644 --- a/crates/tracel-experiment/src/provider.rs +++ b/crates/tracel-experiment/src/provider.rs @@ -19,11 +19,8 @@ pub trait ExperimentProvider: Send + Sync + 'static { ) -> Result; } -/// Called with the run of each experiment started, once it exists. -type OnStart = Arc; - /// The experiments of a backend, and how each experiment started from them starts: the attributes -/// and arguments it records, and the hooks called with its run. +/// and arguments it records. /// /// [`run`](Self::run) runs code in a new experiment and ends the experiment by what the code /// returns. [`start`](Self::start) starts one and returns its run, for code that ends it itself. @@ -47,7 +44,6 @@ pub struct Experiments { provider: Arc, attributes: HashMap, arguments: Option, - on_start: Vec, } impl Experiments { @@ -57,7 +53,6 @@ impl Experiments { provider, attributes: HashMap::new(), arguments: None, - on_start: Vec::new(), } } @@ -97,20 +92,12 @@ impl Experiments { self } - /// Calls `hook` with the run of each experiment started from the returned experiments, once - /// it exists and before [`start`](Self::start) returns it, after the hooks given before. - pub fn on_start(mut self, hook: impl Fn(&ExperimentRun) + Send + Sync + 'static) -> Self { - self.on_start.push(Arc::new(hook)); - self - } - /// Starts the experiment `name` and returns its run. /// - /// The experiment records the attributes and arguments these experiments give it, and each - /// [`on_start`](Self::on_start) hook is called with the run. The run ends as completed - /// through [`ExperimentRun::finish`], and as failed through [`ExperimentRun::fail`] or when it - /// is dropped before either. Unless the program has installed a `tracing` subscriber, this - /// installs one that forwards events to experiments. + /// The experiment records the attributes and arguments these experiments give it. The run + /// ends as completed through [`ExperimentRun::finish`], and as failed through + /// [`ExperimentRun::fail`] or when it is dropped before either. Unless the program has + /// installed a `tracing` subscriber, this installs one that forwards events to experiments. pub fn start(&self, name: &str) -> Result { let _ = try_init_tracing_subscriber(); @@ -120,9 +107,6 @@ impl Experiments { if let Some(arguments) = &self.arguments { run.record_args(arguments.clone()); } - for hook in &self.on_start { - hook(&run); - } Ok(run) } @@ -347,37 +331,4 @@ mod tests { [json!({"epochs": 2}), json!({"epochs": 2, "fold": 1})] ); } - - #[rstest] - fn each_hook_is_called_with_each_run_started(fixture: Fixture) { - let called = Arc::new(Mutex::new(Vec::new())); - let hook = |tag: &'static str| { - let called = called.clone(); - move |run: &ExperimentRun| called.lock().unwrap().push((tag, run.id().clone())) - }; - let experiments = fixture - .experiments - .clone() - .on_start(hook("first")) - .on_start(hook("second")); - - let first = experiments.start("train").unwrap(); - fixture - .experiments - .start("train") - .unwrap() - .finish() - .unwrap(); - let second = experiments.start("train").unwrap(); - - assert_eq!( - *called.lock().unwrap(), - [ - ("first", first.id().clone()), - ("second", first.id().clone()), - ("first", second.id().clone()), - ("second", second.id().clone()) - ] - ); - } } diff --git a/crates/tracel/src/lib.rs b/crates/tracel/src/lib.rs index 1b34c48c..fed9580b 100644 --- a/crates/tracel/src/lib.rs +++ b/crates/tracel/src/lib.rs @@ -83,12 +83,10 @@ //! //! The experiments [`Target::experiments`] builds in a job, on the thread that runs it, are linked //! to that job, even for runs started on other threads: each one started from them records the -//! job's number and input, and prints where it records to stderr as it starts, such as -//! `Recording experiment 42: `, which `tracel run` and a remote job's logs show. Experiments -//! built another way, such as [`console::ProjectHandle::experiments`], are not linked unless the -//! code links them, through [`Experiments::attribute`](experiment::Experiments::attribute), -//! [`arguments`](experiment::Experiments::arguments) and -//! [`on_start`](experiment::Experiments::on_start). +//! job's number and input. Experiments built another way, such as +//! [`console::ProjectHandle::experiments`], are not linked unless the code links them, through +//! [`Experiments::attribute`](experiment::Experiments::attribute) and +//! [`arguments`](experiment::Experiments::arguments). //! //! [`version`](runtime::Jobs::version) gives the binary its `--version`, which prints the binary's //! name and that version; without it, the binary has no `--version`. diff --git a/crates/tracel/src/running_job.rs b/crates/tracel/src/running_job.rs index 4c36a50a..8de792b4 100644 --- a/crates/tracel/src/running_job.rs +++ b/crates/tracel/src/running_job.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use serde_json::Value; -use tracel_experiment::{ExperimentLocation, ExperimentRun, Experiments}; +use tracel_experiment::Experiments; use tracel_runtime::JobScope; /// The experiment attribute that links an experiment to the job that started it: the job's @@ -15,7 +15,7 @@ const JOB_NUM_ATTRIBUTE: &str = "tracel.job_num"; /// /// Each experiment started from linked experiments, on any thread, belongs to the job: it records /// the job's number as its `tracel.job_num` attribute and the job's input, unless it is `null`, as -/// its arguments, and it prints where it records to stderr as it starts. +/// its arguments. pub fn linked(experiments: Experiments) -> Experiments { let Some(job) = JobScope::current() else { return experiments; @@ -27,19 +27,7 @@ pub fn linked(experiments: Experiments) -> Experiments { Value::Null => experiments, input => experiments.arguments(input.clone()), }; - experiments - .attributes(HashMap::from_iter(attributes)) - .on_start(|run| eprintln!("{}", recording(run))) -} - -/// The line saying where `run` records: its number and its page on the console, or its directory -/// offline. -fn recording(run: &ExperimentRun) -> String { - match run.location() { - Some(ExperimentLocation::Url(url)) => format!("Recording experiment {}: {url}", run.id()), - Some(ExperimentLocation::Dir(dir)) => format!("Recording offline in {}", dir.display()), - None => format!("Recording experiment {}", run.id()), - } + experiments.attributes(HashMap::from_iter(attributes)) } /// The attribute value of the job number `num`: a number when it is one, its text otherwise. @@ -52,7 +40,6 @@ fn job_num(num: &str) -> Value { #[cfg(test)] mod tests { - use std::path::PathBuf; use std::sync::{Arc, Mutex}; use rstest::{fixture, rstest}; @@ -62,13 +49,14 @@ mod tests { ExperimentArtifactReader, ExperimentReaderError, LoadedArtifact, }; use tracel_experiment::session::{BundleFn, Event, ExperimentCompletion, ExperimentSession}; - use tracel_experiment::{ArtifactKind, CancelToken, ExperimentId, ExperimentProvider}; + use tracel_experiment::{ + ArtifactKind, CancelToken, ExperimentId, ExperimentProvider, ExperimentRun, + }; use tracel_runtime::{BoxError, Execution, Jobs}; use super::*; const ID: &str = "7"; - const PAGE: &str = "https://console.tracel.ai/users/alice/projects/mnist/experiments/7"; #[derive(Debug, Default, PartialEq)] struct Recorded { @@ -214,29 +202,4 @@ mod tests { ) { assert_eq!(job_num(num), recorded); } - - #[rstest] - #[case::on_the_console(Some(ExperimentLocation::Url(PAGE.to_string())), PAGE)] - #[case::offline( - Some(ExperimentLocation::Dir(PathBuf::from("runs/mnist/1"))), - "runs/mnist/1" - )] - #[case::without_a_location(None, ID)] - fn an_experiment_announces_where_it_records( - #[case] location: Option, - #[case] named: &str, - ) { - let run = ExperimentRun::new( - ID, - Recorder::default(), - Recorder::default(), - CancelToken::new(), - ); - let run = match location { - Some(location) => run.with_location(location), - None => run, - }; - - assert!(recording(&run).contains(named)); - } } diff --git a/crates/tracel/src/target.rs b/crates/tracel/src/target.rs index 19976b80..a9d4924c 100644 --- a/crates/tracel/src/target.rs +++ b/crates/tracel/src/target.rs @@ -135,12 +135,9 @@ impl Target { /// Built in a job, on the thread its adapter runs it on, the experiments are linked to that /// job, as its [`JobScope`](crate::runtime::JobScope) gives it, wherever runs are started from /// them: each experiment records the job's number, when its launcher gave one, as its - /// `tracel.job_num` attribute and the job's input as its arguments, and prints where it - /// records to stderr as it starts, such as `Recording experiment 42: ` or - /// `Recording offline in `. Experiments built another way, such as - /// [`ProjectHandle::experiments`] or [`Experiments::new`], are not linked unless the code links - /// them through [`Experiments::attribute`], [`Experiments::arguments`] and - /// [`Experiments::on_start`]. + /// `tracel.job_num` attribute and the job's input as its arguments. Experiments built another + /// way, such as [`ProjectHandle::experiments`] or [`Experiments::new`], are not linked unless + /// the code links them through [`Experiments::attribute`] and [`Experiments::arguments`]. pub fn experiments(&self) -> Result { let experiments = match self { Target::Offline { dir } => Experiments::new(Arc::new(LocalExperiments::new(dir))),