From dad82305c44771765922bce365aa4685bc013015 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Tue, 17 Nov 2026 08:57:35 -0800 Subject: [PATCH 1/2] fix(ca): don't wake a sleeping agent to put it to sleep `ca sleep` flushed the disk over SSH before requesting sleep. Any SSH connection now wakes a sleeping agent, and the sleep request that follows arrives while it is starting and is refused, so the command failed and left the agent running. `ca sleep --all` did this to every sleeping agent. Read the agent's status first. A RUNNING agent is flushed and slept as before; a SLEEPING one (which includes the orchestrator's STOPPING) is reported as already asleep without connecting; any other state still sends the sleep request, but never over SSH. `--all` only acts on running agents. Co-Authored-By: Claude Opus 5.5 --- src/commands/cloud_agent/lifecycle.rs | 47 ++++++++++- src/commands/code.rs | 13 +-- src/controllers/cloud_agent.rs | 114 +++++++++++++++++++++++++- tests/cloud_agent_lifecycle.rs | 52 +++++++++--- 4 files changed, 200 insertions(+), 26 deletions(-) diff --git a/src/commands/cloud_agent/lifecycle.rs b/src/commands/cloud_agent/lifecycle.rs index d304a457a..741470375 100644 --- a/src/commands/cloud_agent/lifecycle.rs +++ b/src/commands/cloud_agent/lifecycle.rs @@ -531,10 +531,12 @@ pub async fn sleep(args: SleepArgs) -> Result<()> { } None => ca::list_mine(client, &backboard).await?, }; - // A sleeping observation may predate a wake elsewhere. Send the intent - // for every agent; the server handles already-asleep agents as no-ops. + // Only running agents: the pre-sleep flush connects over SSH, and any + // SSH connection wakes a sleeping agent, which the sleep request then + // fails to put back down. + let agents = running_only(agents); if agents.is_empty() { - println!("No cloud agents to sleep."); + println!("No running cloud agents to sleep."); return Ok(()); } telemetry::track_lifecycle("sleep_all", Duration::ZERO, None).await; @@ -570,7 +572,10 @@ pub async fn sleep(args: SleepArgs) -> Result<()> { let spinner = create_spinner(format!("Sleeping agent {}", agent.name)); let result = ca::sleep(client, &backboard, &agent.environment_id, &agent.id).await; spinner.finish_and_clear(); - result?; + if result? == ca::SleepOutcome::AlreadyAsleep { + println!("✓ Agent {} is already asleep.", agent.name.cyan()); + return Ok(()); + } // Present tense, not "is asleep": the mutation returns before the agent has // finished transitioning, so a `railway ca list` run straight afterwards // still reports it running. Claiming a state the next command contradicts is @@ -582,6 +587,14 @@ pub async fn sleep(args: SleepArgs) -> Result<()> { Ok(()) } +/// The agents `railway ca sleep --all` acts on. +fn running_only(agents: Vec) -> Vec { + agents + .into_iter() + .filter(|agent| agent.status == ca::Status::Running) + .collect() +} + pub async fn delete(args: DeleteArgs) -> Result<()> { let mut configs = Configs::new()?; let client = GQLClient::new_authorized(&configs)?; @@ -1306,4 +1319,30 @@ mod tests { assert_eq!(truncate("ünïcödé", 10), "ünïcödé"); assert_eq!(truncate("ünïcödé-project", 8).chars().count(), 8); } + + #[test] + fn sleep_all_skips_agents_that_are_not_running() { + let agent = |id: &str, status: ca::Status| ca::Agent { + id: id.into(), + name: id.into(), + status, + project_id: "project".into(), + environment_id: "env".into(), + created_at: chrono::Utc::now(), + work_status: None, + archived: false, + }; + let agents = vec![ + agent("running", ca::Status::Running), + agent("sleeping", ca::Status::Sleeping), + agent("starting", ca::Status::Starting), + agent("crashed", ca::Status::Crashed), + agent("failed", ca::Status::Failed), + agent("deleting", ca::Status::Deleting), + agent("unknown", ca::Status::Unknown("NEW_STATE".into())), + agent("also-running", ca::Status::Running), + ]; + let ids: Vec<_> = running_only(agents).into_iter().map(|a| a.id).collect(); + assert_eq!(ids, ["running", "also-running"]); + } } diff --git a/src/commands/code.rs b/src/commands/code.rs index adb137e64..23815f6ac 100644 --- a/src/commands/code.rs +++ b/src/commands/code.rs @@ -3710,12 +3710,13 @@ const FLUSH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); /// Flush the agent's filesystem so sleeping it cannot discard recent work. /// -/// `cloudAgentSleep` snapshots the disk without quiescing the guest, so pages -/// still dirty in its page cache are absent from the image the next wake -/// restores. Measured on a scratch agent: a file written a second before a -/// sleep was gone after waking, while the same write followed by `sync` -/// survived — including when the sleep was issued immediately on disconnect. -/// So the variable is the flush, not the timing. +/// Belt-and-braces: since monorepo #37022 a host-initiated stop syncs the +/// guest itself. Before that, `cloudAgentSleep` snapshotted the disk without +/// quiescing the guest, and a file written a second before a sleep was gone +/// after waking unless a `sync` ran first. The flush stays for RUNNING agents +/// only: connecting to a sleeping agent wakes it, so +/// [`crate::controllers::cloud_agent::sleep`] checks the status before calling +/// this. /// /// A side connection is enough because `sync(2)` flushes the whole filesystem /// regardless of which process dirtied it: this covers whatever the durable diff --git a/src/controllers/cloud_agent.rs b/src/controllers/cloud_agent.rs index 61b35ef1f..2dd975321 100644 --- a/src/controllers/cloud_agent.rs +++ b/src/controllers/cloud_agent.rs @@ -373,21 +373,63 @@ pub async fn wake(client: &reqwest::Client, backboard: &str, id: &str) -> Result Ok(()) } -/// Sleep an agent, flushing its filesystem first. +/// What [`sleep`] did. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SleepOutcome { + /// The sleep mutation was sent. + Requested, + /// The agent was already asleep, or on its way there; nothing was sent. + AlreadyAsleep, +} + +/// How [`sleep`] treats an agent in a given state. +#[derive(Debug, PartialEq, Eq)] +enum SleepPlan { + FlushThenSleep, + AlreadyAsleep, + SleepWithoutFlush, +} + +/// Only a RUNNING agent is flushed: any SSH connection wakes a sleeping agent, +/// and a sleep request that lands while it is starting is refused, so flushing +/// anything else either wakes it and leaves it running or reaches nothing. +/// Backboard reports the orchestrator's STOPPING as SLEEPING, so both count as +/// already asleep. Every other state (including an agent the read did not +/// find) still sends the mutation, so the API reports what it makes of it. +fn sleep_plan(status: Option<&Status>) -> SleepPlan { + match status { + Some(Status::Running) => SleepPlan::FlushThenSleep, + Some(Status::Sleeping) => SleepPlan::AlreadyAsleep, + _ => SleepPlan::SleepWithoutFlush, + } +} + +/// Sleep an agent, flushing its filesystem first when it is running. /// /// The flush is paired with the mutation here, rather than left to each caller, /// because forgetting it loses the user's most recent work silently — see /// [`crate::commands::code::flush_disk`]. Every path that suspends an agent goes /// through this function for that reason; there is deliberately no way to reach /// the bare mutation from outside this module. +/// +/// The agent's status is read first so the flush never wakes an agent that is +/// already asleep — see [`sleep_plan`]. pub async fn sleep( client: &reqwest::Client, backboard: &str, environment_id: &str, id: &str, -) -> Result<()> { - crate::commands::code::flush_disk(environment_id, id).await; - sleep_without_flush(client, backboard, id).await +) -> Result { + let status = get(client, backboard, environment_id, id) + .await? + .map(|agent| agent.status); + match sleep_plan(status.as_ref()) { + SleepPlan::AlreadyAsleep => return Ok(SleepOutcome::AlreadyAsleep), + SleepPlan::FlushThenSleep => crate::commands::code::flush_disk(environment_id, id).await, + SleepPlan::SleepWithoutFlush => {} + } + sleep_without_flush(client, backboard, id).await?; + Ok(SleepOutcome::Requested) } async fn sleep_without_flush(client: &reqwest::Client, backboard: &str, id: &str) -> Result<()> { @@ -982,4 +1024,68 @@ mod tests { assert_eq!(humanize_age(Utc::now() - chrono::Duration::hours(3)), "3h"); assert_eq!(humanize_age(Utc::now() - chrono::Duration::days(9)), "9d"); } + + #[test] + fn only_a_running_agent_is_flushed_before_sleeping() { + assert_eq!( + sleep_plan(Some(&Status::Running)), + SleepPlan::FlushThenSleep + ); + assert_eq!( + sleep_plan(Some(&Status::Sleeping)), + SleepPlan::AlreadyAsleep + ); + for status in [ + Status::Starting, + Status::Crashed, + Status::Failed, + Status::Deleting, + Status::Unknown("NEW_STATE".into()), + ] { + assert_eq!( + sleep_plan(Some(&status)), + SleepPlan::SleepWithoutFlush, + "{status:?}" + ); + } + assert_eq!(sleep_plan(None), SleepPlan::SleepWithoutFlush); + } + + fn agent_node(status: &str) -> serde_json::Value { + json!({"cloudAgent": { + "id": "agent", "name": "agent", "status": status, + "projectId": "project", "environmentId": "env", + "createdAt": "2026-09-10T00:00:00Z" + }}) + } + + #[tokio::test] + async fn sleeping_an_asleep_agent_sends_nothing() { + let server = MockBackboard::spawn(); + server.stub("CloudAgent", agent_node("SLEEPING")); + let outcome = sleep(&reqwest::Client::new(), &server.url(), "env", "agent") + .await + .unwrap(); + assert_eq!(outcome, SleepOutcome::AlreadyAsleep); + assert!(server.variables_for("CloudAgentSleep").is_empty()); + } + + #[tokio::test] + async fn sleeping_a_starting_agent_still_requests_sleep() { + // Not RUNNING, so no flush: the request goes straight to the API. + let server = MockBackboard::spawn(); + server.stub("CloudAgent", agent_node("STARTING")); + server.stub( + "CloudAgentSleep", + json!({"cloudAgentSleep": {"id": "agent", "status": "STARTING"}}), + ); + let outcome = sleep(&reqwest::Client::new(), &server.url(), "env", "agent") + .await + .unwrap(); + assert_eq!(outcome, SleepOutcome::Requested); + assert_eq!( + server.variables_for("CloudAgentSleep"), + vec![json!({"id": "agent"})] + ); + } } diff --git a/tests/cloud_agent_lifecycle.rs b/tests/cloud_agent_lifecycle.rs index dd4898bea..c0e9a3275 100644 --- a/tests/cloud_agent_lifecycle.rs +++ b/tests/cloud_agent_lifecycle.rs @@ -136,7 +136,7 @@ fn agent(id: &str, status: &str) -> Value { } #[test] -fn sleep_sends_every_observed_state_and_reports_acceptance() { +fn sleep_skips_an_asleep_agent_and_sends_every_other_state() { for status in [ "RUNNING", "STARTING", @@ -146,7 +146,8 @@ fn sleep_sends_every_observed_state_and_reports_acceptance() { "DELETING", "FUTURE_STATE", ] { - let server = Backboard::new(vec![agent("existing", status)], None, "RUNNING"); + // The status sleep reads before acting is the `CloudAgent` response. + let server = Backboard::new(vec![agent("existing", status)], None, status); let home = tempfile::tempdir().unwrap(); let output = server.run(home.path(), &["sleep", "existing"]); assert!( @@ -154,21 +155,30 @@ fn sleep_sends_every_observed_state_and_reports_acceptance() { "{}", String::from_utf8_lossy(&output.stderr) ); - assert_eq!(server.ids_for("CloudAgentSleep"), ["existing"], "{status}"); - assert!(String::from_utf8_lossy(&output.stdout).contains("Sleep requested")); - assert!( - server.ids_for("CloudAgent").is_empty(), - "acceptance does not poll for completion" + let stdout = String::from_utf8_lossy(&output.stdout); + if status == "SLEEPING" { + // Connecting to flush would wake it; nothing is sent. + assert!(server.ids_for("CloudAgentSleep").is_empty(), "{status}"); + assert!(stdout.contains("already asleep"), "{stdout}"); + } else { + assert_eq!(server.ids_for("CloudAgentSleep"), ["existing"], "{status}"); + assert!(stdout.contains("Sleep requested"), "{stdout}"); + } + assert_eq!( + server.ids_for("CloudAgent"), + ["existing"], + "one status read; acceptance does not poll for completion" ); } } #[test] -fn bulk_sleep_attempts_every_agent_and_reports_partial_failure() { - for refused in [None, Some("sleeping")] { +fn bulk_sleep_attempts_every_running_agent_and_reports_partial_failure() { + for refused in [None, Some("also-running")] { let server = Backboard::new( vec![ agent("running", "RUNNING"), + agent("also-running", "RUNNING"), agent("sleeping", "SLEEPING"), agent("unknown", "FUTURE_STATE"), ], @@ -180,20 +190,38 @@ fn bulk_sleep_attempts_every_agent_and_reports_partial_failure() { assert_eq!(output.status.success(), refused.is_none()); assert_eq!( server.ids_for("CloudAgentSleep"), - ["running", "sleeping", "unknown"] + ["also-running", "running"] ); - let count = if refused.is_some() { 2 } else { 3 }; + let count = if refused.is_some() { 1 } else { 2 }; assert!( String::from_utf8_lossy(&output.stdout) .contains(&format!("Sleep requested for {count} agents")) ); if refused.is_some() { let stderr = String::from_utf8_lossy(&output.stderr); - assert!(stderr.contains("sleeping — mutation refused"), "{stderr}"); + assert!( + stderr.contains("also-running — mutation refused"), + "{stderr}" + ); } } } +#[test] +fn bulk_sleep_with_nothing_running_sends_nothing() { + let server = Backboard::new( + vec![agent("sleeping", "SLEEPING"), agent("crashed", "CRASHED")], + None, + "SLEEPING", + ); + let home = tempfile::tempdir().unwrap(); + let output = server.run(home.path(), &["sleep", "--all"]); + assert!(output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).contains("No running cloud agents")); + assert!(server.ids_for("CloudAgentSleep").is_empty()); + assert!(server.ids_for("CloudAgent").is_empty()); +} + #[test] fn single_sleep_refusal_is_not_reported_as_success() { let server = Backboard::new( From b40560f54372e98743ecb7d23bd01b3ada51226e Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Tue, 17 Nov 2026 09:02:53 -0800 Subject: [PATCH 2/2] fix(ca): adopt a restarted OpenCode/Codex server after wake express-agent now restarts recorded listeners on the first boot after a power-off, so the OpenCode or Codex server this CLI started comes back under a new PID. The in-VM scripts only treated a listener as their own when the saved PID and start time matched, so rerunning desktop setup (or `railway code --opencode/--codex` reconnecting) failed with "Port 4096 is occupied by another process". When the saved-PID check fails and the port's listener authenticates with the saved credentials (OpenCode's server.json password, Codex's token), find it through /proc, record its process group leader and start time, and continue. Discovery does the same on a copy it never saves, and OpenCode's stop adopts before stopping. A listener that rejects the saved credentials is still refused. Co-Authored-By: Claude Opus 5.5 --- src/commands/cloud_agent/codex.py | 57 ++++++++++++++++++++ src/commands/cloud_agent/opencode.py | 73 ++++++++++++++++++++++++-- tests/codex_server.py | 52 ++++++++++++++++++- tests/opencode_desktop.py | 77 ++++++++++++++++++++++++++-- 4 files changed, 251 insertions(+), 8 deletions(-) diff --git a/src/commands/cloud_agent/codex.py b/src/commands/cloud_agent/codex.py index 060150500..7567e1283 100644 --- a/src/commands/cloud_agent/codex.py +++ b/src/commands/cloud_agent/codex.py @@ -49,6 +49,56 @@ def owned_process(state): return isinstance(pid, int) and pid > 1 and start is not None and process_start(pid) == start +def listener_pid(port): + """The PID of the process listening on TCP `port`, read from /proc.""" + sockets = set() + for table in ("/proc/net/tcp", "/proc/net/tcp6"): + try: + rows = Path(table).read_text().splitlines()[1:] + except OSError: + continue + for row in rows: + fields = row.split() + # local_address is HEXIP:HEXPORT; state 0A is LISTEN. + if len(fields) > 9 and fields[3] == "0A" and int(fields[1].rsplit(":", 1)[1], 16) == port: + sockets.add(f"socket:[{fields[9]}]") + if not sockets: + return None + for process in Path("/proc").iterdir(): + if not process.name.isdecimal(): + continue + try: + for descriptor in (process / "fd").iterdir(): + if os.readlink(descriptor) in sockets: + return int(process.name) + except OSError: + continue + return None + + +def adopt_listener(state, port, authenticated): + """Record a server that express-agent restarted after the VM powered off. + + express-agent relaunches recorded listeners on the first boot after a + power-off, so the server this script started comes back with a new PID and + the saved PID check fails. A listener that accepts the saved token is + that server: record its process group leader (what stop signals) and start + time in `state`. Returns whether it was adopted. + """ + if restored_state(state) or not authenticated: + return False + pid = listener_pid(port) + try: + leader = os.getpgid(pid) if pid else None + except OSError: + return False + start = process_start(leader) if leader and leader > 1 else None + if start is None: + return False + state.update(pid=leader, start=start) + return True + + def probe(port, token=None, websocket=False): # Direct loopback probes bypass HTTP_PROXY. Health endpoints are intentionally # public; only a WebSocket upgrade tests the server's authentication boundary. @@ -182,6 +232,10 @@ def setup(request, home): if action == "inspect": state = json.loads(state_path.read_text()) if state_path.exists() else {} port = server_port(state) + if not owned_process(state) and state.get("token") and not port_available(port): + # Read-only: adopt into a copy that is never saved. + state = dict(state) + adopt_listener(state, port, healthy(port, state["token"])) if (owned_process(state) and state.get("token") and state.get("directory") and healthy(port, state["token"])): return {"directory": state["directory"], "version": state.get("version")} @@ -213,6 +267,9 @@ def setup(request, home): if not isinstance(token, str) or not re.fullmatch(r"[a-zA-Z0-9_-]{43,}", token): raise SetupError("Codex requires a high-entropy URL-safe connection token.") + if (not owned_process(state) and state.get("token") and not port_available(port) + and adopt_listener(state, port, healthy(port, state["token"]))): + save(state_path, state) reused = owned_process(state) if reused: if not healthy(port, token): diff --git a/src/commands/cloud_agent/opencode.py b/src/commands/cloud_agent/opencode.py index 4b3e1e474..1b0821a25 100644 --- a/src/commands/cloud_agent/opencode.py +++ b/src/commands/cloud_agent/opencode.py @@ -65,6 +65,56 @@ def owned_process(state): return isinstance(pid, int) and pid > 1 and start is not None and process_start(pid) == start +def listener_pid(port): + """The PID of the process listening on TCP `port`, read from /proc.""" + sockets = set() + for table in ("/proc/net/tcp", "/proc/net/tcp6"): + try: + rows = Path(table).read_text().splitlines()[1:] + except OSError: + continue + for row in rows: + fields = row.split() + # local_address is HEXIP:HEXPORT; state 0A is LISTEN. + if len(fields) > 9 and fields[3] == "0A" and int(fields[1].rsplit(":", 1)[1], 16) == port: + sockets.add(f"socket:[{fields[9]}]") + if not sockets: + return None + for process in Path("/proc").iterdir(): + if not process.name.isdecimal(): + continue + try: + for descriptor in (process / "fd").iterdir(): + if os.readlink(descriptor) in sockets: + return int(process.name) + except OSError: + continue + return None + + +def adopt_listener(state, port, authenticated): + """Record a server that express-agent restarted after the VM powered off. + + express-agent relaunches recorded listeners on the first boot after a + power-off, so the server this script started comes back with a new PID and + the saved PID check fails. A listener that accepts the saved credentials is + that server: record its process group leader (what stop signals) and start + time in `state`. Returns whether it was adopted. + """ + if restored_state(state) or not authenticated: + return False + pid = listener_pid(port) + try: + leader = os.getpgid(pid) if pid else None + except OSError: + return False + start = process_start(leader) if leader and leader > 1 else None + if start is None: + return False + state.update(pid=leader, start=start) + return True + + def probe(port, credentials, path): request = urllib.request.Request(f"http://127.0.0.1:{port}/{path}") if credentials: @@ -101,6 +151,11 @@ def health(port, credentials=None): return status, ready +def authenticates(port, credentials): + """The server on `port` accepts `credentials` and refuses requests without them.""" + return health(port, credentials) == (200, True) and health(port)[0] == 401 + + def stop_owned(state): if not owned_process(state): return @@ -224,9 +279,11 @@ def setup(request, home): if not saved_v2(state): return None port = server_port(state) - if (owned_process(state) - and health(port, state) == (200, True) - and health(port)[0] == 401): + if not owned_process(state) and state.get("password") and not port_available(port): + # Read-only: adopt into a copy that is never saved. + state = dict(state) + adopt_listener(state, port, authenticates(port, state)) + if owned_process(state) and authenticates(port, state): return {"directory": state.get("directory") or str(Path(f"/proc/{state['pid']}/cwd").resolve())} return None if request.get("action") == "connect" and not state_path.exists(): @@ -241,6 +298,13 @@ def setup(request, home): state = json.loads(state_path.read_text()) if state_path.exists() else {} if request.get("action") == "stop": # A V1 server is still ours to stop; nothing else touches it. + if not owned_process(state) and state.get("password"): + try: + port = server_port(state) + except SetupError: + port = None + if port and not port_available(port): + adopt_listener(state, port, authenticates(port, state)) stop_owned(state) state.pop("pid", None) state.pop("start", None) @@ -280,6 +344,9 @@ def setup(request, home): raise SetupError("OpenCode requires a nonempty username and password.") runtime = None + if (not port_available(port) and not owned_process(state) and state.get("password") + and adopt_listener(state, port, authenticates(port, credentials))): + save(state_path, state) if not port_available(port) and not owned_process(state): raise SetupError(f"Port {port} is occupied by another process. Stop it or use --new for a fresh agent.") current = server_status(port, credentials)[1].get("version") if owned_process(state) else None diff --git a/tests/codex_server.py b/tests/codex_server.py index b6e6cd50b..dc8f6399e 100644 --- a/tests/codex_server.py +++ b/tests/codex_server.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile +import time import unittest from unittest.mock import patch @@ -136,6 +137,12 @@ def launch(*args, **kwargs): str(pid) if any(c.pid == pid and c.poll() is None for c in self.children) else None) mock.start() self.addCleanup(mock.stop) + # /proc's socket tables stand in for lsof on Linux. + mock = patch.object(bootstrap, "listener_pid", side_effect=lambda port: next(iter(map(int, subprocess.run( + ["lsof", "-nP", "-t", f"-iTCP:{port}", "-sTCP:LISTEN"], + capture_output=True, text=True).stdout.split())), None)) + mock.start() + self.addCleanup(mock.stop) self.addCleanup(self.stop_children) def stop_children(self): @@ -144,6 +151,24 @@ def stop_children(self): os.killpg(child.pid, signal.SIGTERM) child.wait(timeout=5) + def relaunch(self): + """What express-agent does on the first boot after a power-off: the + recorded server's command again, in its own session, under a new PID.""" + self.stop_children() + child = subprocess.Popen( + [str(self.home / ".local/bin/codex"), "app-server", "--listen", f"ws://0.0.0.0:{self.port}", + "--ws-auth", "capability-token", "--ws-token-file", str(self.root / "server-token"), + "-c", 'approval_policy="never"', "-c", 'sandbox_mode="danger-full-access"'], + cwd=self.directory, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True, + ) + self.children.append(child) + deadline = time.monotonic() + 10 + while bootstrap.probe(self.port) != 200: + self.assertLess(time.monotonic(), deadline, "relaunched server never listened") + time.sleep(0.05) + return child + def start(self, **extra): return bootstrap.setup(dict(directory=str(self.directory), token=self.token, **extra), self.home) @@ -368,14 +393,37 @@ def test_old_binary_is_rejected_before_server_start(self): self.start() self.assertEqual(self.children, []) - def test_stale_pid_cannot_claim_an_existing_listener(self): + def test_server_relaunched_after_power_off_is_adopted(self): + first = self.start() + state_path = self.root / "server.json" + relaunched = self.relaunch() + before = state_path.read_bytes() + info = bootstrap.setup({"action": "inspect"}, self.home) + self.assertEqual(info, {"directory": str(self.directory), "version": first["version"]}) + self.assertEqual(state_path.read_bytes(), before) + connected = bootstrap.setup({"action": "connect"}, self.home) + self.assertTrue(connected["reused"]) + self.assertEqual(connected["token"], first["token"]) + self.assertEqual(json.loads(state_path.read_text())["pid"], relaunched.pid) + self.assertTrue(self.start()["reused"]) + self.assertIsNone(relaunched.poll()) + # Adopted means owned: an update stops the relaunched server. + with patch.dict(os.environ, {"FAKE_LATEST": "0.154.0"}): + self.assertFalse(self.start()["reused"]) + relaunched.wait(timeout=5) + + def test_listener_that_rejects_the_saved_token_is_not_adopted(self): self.start() state_path = self.root / "server.json" state = json.loads(state_path.read_text()) - state["start"] = "not-the-process-start-time" + state.update(start="not-the-process-start-time", token="b" * 43) bootstrap.save(state_path, state) + self.assertIsNone(bootstrap.setup({"action": "inspect"}, self.home)) with self.assertRaisesRegex(bootstrap.SetupError, "occupied by another process"): self.start() + with self.assertRaisesRegex(bootstrap.SetupError, "occupied by another process"): + bootstrap.setup({"action": "connect"}, self.home) + self.assertEqual(json.loads(state_path.read_text())["start"], "not-the-process-start-time") self.assertIsNone(self.children[0].poll()) def test_concurrent_setup_is_rejected(self): diff --git a/tests/opencode_desktop.py b/tests/opencode_desktop.py index d84b2791c..82b57bf12 100644 --- a/tests/opencode_desktop.py +++ b/tests/opencode_desktop.py @@ -13,8 +13,9 @@ import unittest BOOTSTRAP = Path(__file__).resolve().parents[1] / 'src/commands/cloud_agent/opencode.py' -# Production uses /proc PID start times. macOS CI exercises the same lifecycle -# using ps, keeping the production bootstrap Linux-specific. +# Production uses /proc PID start times and socket tables. macOS CI exercises +# the same lifecycle using ps and lsof, keeping the production bootstrap +# Linux-specific. WRAPPER = ''' import importlib.util,json,sys,subprocess from pathlib import Path @@ -30,6 +31,10 @@ def process_start(pid): # Process status changes between probes; only start time identifies it. return fields[1] if result.returncode == 0 and len(fields) == 2 and not fields[0].startswith('Z') else None m.process_start = process_start + def listener_pid(port): + pids = subprocess.run(['lsof', '-nP', '-t', f'-iTCP:{port}', '-sTCP:LISTEN'], capture_output=True, text=True).stdout.split() + return int(pids[0]) if pids else None + m.listener_pid = listener_pid try: print(json.dumps(m.setup(json.load(sys.stdin), Path(sys.argv[2])))) except Exception as error: @@ -428,10 +433,76 @@ def test_unprotected_server_is_stopped(self): self.assertNotEqual(result.returncode, 0) self.assertIn('not enforcing password', result.stderr) + def relaunch(self, password=None): + """What express-agent does on the first boot after a power-off: the + recorded server's command again, in its own session, under a new PID.""" + state = json.loads(self.state.read_text()) + os.killpg(state['pid'], signal.SIGTERM) + deadline = time.monotonic() + 10 + while not self.port_free(): + self.assertLess(time.monotonic(), deadline, 'saved server never stopped') + time.sleep(0.05) + env = dict(self.env, HOME=str(self.home), OPENCODE_SERVER_USERNAME=state['username'], + OPENCODE_SERVER_PASSWORD=password or state['password']) + child = subprocess.Popen( + [state['binary'], 'serve', '--hostname', '0.0.0.0', '--port', str(state['port'])], + cwd=state['directory'], env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True, + ) + self.addCleanup(self.reap, child) + while self.port_free(): + self.assertLess(time.monotonic(), deadline, 'relaunched server never listened') + time.sleep(0.05) + return child + + def port_free(self): + with socket.socket() as sock: + return sock.connect_ex(('127.0.0.1', self.port)) != 0 + + def reap(self, child): + if child.poll() is None: + os.killpg(child.pid, signal.SIGTERM) + child.wait(timeout=5) + + def test_server_relaunched_after_power_off_is_adopted(self): + first = self.run_bootstrap() + relaunched = self.relaunch() + before = self.state.read_bytes() + self.assertEqual(self.run_bootstrap({'action': 'inspect'}), {'directory': first['directory']}) + self.assertEqual(self.state.read_bytes(), before) + connected = self.run_bootstrap({'action': 'connect'}) + self.assertTrue(connected['reused']) + self.assertEqual(connected['password'], first['password']) + self.assertEqual(json.loads(self.state.read_text())['pid'], relaunched.pid) + self.assertTrue(self.run_bootstrap()['reused']) + self.assertIsNone(relaunched.poll()) + self.run_bootstrap({'action': 'stop'}) + relaunched.wait(timeout=5) + + def test_stop_adopts_a_relaunched_server(self): + self.run_bootstrap() + relaunched = self.relaunch() + self.assertEqual(self.run_bootstrap({'action': 'stop'}), {'stopped': True}) + relaunched.wait(timeout=5) + + def test_listener_that_rejects_the_saved_password_is_not_adopted(self): + self.run_bootstrap() + relaunched = self.relaunch(password='someone-else') + before = self.state.read_bytes() + self.assertIsNone(self.run_bootstrap({'action': 'inspect'})) + for request in ({'action': 'connect'}, None): + result = self.run_bootstrap(request, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertIn('occupied', result.stderr) + self.run_bootstrap({'action': 'stop'}) + self.assertIsNone(relaunched.poll()) + self.assertEqual(json.loads(self.state.read_text()).get('password'), json.loads(before)['password']) + def test_stop_does_not_signal_reused_pid(self): self.run_bootstrap() real_state = json.loads(self.state.read_text()) - altered = dict(real_state, pid=os.getpid(), start='wrong-start-time') + # A password the server rejects, so stop has no listener to adopt either. + altered = dict(real_state, pid=os.getpid(), start='wrong-start-time', password='not-the-servers') self.state.write_text(json.dumps(altered)) self.run_bootstrap({'action': 'stop'}) self.state.write_text(json.dumps(real_state))