Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions src/commands/cloud_agent/codex.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,56 @@ def owned_process(state):
return isinstance(pid, int) and pid > 1 and start is not None and process_start(pid) == start


def listener_pid(port):
"""The PID of the process listening on TCP `port`, read from /proc."""
sockets = set()
for table in ("/proc/net/tcp", "/proc/net/tcp6"):
try:
rows = Path(table).read_text().splitlines()[1:]
except OSError:
continue
for row in rows:
fields = row.split()
# local_address is HEXIP:HEXPORT; state 0A is LISTEN.
if len(fields) > 9 and fields[3] == "0A" and int(fields[1].rsplit(":", 1)[1], 16) == port:
sockets.add(f"socket:[{fields[9]}]")
if not sockets:
return None
for process in Path("/proc").iterdir():
if not process.name.isdecimal():
continue
try:
for descriptor in (process / "fd").iterdir():
if os.readlink(descriptor) in sockets:
return int(process.name)
except OSError:
continue
return None


def adopt_listener(state, port, authenticated):
"""Record a server that express-agent restarted after the VM powered off.

express-agent relaunches recorded listeners on the first boot after a
power-off, so the server this script started comes back with a new PID and
the saved PID check fails. A listener that accepts the saved token is
that server: record its process group leader (what stop signals) and start
time in `state`. Returns whether it was adopted.
"""
if restored_state(state) or not authenticated:
return False
pid = listener_pid(port)
try:
leader = os.getpgid(pid) if pid else None
except OSError:
return False
start = process_start(leader) if leader and leader > 1 else None
if start is None:
return False
state.update(pid=leader, start=start)
return True


def probe(port, token=None, websocket=False):
# Direct loopback probes bypass HTTP_PROXY. Health endpoints are intentionally
# public; only a WebSocket upgrade tests the server's authentication boundary.
Expand Down Expand Up @@ -182,6 +232,10 @@ def setup(request, home):
if action == "inspect":
state = json.loads(state_path.read_text()) if state_path.exists() else {}
port = server_port(state)
if not owned_process(state) and state.get("token") and not port_available(port):
# Read-only: adopt into a copy that is never saved.
state = dict(state)
adopt_listener(state, port, healthy(port, state["token"]))
if (owned_process(state) and state.get("token") and state.get("directory")
and healthy(port, state["token"])):
return {"directory": state["directory"], "version": state.get("version")}
Expand Down Expand Up @@ -213,6 +267,9 @@ def setup(request, home):
if not isinstance(token, str) or not re.fullmatch(r"[a-zA-Z0-9_-]{43,}", token):
raise SetupError("Codex requires a high-entropy URL-safe connection token.")

if (not owned_process(state) and state.get("token") and not port_available(port)
and adopt_listener(state, port, healthy(port, state["token"]))):
save(state_path, state)
reused = owned_process(state)
if reused:
if not healthy(port, token):
Expand Down
47 changes: 43 additions & 4 deletions src/commands/cloud_agent/lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -531,10 +531,12 @@ pub async fn sleep(args: SleepArgs) -> Result<()> {
}
None => ca::list_mine(client, &backboard).await?,
};
// A sleeping observation may predate a wake elsewhere. Send the intent
// for every agent; the server handles already-asleep agents as no-ops.
// Only running agents: the pre-sleep flush connects over SSH, and any
// SSH connection wakes a sleeping agent, which the sleep request then
// fails to put back down.
let agents = running_only(agents);
if agents.is_empty() {
println!("No cloud agents to sleep.");
println!("No running cloud agents to sleep.");
return Ok(());
}
telemetry::track_lifecycle("sleep_all", Duration::ZERO, None).await;
Expand Down Expand Up @@ -570,7 +572,10 @@ pub async fn sleep(args: SleepArgs) -> Result<()> {
let spinner = create_spinner(format!("Sleeping agent {}", agent.name));
let result = ca::sleep(client, &backboard, &agent.environment_id, &agent.id).await;
spinner.finish_and_clear();
result?;
if result? == ca::SleepOutcome::AlreadyAsleep {
println!("✓ Agent {} is already asleep.", agent.name.cyan());
return Ok(());
}
// Present tense, not "is asleep": the mutation returns before the agent has
// finished transitioning, so a `railway ca list` run straight afterwards
// still reports it running. Claiming a state the next command contradicts is
Expand All @@ -582,6 +587,14 @@ pub async fn sleep(args: SleepArgs) -> Result<()> {
Ok(())
}

/// The agents `railway ca sleep --all` acts on.
fn running_only(agents: Vec<ca::Agent>) -> Vec<ca::Agent> {
agents
.into_iter()
.filter(|agent| agent.status == ca::Status::Running)
.collect()
}

pub async fn delete(args: DeleteArgs) -> Result<()> {
let mut configs = Configs::new()?;
let client = GQLClient::new_authorized(&configs)?;
Expand Down Expand Up @@ -1306,4 +1319,30 @@ mod tests {
assert_eq!(truncate("ünïcödé", 10), "ünïcödé");
assert_eq!(truncate("ünïcödé-project", 8).chars().count(), 8);
}

#[test]
fn sleep_all_skips_agents_that_are_not_running() {
let agent = |id: &str, status: ca::Status| ca::Agent {
id: id.into(),
name: id.into(),
status,
project_id: "project".into(),
environment_id: "env".into(),
created_at: chrono::Utc::now(),
work_status: None,
archived: false,
};
let agents = vec![
agent("running", ca::Status::Running),
agent("sleeping", ca::Status::Sleeping),
agent("starting", ca::Status::Starting),
agent("crashed", ca::Status::Crashed),
agent("failed", ca::Status::Failed),
agent("deleting", ca::Status::Deleting),
agent("unknown", ca::Status::Unknown("NEW_STATE".into())),
agent("also-running", ca::Status::Running),
];
let ids: Vec<_> = running_only(agents).into_iter().map(|a| a.id).collect();
assert_eq!(ids, ["running", "also-running"]);
}
}
73 changes: 70 additions & 3 deletions src/commands/cloud_agent/opencode.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,56 @@ def owned_process(state):
return isinstance(pid, int) and pid > 1 and start is not None and process_start(pid) == start


def listener_pid(port):
"""The PID of the process listening on TCP `port`, read from /proc."""
sockets = set()
for table in ("/proc/net/tcp", "/proc/net/tcp6"):
try:
rows = Path(table).read_text().splitlines()[1:]
except OSError:
continue
for row in rows:
fields = row.split()
# local_address is HEXIP:HEXPORT; state 0A is LISTEN.
if len(fields) > 9 and fields[3] == "0A" and int(fields[1].rsplit(":", 1)[1], 16) == port:
sockets.add(f"socket:[{fields[9]}]")
if not sockets:
return None
for process in Path("/proc").iterdir():
if not process.name.isdecimal():
continue
try:
for descriptor in (process / "fd").iterdir():
if os.readlink(descriptor) in sockets:
return int(process.name)
except OSError:
continue
return None


def adopt_listener(state, port, authenticated):
"""Record a server that express-agent restarted after the VM powered off.

express-agent relaunches recorded listeners on the first boot after a
power-off, so the server this script started comes back with a new PID and
the saved PID check fails. A listener that accepts the saved credentials is
that server: record its process group leader (what stop signals) and start
time in `state`. Returns whether it was adopted.
"""
if restored_state(state) or not authenticated:
return False
pid = listener_pid(port)
try:
leader = os.getpgid(pid) if pid else None
except OSError:
return False
start = process_start(leader) if leader and leader > 1 else None
if start is None:
return False
state.update(pid=leader, start=start)
return True


def probe(port, credentials, path):
request = urllib.request.Request(f"http://127.0.0.1:{port}/{path}")
if credentials:
Expand Down Expand Up @@ -101,6 +151,11 @@ def health(port, credentials=None):
return status, ready


def authenticates(port, credentials):
"""The server on `port` accepts `credentials` and refuses requests without them."""
return health(port, credentials) == (200, True) and health(port)[0] == 401


def stop_owned(state):
if not owned_process(state):
return
Expand Down Expand Up @@ -224,9 +279,11 @@ def setup(request, home):
if not saved_v2(state):
return None
port = server_port(state)
if (owned_process(state)
and health(port, state) == (200, True)
and health(port)[0] == 401):
if not owned_process(state) and state.get("password") and not port_available(port):
# Read-only: adopt into a copy that is never saved.
state = dict(state)
adopt_listener(state, port, authenticates(port, state))
if owned_process(state) and authenticates(port, state):
return {"directory": state.get("directory") or str(Path(f"/proc/{state['pid']}/cwd").resolve())}
return None
if request.get("action") == "connect" and not state_path.exists():
Expand All @@ -241,6 +298,13 @@ def setup(request, home):
state = json.loads(state_path.read_text()) if state_path.exists() else {}
if request.get("action") == "stop":
# A V1 server is still ours to stop; nothing else touches it.
if not owned_process(state) and state.get("password"):
try:
port = server_port(state)
except SetupError:
port = None
if port and not port_available(port):
adopt_listener(state, port, authenticates(port, state))
stop_owned(state)
state.pop("pid", None)
state.pop("start", None)
Expand Down Expand Up @@ -280,6 +344,9 @@ def setup(request, home):
raise SetupError("OpenCode requires a nonempty username and password.")

runtime = None
if (not port_available(port) and not owned_process(state) and state.get("password")
and adopt_listener(state, port, authenticates(port, credentials))):
save(state_path, state)
if not port_available(port) and not owned_process(state):
raise SetupError(f"Port {port} is occupied by another process. Stop it or use --new for a fresh agent.")
current = server_status(port, credentials)[1].get("version") if owned_process(state) else None
Expand Down
13 changes: 7 additions & 6 deletions src/commands/code.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3710,12 +3710,13 @@ const FLUSH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);

/// Flush the agent's filesystem so sleeping it cannot discard recent work.
///
/// `cloudAgentSleep` snapshots the disk without quiescing the guest, so pages
/// still dirty in its page cache are absent from the image the next wake
/// restores. Measured on a scratch agent: a file written a second before a
/// sleep was gone after waking, while the same write followed by `sync`
/// survived — including when the sleep was issued immediately on disconnect.
/// So the variable is the flush, not the timing.
/// Belt-and-braces: since monorepo #37022 a host-initiated stop syncs the
/// guest itself. Before that, `cloudAgentSleep` snapshotted the disk without
/// quiescing the guest, and a file written a second before a sleep was gone
/// after waking unless a `sync` ran first. The flush stays for RUNNING agents
/// only: connecting to a sleeping agent wakes it, so
/// [`crate::controllers::cloud_agent::sleep`] checks the status before calling
/// this.
///
/// A side connection is enough because `sync(2)` flushes the whole filesystem
/// regardless of which process dirtied it: this covers whatever the durable
Expand Down
Loading
Loading