Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
bef510b
fix(engine): 错过任务检测漏查时间规则,且 00:00 任务永远触发不到
AmberCXX Aug 21, 2026
7f63c1a
fix(engine): 已跑完的任务被重报成「错过」——错过判定加一道已触发闸
AmberCXX Aug 29, 2026
6e0cf07
feat(engine): 运行日志落盘 + 崩溃可见性
AmberCXX Sep 3, 2026
3669081
merge: 错过任务检测修复(PR #46)合入本地 main
AmberCXX Sep 4, 2026
ffd4051
merge: engine 运行日志落盘 + 崩溃可见性(PR #47)合入本地 main
AmberCXX Sep 4, 2026
cc25b27
fix(engine): passive 实例在 active 退出后重新竞选,不再终身旁观
AmberCXX Sep 8, 2026
45d1bb8
merge: passive 实例在 active 退出后重新竞选(PR #48)合入本地 main
AmberCXX Sep 8, 2026
45660dc
fix(feishu): 一条消息夹带多张图时只保留第一张、文字全丢
AmberCXX Sep 8, 2026
93ef36b
merge: 飞书多图消息只收到第一张(PR #49)合入本地 main
AmberCXX Sep 8, 2026
eaa6c78
fix(hub): 启动权归属不明导致 sync 静默失效——hub-client 的孤儿 Hub 顶掉 launchd
AmberCXX Sep 8, 2026
cda2075
merge: Hub 启动权归属修复(PR #50)合入本地 main
AmberCXX Sep 8, 2026
a9e8dc1
fix(feishu): send file with cwd-relative path so lark-cli accepts it
AmberCXX Sep 12, 2026
65475ea
fix(engine): isProcessAlive 补 self 判断,Linux CI 上 process.kill(self,0)…
AmberCXX Nov 2, 2026
9641b60
test(engine): PID lock 测试隔离统一走根 test-setup.ts,避免跨文件模块缓存把 DATA_DIR 顶回真实目录
AmberCXX Nov 2, 2026
20ea531
fix(engine): 数据目录在模块加载时建好,测试在 startScheduler 前写 PID 文件不再 ENOENT
AmberCXX Nov 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { cleanDirContents, HUB_INSTALL_PRESERVE_ENTRIES, syncApiTokenFileAt } from "./cli.js";
import { cleanDirContents, HUB_INSTALL_PRESERVE_ENTRIES, syncApiTokenFileAt, waitUntil } from "./cli.js";

let tmpDir: string;

Expand Down Expand Up @@ -75,3 +75,35 @@ describe("syncApiTokenFileAt", () => {
expect(fs.existsSync(attackerTarget)).toBe(false);
});
});

describe("waitUntil", () => {
test("returns true immediately when the predicate already holds", async () => {
let calls = 0;
const ok = await waitUntil(() => { calls += 1; return true; }, 1000, 10);

expect(ok).toBe(true);
expect(calls).toBe(1); // 已经成立就不该再轮询
});

test("returns true once the predicate flips partway through", async () => {
let n = 0;
const ok = await waitUntil(() => (n += 1) >= 3, 1000, 5);

expect(ok).toBe(true);
expect(n).toBe(3);
});

test("returns false when the predicate never holds before the timeout", async () => {
const ok = await waitUntil(() => false, 60, 10);

expect(ok).toBe(false); // 超时必须能报失败——否则调用方会把"没等到"当成"成功了"
});

test("awaits async predicates", async () => {
let n = 0;
const ok = await waitUntil(async () => (n += 1) >= 2, 1000, 5);

expect(ok).toBe(true);
expect(n).toBe(2);
});
});
57 changes: 54 additions & 3 deletions cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,29 @@ function installCmd(): void {
`);
}

/**
* 轮询 predicate 直到为真或超时,返回是否在超时前成立。
*
* 存在的理由:`launchctl bootout` / `bootstrap` 的退出码只说明命令被接受了,
* 不说明进程真的退出或真的起来了。把"发了命令"和"状态真的变了"分开,
* 否则就会出现"报告已重启、实际没换进程"。
*/
export async function waitUntil(
predicate: () => boolean | Promise<boolean>,
timeoutMs: number,
intervalMs = 250,
): Promise<boolean> {
const deadline = Date.now() + timeoutMs;
for (;;) {
if (await predicate()) return true;
if (Date.now() >= deadline) return false;
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
}

// ── sync ────────────────────────────────────────────────────────────────────

function syncCmd(): void {
async function syncCmd(): Promise<void> {
log("🔄 Forge Hub sync\n");

// 1. Re-stage package snapshot from current source
Expand Down Expand Up @@ -336,14 +356,45 @@ function syncCmd(): void {
const uid = os.userInfo().uid;
const domain = `gui/${uid}`;
const label = `${domain}/com.forge-hub`;
const baseUrl = process.env.FORGE_HUB_URL ?? "http://localhost:9900";

// 有任何 HTTP 响应就算活着——状态码不重要,能应答就说明端口后面有 Hub
const hubResponding = async (): Promise<boolean> => {
try {
await fetch(baseUrl, { signal: AbortSignal.timeout(1500) });
return true;
} catch {
return false;
}
};

try {
execFileSync("launchctl", ["bootout", label], { stdio: "ignore" });
} catch { /* might not be bootstrapped */ }

// bootout 的退出码只说明命令被接受,不说明进程已经退出。必须等它真的下线:
// 否则紧接着的 bootstrap 会撞上仍占着端口的旧进程,launchd 每 ThrottleInterval
// 崩一次,而这里照样打印"已重启"。
if (!(await waitUntil(async () => !(await hubResponding()), 15_000, 500))) {
log("⚠️ 旧 Hub 仍在响应,bootout 没能让它下线——它多半不是 launchd 启动的");
log(" (hub-client 在 Hub 不可达时会 detached spawn 一个,那种进程不是 launchd 的子进程,bootout 管不到)");
log(" 运行时文件已同步到磁盘,但跑着的仍是旧代码。手动处理:");
log(" lsof -nP -iTCP:9900 -sTCP:LISTEN # 找出占用者");
log(` kill <PID> && launchctl bootstrap ${domain} ${LAUNCHD_PLIST}`);
return;
}

try {
execFileSync("launchctl", ["bootstrap", domain, LAUNCHD_PLIST], { stdio: "inherit" });
log("✓ Hub 已重启");
} catch {
log(`⚠️ 无法重启 Hub。手动执行:launchctl bootout ${label} && launchctl bootstrap ${domain} ${LAUNCHD_PLIST}`);
return;
}

if (await waitUntil(hubResponding, 20_000, 500)) {
log("✓ Hub 已重启(已验证重新响应)");
} else {
log("⚠️ bootstrap 已提交,但 Hub 在 20s 内没有恢复响应——查 ~/.forge-hub/hub-stderr.log");
}
}

Expand Down Expand Up @@ -751,7 +802,7 @@ if (import.meta.main) {
installCmd();
break;
case "sync":
syncCmd();
await syncCmd();
break;
case "uninstall":
uninstallCmd();
Expand Down
14 changes: 14 additions & 0 deletions forge-engine/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,20 @@
> [!IMPORTANT]
> Forge Engine 目前是 **experimental / manual setup**。源码、MCP server 和 CLI 都在仓库里,但 **`forge-hub install` 默认不会部署或注册它**。想用的话,按下面步骤单独配置。

## 运行日志与崩溃可见性

`log()` / `logError()` 除了写 stderr,**同时追加到 `<DATA_DIR>/engine.log`**,超过 2MB 转存 `engine.log.1`(只留一份)。

**为什么需要文件日志**:stderr 是给 MCP 宿主看的,但宿主通常只在**连接建立那一刻**捕获 stderr —— engine 启动之后打印的任何东西都不再被记录。一旦进程异常退出,什么线索都不剩。

实测案例(2026-09-03):engine 在 00:00–03:00 之间消失,宿主日志无断开记录、系统无崩溃报告、无内存压力事件,调度静默停摆 **10 小时 31 分**、漏跑 12 个任务。事后无从判断死因。

**崩溃捕获**:`uncaughtException` 与 `unhandledRejection` 会把完整堆栈写进 `engine.log`,随后 `stopScheduler()` 释放 PID 锁并退出(exit 1)。

**为什么退出而不是继续跑**:调度器状态可能已损坏,而一个状态损坏的调度器发出的推送比不推送更坏。

**为什么不自动重起**:多实例 + 自动重起 + 抢 PID 锁是本项目已经踩过的坑(`cleanOrphans` + 一次性 `acquirePidLock` 的组合)。恢复由**外部**检测触发人工重连,engine 自己不做进程管理。

## 架构

```
Expand Down
49 changes: 48 additions & 1 deletion forge-engine/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* Forge Engine 路径常量与日志
*/

import fs from "node:fs";
import path from "node:path";

// ── Channel Identity ────────────────────────────────────────────────────────
Expand All @@ -16,6 +17,10 @@ const CODE_DIR = import.meta.dir;
const DATA_DIR = process.env.FORGE_ENGINE_DATA
?? path.join(process.env.HOME || "~", ".forge-hub", "engine-data");

// 数据目录在模块加载时就建好:测试会在 startScheduler 之前写 PID_FILE,
// 若目录未建会 ENOENT。
fs.mkdirSync(DATA_DIR, { recursive: true });

export const DIR = DATA_DIR;

export const CONFIG_FILE = path.join(DATA_DIR, "engine-config.json");
Expand All @@ -25,13 +30,55 @@ export const HANDLERS_DIR = path.resolve(CODE_DIR, "handlers");
export const SCHEDULE_FILE = path.join(DATA_DIR, "engine-schedule.json");
export const ACTION_LOG_FILE = path.join(DATA_DIR, "engine-trigger-log.md");
export const PID_FILE = path.join(DATA_DIR, "engine.pid");
export const RUNTIME_LOG_FILE = path.join(DATA_DIR, "engine.log");

// ── Logging ─────────────────────────────────────────────────────────────────
//
// stderr 是给宿主看的(stdout 被 MCP stdio 占用)。但宿主只在**连接建立那一刻**
// 捕获 stderr —— engine 启动之后打印的任何东西都掉进黑洞。
//
// 后果实测(2026-09-03):engine 在 00:00–03:00 之间死亡,无崩溃报告、无内存压力、
// 宿主日志里一条断开记录都没有,调度静默停摆 10 小时 31 分、漏跑 12 个任务。
// 它死之前很可能打印过原因,只是没人听见。
//
// 所以 log/logError 同时落盘。文件是唯一能在进程死后还留下痕迹的地方。

const MAX_LOG_BYTES = 2 * 1024 * 1024; // 2MB 转存一次,只留一份 .1

function appendToFile(line: string): void {
try {
fs.mkdirSync(DATA_DIR, { recursive: true });
// 轮转:日志本身不能变成下一个「不停长大且没人敢删」的文件
try {
if (fs.statSync(RUNTIME_LOG_FILE).size > MAX_LOG_BYTES) {
fs.renameSync(RUNTIME_LOG_FILE, RUNTIME_LOG_FILE + ".1");
}
} catch { /* 文件还不存在 —— 首次写入,正常 */ }
fs.appendFileSync(RUNTIME_LOG_FILE, line);
} catch { /* 日志写不进去不能反过来把进程搞死 */ }
}

// ── Logging (stderr — stdout is MCP stdio) ──────────────────────────────────
function stamp(): string {
return new Date().toISOString().replace("T", " ").slice(0, 19);
}

export function log(msg: string) {
process.stderr.write(`[engine] ${msg}\n`);
appendToFile(`${stamp()} [engine] ${msg}\n`);
}

export function logError(msg: string) {
process.stderr.write(`[engine] ERROR: ${msg}\n`);
appendToFile(`${stamp()} [engine] ERROR: ${msg}\n`);
}

/** 致命错误:带完整堆栈落盘。进程即将退出时用。 */
export function logFatal(kind: string, err: unknown): void {
// err.stack 本身已含 "Name: message" 首行,不要再拼一次
const detail = err instanceof Error
? (err.stack ?? `${err.name}: ${err.message}\n(无堆栈)`)
: String(err);
const block = `${stamp()} [engine] FATAL ${kind} · PID ${process.pid}\n${detail}\n`;
process.stderr.write(block);
appendToFile(block);
}
26 changes: 25 additions & 1 deletion forge-engine/engine-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
SCHEDULE_DIR,
log,
logError,
logFatal,
} from "./config.js";
import { startScheduler, stopScheduler } from "./scheduler.js";
import { resolveTaskTiming } from "./task-timing.js";
Expand Down Expand Up @@ -299,7 +300,30 @@ async function main() {
process.on("SIGTERM", () => shutdown("SIGTERM"));
process.on("SIGINT", () => shutdown("SIGINT"));

log("engine started");
// ── Crash Visibility ─────────────────────────────────────────────────────
//
// 2026-09-03:engine 在 00:00–03:00 之间消失,宿主日志里没有任何断开记录、
// 没有崩溃报告、没有内存压力事件,调度静默停摆 10h31m、漏跑 12 个任务。
//
// 定时器回调里的未捕获异常在 Bun 下会直接带走进程,而 log() 当时只写 stderr,
// 宿主又只在连接建立那一刻捕获 stderr —— **它死之前很可能喊过,只是没人听见。**
//
// 取舍:抓到之后**退出,不带病续跑**。scheduler 的状态可能已经坏了,
// 而一个状态损坏的调度器推送出去的东西,比不推送更坏。
// 退出后由外部检测(宿主侧的存活检查)发现并提示人工重连 —— 不做自动重起:
// 多实例 + 自动重起 + 抢 PID 锁,正是本项目已经踩过的坑。
process.on("uncaughtException", (err) => {
logFatal("uncaughtException", err);
stopScheduler(); // 释放 PID 锁,别留一个 stale 锁给下一个实例
process.exit(1);
});
process.on("unhandledRejection", (reason) => {
logFatal("unhandledRejection", reason);
stopScheduler();
process.exit(1);
});

log(`engine started · PID ${process.pid}`);
}

if (import.meta.main) {
Expand Down
87 changes: 87 additions & 0 deletions forge-engine/scheduler-pid-lock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { afterEach, describe, expect, test } from "bun:test";
import fs from "node:fs";
import type { Server } from "@modelcontextprotocol/sdk/server/index.js";

// FORGE_ENGINE_DATA 由根 test-setup.ts 统一隔离;这里不再自建 tempDir(否则会被跨文件模块缓存顶掉)。
import { PID_FILE } from "./config.js";
import {
startScheduler,
stopScheduler,
retryPassivePromotion,
isPassiveMode,
} from "./scheduler.js";

/** fire() 只用到 server.notification,最小替身足够。 */
function fakeServer(): Server {
return { notification: async () => {} } as unknown as Server;
}

/** 一个必定不存在的 PID,用来伪造崩溃进程留下的陈旧锁。 */
function deadPid(): number {
for (let pid = 40000; pid < 41000; pid++) {
try {
process.kill(pid, 0);
} catch {
return pid; // kill 抛错 = 进程不存在
}
}
throw new Error("找不到空闲 PID");
}

afterEach(() => {
stopScheduler();
try {
fs.unlinkSync(PID_FILE);
} catch {
/* 已经没了 */
}
});

describe("PID lock re-election", () => {
test("second instance enters passive mode while the lock holder is alive", async () => {
fs.writeFileSync(PID_FILE, String(process.pid)); // 本进程当然活着

await startScheduler(fakeServer());

expect(isPassiveMode()).toBe(true);
});

test("passive instance promotes itself once the lock holder exits", async () => {
fs.writeFileSync(PID_FILE, String(process.pid));
await startScheduler(fakeServer());
expect(isPassiveMode()).toBe(true);

fs.unlinkSync(PID_FILE); // 主实例正常退出会 releasePidLock()

const promoted = await retryPassivePromotion(fakeServer());

expect(promoted).toBe(true);
expect(isPassiveMode()).toBe(false);
expect(fs.readFileSync(PID_FILE, "utf-8").trim()).toBe(String(process.pid));
});

test("passive instance reclaims a stale lock left by a crashed holder", async () => {
fs.writeFileSync(PID_FILE, String(process.pid));
await startScheduler(fakeServer());
expect(isPassiveMode()).toBe(true);

fs.writeFileSync(PID_FILE, String(deadPid())); // 崩溃:文件还在,进程没了

const promoted = await retryPassivePromotion(fakeServer());

expect(promoted).toBe(true);
expect(isPassiveMode()).toBe(false);
});

test("passive instance stays passive while the holder is still alive", async () => {
fs.writeFileSync(PID_FILE, String(process.pid));
await startScheduler(fakeServer());
expect(isPassiveMode()).toBe(true);

const promoted = await retryPassivePromotion(fakeServer()); // 锁没释放

expect(promoted).toBe(false);
expect(isPassiveMode()).toBe(true);
expect(fs.readFileSync(PID_FILE, "utf-8").trim()).toBe(String(process.pid));
});
});
Loading
Loading