🐛 Wait for local services to stop before rebuilding
Reproduce and fix the launchd SIGTERMed shutdown race; wait for owned processes to exit and report timeouts. Describe local task workspaces without claiming OS isolation.
This commit is contained in:
parent
6fc7eaf00c
commit
24c8b8b6b8
@ -8,7 +8,7 @@ import { ButtonLink, EmptyState, ErrorNotice, Loading, PageHeader, Panel } from
|
||||
import { RunTable } from '@/components/run-table';
|
||||
const journey = [
|
||||
['许愿与澄清', '将想法转化为可执行的任务,与 Agent 一起澄清细节。'],
|
||||
['隔离执行', '在隔离的环境中完成构建、测试与验证。'],
|
||||
['独立工作区', '在独立任务分支实现代码,按环境配置完成构建与验证。'],
|
||||
['证据验证', '收集执行结果与证据,确认符合预期。'],
|
||||
['发布审批', '绑定确切的代码与制品,完成发布评审。'],
|
||||
['部署与观察', '部署到目标环境,并持续观察运行状态。'],
|
||||
|
||||
@ -18,6 +18,8 @@ node scripts/local-poc-service.mjs start
|
||||
|
||||
首次切换到后台服务时,先由原启动终端正常结束本任务的前台 `scripts/dev.mjs` 与 Runner,再执行 `install`。不要使用 `pkill node`、按端口批量杀进程或停止 Colima。已有依赖服务继续运行;本脚本不启动、停止或重建 Docker、Colima、数据库和 Temporal。
|
||||
|
||||
`stop` 会等待 launchd 完成注销,并确认停止前记录的 supervisor、platform、Runner 进程已退出。只有 `stopCompleted=true` 才返回成功;如果仍处于 `SIGTERMed`、子进程尚未退出或无法读取进程状态,它最多等待约 45 秒后返回非零状态及诊断。遇到这种情况先重新检查 `status`,不要立即接着构建或启动,也不要通过批量杀进程绕过检查。
|
||||
|
||||
监督进程只启动两个直接子进程:`scripts/dev.mjs` 与 `node --env-file=.wishforge/local-poc/runner.env --import tsx apps/runner/src/main.ts`。前者自行读取根目录 `.env.local`;后者只获得最小系统环境和独立 `runner.env`。Runner 不继承根目录 API、数据库或模型凭据。子进程退出后整个本任务服务组退出,由 launchd 在节流间隔后重新启动。停止仅处理该 LaunchAgent 及它创建的子进程。
|
||||
|
||||
## 入口与登录
|
||||
|
||||
@ -15,6 +15,7 @@ import {
|
||||
import { homedir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { localPocDoctor, waitForReadiness } from './local-poc-doctor.mjs';
|
||||
|
||||
export const LABEL = 'cc.d-robotics.wishforge.local-poc';
|
||||
@ -166,6 +167,60 @@ function status() {
|
||||
return processStatus(result, state);
|
||||
}
|
||||
|
||||
function remainingOwnedProcesses(tracked) {
|
||||
if (!tracked.length) return [];
|
||||
const result = spawnSync('/bin/ps', ['-axo', 'pid=,command='], { encoding: 'utf8', timeout: 3000 });
|
||||
if (result.status !== 0) return null;
|
||||
const current = new Map(
|
||||
result.stdout.split('\n').flatMap(line => {
|
||||
const match = line.trim().match(/^(\d+)\s+(.+)$/);
|
||||
return match ? [[Number(match[1]), match[2]]] : [];
|
||||
}),
|
||||
);
|
||||
// Parent IDs can change while launchd tears down the service. Match the captured
|
||||
// PID and its known script, so an orphan is still awaited and a reused PID is not.
|
||||
return tracked.filter(item => current.get(item.pid)?.includes(item.script)).map(item => item.pid);
|
||||
}
|
||||
|
||||
export async function stopLocalService({
|
||||
inspect = status,
|
||||
control = launchctl,
|
||||
remaining = remainingOwnedProcesses,
|
||||
timeoutMs = 45000,
|
||||
intervalMs = 500,
|
||||
now = Date.now,
|
||||
sleep = delay,
|
||||
} = {}) {
|
||||
const deadline = now() + timeoutMs;
|
||||
const before = inspect();
|
||||
const tracked = [
|
||||
...(before.pid ? [{ pid: before.pid, script: SCRIPT }] : []),
|
||||
...(before.children ?? []).map(child => ({
|
||||
pid: child.pid,
|
||||
script: path.join(ROOT, child.name === 'platform' ? 'scripts/dev.mjs' : 'apps/runner/src/main.ts'),
|
||||
})),
|
||||
];
|
||||
control(['disable', SERVICE]);
|
||||
if (before.registered) control(['bootout', SERVICE]);
|
||||
while (true) {
|
||||
const report = inspect();
|
||||
const active = remaining(tracked);
|
||||
if (!report.registered && active !== null && !active.length)
|
||||
return { ...report, stopCompleted: true, remainingOwnedProcesses: [] };
|
||||
if (now() >= deadline)
|
||||
return {
|
||||
...report,
|
||||
stopCompleted: false,
|
||||
remainingOwnedProcesses: active,
|
||||
stopDetail:
|
||||
active === null
|
||||
? 'Could not verify owned process termination; inspect status before building or starting'
|
||||
: 'Timed out waiting for launchd removal and owned process termination; inspect status before building or starting',
|
||||
};
|
||||
await sleep(Math.min(intervalMs, Math.max(0, deadline - now())));
|
||||
}
|
||||
}
|
||||
|
||||
async function supervise() {
|
||||
preflight();
|
||||
process.umask(0o077);
|
||||
@ -273,8 +328,9 @@ export async function main(argv = process.argv.slice(2)) {
|
||||
if (!registered()) launchctl(['bootstrap', DOMAIN, PLIST]);
|
||||
launchctl(['kickstart', SERVICE]);
|
||||
} else if (action === 'stop') {
|
||||
launchctl(['disable', SERVICE]);
|
||||
if (registered()) launchctl(['bootout', SERVICE]);
|
||||
const report = await stopLocalService();
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
return report.stopCompleted ? 0 : 1;
|
||||
}
|
||||
if (action === 'install' || action === 'start') {
|
||||
const report = await waitForReadiness(() => localPocDoctor(ROOT, status()));
|
||||
|
||||
@ -11,8 +11,72 @@ import {
|
||||
parseServiceAction,
|
||||
processStatus,
|
||||
rotateAppend,
|
||||
stopLocalService,
|
||||
} from './local-poc-service.mjs';
|
||||
|
||||
test('stop waits through SIGTERMed and an orphaned owned child before reporting completion', async () => {
|
||||
let time = 0,
|
||||
inspected = 0,
|
||||
checked = 0;
|
||||
const calls = [];
|
||||
const snapshots = [
|
||||
{ registered: true, state: 'running', pid: 100, children: [{ name: 'runner', pid: 101 }] },
|
||||
{ registered: true, state: 'SIGTERMed', pid: 100, children: [{ name: 'runner', pid: 101 }] },
|
||||
{ registered: false, state: 'stopped', pid: null, children: [] },
|
||||
{ registered: false, state: 'stopped', pid: null, children: [] },
|
||||
];
|
||||
const result = await stopLocalService({
|
||||
inspect: () => snapshots[Math.min(inspected++, snapshots.length - 1)],
|
||||
control: args => {
|
||||
calls.push(args[0]);
|
||||
},
|
||||
remaining: tracked => {
|
||||
assert.ok(tracked.some(item => item.pid === 101));
|
||||
return ++checked < 3 ? [101] : [];
|
||||
},
|
||||
now: () => time,
|
||||
sleep: async ms => {
|
||||
time += ms;
|
||||
},
|
||||
intervalMs: 10,
|
||||
timeoutMs: 100,
|
||||
});
|
||||
assert.equal(result.stopCompleted, true);
|
||||
assert.equal(result.registered, false);
|
||||
assert.equal(time, 20);
|
||||
assert.deepEqual(calls, ['disable', 'bootout']);
|
||||
});
|
||||
|
||||
test('stop has a bounded deadline and returns a failed diagnostic when owned processes remain', async () => {
|
||||
let time = 0;
|
||||
const result = await stopLocalService({
|
||||
inspect: () => ({ registered: true, state: 'SIGTERMed', pid: 100, children: [] }),
|
||||
control: () => undefined,
|
||||
remaining: () => [100],
|
||||
now: () => time,
|
||||
sleep: async ms => {
|
||||
time += ms;
|
||||
},
|
||||
intervalMs: 10,
|
||||
timeoutMs: 30,
|
||||
});
|
||||
assert.equal(result.stopCompleted, false);
|
||||
assert.deepEqual(result.remainingOwnedProcesses, [100]);
|
||||
assert.equal(time, 30);
|
||||
});
|
||||
|
||||
test('stop does not report success when process termination cannot be checked', async () => {
|
||||
const result = await stopLocalService({
|
||||
inspect: () => ({ registered: false, state: 'stopped', pid: null, children: [] }),
|
||||
control: () => undefined,
|
||||
remaining: () => null,
|
||||
timeoutMs: 0,
|
||||
});
|
||||
assert.equal(result.stopCompleted, false);
|
||||
assert.equal(result.remainingOwnedProcesses, null);
|
||||
assert.match(result.stopDetail, /Could not verify/);
|
||||
});
|
||||
|
||||
test('a restarted supervisor must not report children retained from an older process', () => {
|
||||
const report = processStatus(
|
||||
{ status: 0, stdout: 'state = running\n pid = 200\n' },
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user