wishforge-ide/scripts/cluster-poc-projects.ts
eust-w 0e4ef42771 ♻️ Unify project lessons into self-evolving project skills
Lessons become skills with a category (workflow / pitfall / convention):
one model, one prompt section, one permission rule. A repaired failure
records or updates a pitfall skill deduplicated by failure signature;
harvest sees every skill and may merge into an existing one; corroborated
skills auto-activate per project (skillAutoActivation, default on).
Runs given a skill are counted (skill_usage); platform-maintained skills
retire after 3 failing runs in a row or 90 days unused. Skills can be
imported from SKILL.md / zip, copied between projects and searched across
projects. Migration 004 converts existing lesson rows to skills.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-04 12:37:36 +08:00

260 lines
11 KiB
TypeScript

/**
* Replaces every project of the cluster's WishForge with the two DroboticFlow projects it delivers
* (docs/cluster-poc-deployment.md): the POC (branch poc) and Fanyu's (branch dev). Run by
* `cluster-poc.mjs projects --replace` in the platform container, with nothing running.
*
* Deletes each project with all its wishes, runs, releases, environments, Runners, evidence and
* reviews, their run events, artifacts and Git repositories; keeps accounts, memberships, the
* usage ledger and the audit log. Then creates both projects, their environments and Runners, and
* writes the bindings and Runner registrations, which hold the Runners' tokens, to a private file
* the operator's Mac copies; it prints none of them.
*/
import { randomUUID } from 'node:crypto';
import { chmod, readdir, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';
import pg from 'pg';
import { DeleteObjectsCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
import { PgStore } from '../apps/api/src/db/store.js';
import { ControlPlane, touch } from '../apps/api/src/domain/control-plane.js';
import { Releases } from '../apps/api/src/domain/releases.js';
import { RunnerBroker } from '../apps/api/src/domain/runner-broker.js';
import { loadPocProfile, localPocBindingSchema, type LocalPocBinding } from '../packages/runtime/src/index.js';
import type { Actor, Project } from '../packages/contracts/src/index.js';
/** Everything that belongs to a project; the rest is the tenant's own. */
export const PROJECT_KINDS = [
'project',
'wish',
'run',
'workspace',
'environment',
'release',
'runner',
'runner_task',
'approval',
'artifact',
'checkpoint',
'evidence',
'code_review',
'lesson',
'bundle',
'skill',
'skill_harvest',
'skill_usage',
];
/** Where the containers keep POC state; a rehearsal of this script names another directory. */
const POC_STATE = process.env.WF_CLUSTER_POC_STATE ?? '/var/lib/wishforge/poc-state';
const CHECK = ['/usr/local/bin/node', '/opt/wishforge/scripts/poc-project-check.mjs'];
const REPOSITORY = 'https://gitea.d-robotics.cc/longtao.wu/droboticflow.git';
/** Which code is backend, as the POC project said: frontend-only accounts may change the rest. */
const CODE_REVIEW = { paths: ['**'], exceptPaths: ['src/', 'tests/frontend/', '**/*.md'] };
export const PROJECTS = [
{
key: 'poc',
name: 'DroboticFlow POC',
environment: 'POC · droboticflow',
profileFile: `${POC_STATE}/poc-profile.json`,
stateDirectory: `${POC_STATE}/poc`,
// Who could wish for the POC before keep doing so.
members: ['local:qianduan'],
},
{
key: 'fanyu',
name: 'DroboticFlow Fanyu · dev',
environment: 'Fanyu · droboticflow-fanyu',
profileFile: `${POC_STATE}/fanyu-profile.json`,
stateDirectory: `${POC_STATE}/fanyu`,
// Fanyu serves its users: whoever else may wish for it, its owner grants in account management.
members: [],
},
] as const;
const OUTPUT = process.env.WF_CLUSTER_PROJECTS_OUTPUT ?? '/var/lib/wishforge/platform-state/projects-setup.json';
const env = { ...process.env };
if (env.WF_DEPLOYMENT_MODE !== 'development') throw Error('Explicit development mode required');
if (process.argv[2] !== '--replace') throw Error('Usage: cluster-poc-projects.ts --replace');
const tenantId = env.WF_TENANT_ID!;
async function deleteEverything() {
const client = new pg.Client({ connectionString: env.DATABASE_ADMIN_URL });
await client.connect();
try {
await client.query('begin');
const unfinished = await client.query(
`select id from entities where tenant_id = $1 and ((kind = 'run' and data->>'status' in ('queued', 'running')) or
(kind = 'runner_task' and data->>'status' in ('pending', 'dispatching', 'unknown')) or
(kind = 'release' and data->>'status' in ('ready', 'deploying', 'unknown')))`,
[tenantId],
);
if (unfinished.rowCount) throw Error(`Still working: ${unfinished.rows.map(row => row.id).join(', ')}`);
const projects = (
await client.query(`select id from entities where tenant_id = $1 and kind = 'project'`, [tenantId])
).rows.map(row => row.id as string);
const runs = (
await client.query(`select id from entities where tenant_id = $1 and kind = 'run'`, [tenantId])
).rows.map(row => row.id as string);
const events = await client.query(`delete from run_events where tenant_id = $1 and run_id = any($2)`, [
tenantId,
runs,
]);
const entities = await client.query(`delete from entities where tenant_id = $1 and kind = any($2)`, [
tenantId,
PROJECT_KINDS,
]);
await client.query('commit');
return { projects, runs, runEvents: events.rowCount ?? 0, entities: entities.rowCount ?? 0 };
} catch (error) {
await client.query('rollback').catch(() => undefined);
throw error;
} finally {
await client.end();
}
}
/** The projects' artifacts and hosted Git repositories, which nothing refers to any more. */
async function deleteFiles(projects: string[]) {
const s3 = new S3Client({
endpoint: env.S3_ENDPOINT,
region: env.S3_REGION ?? 'us-east-1',
forcePathStyle: true,
credentials: { accessKeyId: env.S3_ACCESS_KEY_ID!, secretAccessKey: env.S3_SECRET_ACCESS_KEY! },
});
const prefix = env.S3_ARTIFACT_PREFIX ?? 'wishforge';
let objects = 0;
for (const project of projects)
for (let token: string | undefined; ;) {
const listed = await s3.send(
new ListObjectsV2Command({
Bucket: env.S3_BUCKET,
Prefix: `${prefix}/tenants/${tenantId}/projects/${project}/`,
ContinuationToken: token,
}),
);
const keys = (listed.Contents ?? []).map(item => ({ Key: item.Key! }));
if (keys.length) {
await s3.send(new DeleteObjectsCommand({ Bucket: env.S3_BUCKET, Delete: { Objects: keys } }));
objects += keys.length;
}
if (!listed.IsTruncated) break;
token = listed.NextContinuationToken;
}
const gitRoot = path.join(env.WF_GIT_ROOT!, tenantId);
let repositories = 0;
for (const name of await readdir(gitRoot).catch(() => []))
if (projects.includes(name.replace(/\.git$/, ''))) {
await rm(path.join(gitRoot, name), { recursive: true, force: true });
repositories++;
}
// Builds and deployment journals of the releases that are gone.
for (const project of PROJECTS) await rm(project.stateDirectory, { recursive: true, force: true });
return { objects, repositories };
}
const store = new PgStore(env.DATABASE_URL!);
try {
await store.assertSafeRole();
// The tenant's owner, as the operator bootstrap created it (db:seed).
const ownerId = '11111111-1111-4111-8111-111111111111';
const owner = (await store.read(tenantId, tx => tx.membership(ownerId))) === 'owner' ? ownerId : undefined;
if (!owner) throw Error('The tenant has no owner to create the projects as');
const actor: Actor = { tenantId, userId: owner, kind: 'human', role: 'owner' };
for (const project of PROJECTS) await loadPocProfile(project.profileFile);
const deleted = await deleteEverything();
const files = await deleteFiles(deleted.projects);
const control = new ControlPlane(store, env);
const bindings: LocalPocBinding[] = [];
const created: { key: string; projectId: string; environmentId: string }[] = [];
for (const spec of PROJECTS) {
const profile = await loadPocProfile(spec.profileFile);
const project = await control.createProject(actor, {
name: spec.name,
repositoryUrl: REPOSITORY,
targetBranch: profile.branch,
});
const environment = await new Releases(store, env).createEnvironment(actor, {
projectId: project.id,
name: spec.environment,
type: 'kubernetes',
production: false,
// Every release waits for a person's approval.
autoDeploy: false,
architecture: 'linux/amd64',
configVersion: 'adopt-existing-v1',
secretVersionRefs: [],
});
bindings.push(
localPocBindingSchema.parse({
schemaVersion: 1,
tenantId,
projectId: project.id,
environmentId: environment.id,
repositoryUrl: REPOSITORY,
targetBranch: profile.branch,
profileFile: spec.profileFile,
stateDirectory: spec.stateDirectory,
gitHome: '/home/wishforge',
}),
);
created.push({ key: spec.key, projectId: project.id, environmentId: environment.id });
}
// The application addresses are plain HTTP, which only bound environments may use.
const bindingFile = `/tmp/projects-${randomUUID()}.json`;
await writeFile(bindingFile, JSON.stringify(bindings), { mode: 0o600 });
const bound = { ...env, WF_LOCAL_POC_BINDING_FILE: bindingFile };
const runners: Record<string, { environmentId: string; token: string; publicKeys: unknown }> = {};
try {
for (const [index, spec] of PROJECTS.entries()) {
const profile = await loadPocProfile(spec.profileFile);
const { projectId, environmentId } = created[index]!;
const releases = new Releases(store, bound);
await releases.updateApplicationUrl(actor, environmentId, { applicationUrl: profile.applicationUrl });
await store.transaction(tenantId, async tx => {
const project = await tx.get<Project>('project', projectId);
if (!project) throw Error('Project disappeared');
project.installCommands = [[...CHECK, 'install']];
project.verificationCommands = [[...CHECK, 'verify']];
project.autoDeploy = true;
project.codeReview = CODE_REVIEW;
project.memberIds = [...new Set([...project.memberIds, ...spec.members])];
project.deliveryConfig = {
environmentId,
imageRepository: `${profile.registry}/droboticflow-api`,
architecture: 'linux/amd64',
dockerfile: 'Dockerfile.api',
sourceContext: '.',
containerPort: 3001,
healthPath: '/api/health',
smokeChecks: [
{ path: '/api/ready', expectedStatus: 200, bodyIncludes: '"ready":true' },
{ path: '/', expectedStatus: 200, bodyIncludes: '<div id="root">' },
],
};
await tx.put('project', touch(project));
});
const registration = await new RunnerBroker(store, releases, bound).register(actor, environmentId);
runners[spec.key] = { environmentId, token: registration.token, publicKeys: registration.publicKeys };
}
} finally {
await rm(bindingFile, { force: true });
}
await writeFile(OUTPUT, `${JSON.stringify({ tenantId, bindings, runners }, null, 2)}\n`, { mode: 0o600 });
await chmod(OUTPUT, 0o600);
console.log(
JSON.stringify({
deleted: {
projects: deleted.projects.length,
runs: deleted.runs.length,
entities: deleted.entities,
runEvents: deleted.runEvents,
...files,
},
created,
output: OUTPUT,
}),
);
} finally {
await store.close();
}