Lessons become skills with a category (workflow / pitfall / convention): one model, one prompt section, one permission rule. A repaired failure records or updates a pitfall skill deduplicated by failure signature; harvest sees every skill and may merge into an existing one; corroborated skills auto-activate per project (skillAutoActivation, default on). Runs given a skill are counted (skill_usage); platform-maintained skills retire after 3 failing runs in a row or 90 days unused. Skills can be imported from SKILL.md / zip, copied between projects and searched across projects. Migration 004 converts existing lesson rows to skills. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
260 lines
11 KiB
TypeScript
260 lines
11 KiB
TypeScript
/**
|
|
* Replaces every project of the cluster's WishForge with the two DroboticFlow projects it delivers
|
|
* (docs/cluster-poc-deployment.md): the POC (branch poc) and Fanyu's (branch dev). Run by
|
|
* `cluster-poc.mjs projects --replace` in the platform container, with nothing running.
|
|
*
|
|
* Deletes each project with all its wishes, runs, releases, environments, Runners, evidence and
|
|
* reviews, their run events, artifacts and Git repositories; keeps accounts, memberships, the
|
|
* usage ledger and the audit log. Then creates both projects, their environments and Runners, and
|
|
* writes the bindings and Runner registrations, which hold the Runners' tokens, to a private file
|
|
* the operator's Mac copies; it prints none of them.
|
|
*/
|
|
import { randomUUID } from 'node:crypto';
|
|
import { chmod, readdir, rm, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import pg from 'pg';
|
|
import { DeleteObjectsCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3';
|
|
import { PgStore } from '../apps/api/src/db/store.js';
|
|
import { ControlPlane, touch } from '../apps/api/src/domain/control-plane.js';
|
|
import { Releases } from '../apps/api/src/domain/releases.js';
|
|
import { RunnerBroker } from '../apps/api/src/domain/runner-broker.js';
|
|
import { loadPocProfile, localPocBindingSchema, type LocalPocBinding } from '../packages/runtime/src/index.js';
|
|
import type { Actor, Project } from '../packages/contracts/src/index.js';
|
|
|
|
/** Everything that belongs to a project; the rest is the tenant's own. */
|
|
export const PROJECT_KINDS = [
|
|
'project',
|
|
'wish',
|
|
'run',
|
|
'workspace',
|
|
'environment',
|
|
'release',
|
|
'runner',
|
|
'runner_task',
|
|
'approval',
|
|
'artifact',
|
|
'checkpoint',
|
|
'evidence',
|
|
'code_review',
|
|
'lesson',
|
|
'bundle',
|
|
'skill',
|
|
'skill_harvest',
|
|
'skill_usage',
|
|
];
|
|
/** Where the containers keep POC state; a rehearsal of this script names another directory. */
|
|
const POC_STATE = process.env.WF_CLUSTER_POC_STATE ?? '/var/lib/wishforge/poc-state';
|
|
const CHECK = ['/usr/local/bin/node', '/opt/wishforge/scripts/poc-project-check.mjs'];
|
|
const REPOSITORY = 'https://gitea.d-robotics.cc/longtao.wu/droboticflow.git';
|
|
/** Which code is backend, as the POC project said: frontend-only accounts may change the rest. */
|
|
const CODE_REVIEW = { paths: ['**'], exceptPaths: ['src/', 'tests/frontend/', '**/*.md'] };
|
|
export const PROJECTS = [
|
|
{
|
|
key: 'poc',
|
|
name: 'DroboticFlow POC',
|
|
environment: 'POC · droboticflow',
|
|
profileFile: `${POC_STATE}/poc-profile.json`,
|
|
stateDirectory: `${POC_STATE}/poc`,
|
|
// Who could wish for the POC before keep doing so.
|
|
members: ['local:qianduan'],
|
|
},
|
|
{
|
|
key: 'fanyu',
|
|
name: 'DroboticFlow Fanyu · dev',
|
|
environment: 'Fanyu · droboticflow-fanyu',
|
|
profileFile: `${POC_STATE}/fanyu-profile.json`,
|
|
stateDirectory: `${POC_STATE}/fanyu`,
|
|
// Fanyu serves its users: whoever else may wish for it, its owner grants in account management.
|
|
members: [],
|
|
},
|
|
] as const;
|
|
const OUTPUT = process.env.WF_CLUSTER_PROJECTS_OUTPUT ?? '/var/lib/wishforge/platform-state/projects-setup.json';
|
|
|
|
const env = { ...process.env };
|
|
if (env.WF_DEPLOYMENT_MODE !== 'development') throw Error('Explicit development mode required');
|
|
if (process.argv[2] !== '--replace') throw Error('Usage: cluster-poc-projects.ts --replace');
|
|
const tenantId = env.WF_TENANT_ID!;
|
|
|
|
async function deleteEverything() {
|
|
const client = new pg.Client({ connectionString: env.DATABASE_ADMIN_URL });
|
|
await client.connect();
|
|
try {
|
|
await client.query('begin');
|
|
const unfinished = await client.query(
|
|
`select id from entities where tenant_id = $1 and ((kind = 'run' and data->>'status' in ('queued', 'running')) or
|
|
(kind = 'runner_task' and data->>'status' in ('pending', 'dispatching', 'unknown')) or
|
|
(kind = 'release' and data->>'status' in ('ready', 'deploying', 'unknown')))`,
|
|
[tenantId],
|
|
);
|
|
if (unfinished.rowCount) throw Error(`Still working: ${unfinished.rows.map(row => row.id).join(', ')}`);
|
|
const projects = (
|
|
await client.query(`select id from entities where tenant_id = $1 and kind = 'project'`, [tenantId])
|
|
).rows.map(row => row.id as string);
|
|
const runs = (
|
|
await client.query(`select id from entities where tenant_id = $1 and kind = 'run'`, [tenantId])
|
|
).rows.map(row => row.id as string);
|
|
const events = await client.query(`delete from run_events where tenant_id = $1 and run_id = any($2)`, [
|
|
tenantId,
|
|
runs,
|
|
]);
|
|
const entities = await client.query(`delete from entities where tenant_id = $1 and kind = any($2)`, [
|
|
tenantId,
|
|
PROJECT_KINDS,
|
|
]);
|
|
await client.query('commit');
|
|
return { projects, runs, runEvents: events.rowCount ?? 0, entities: entities.rowCount ?? 0 };
|
|
} catch (error) {
|
|
await client.query('rollback').catch(() => undefined);
|
|
throw error;
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
/** The projects' artifacts and hosted Git repositories, which nothing refers to any more. */
|
|
async function deleteFiles(projects: string[]) {
|
|
const s3 = new S3Client({
|
|
endpoint: env.S3_ENDPOINT,
|
|
region: env.S3_REGION ?? 'us-east-1',
|
|
forcePathStyle: true,
|
|
credentials: { accessKeyId: env.S3_ACCESS_KEY_ID!, secretAccessKey: env.S3_SECRET_ACCESS_KEY! },
|
|
});
|
|
const prefix = env.S3_ARTIFACT_PREFIX ?? 'wishforge';
|
|
let objects = 0;
|
|
for (const project of projects)
|
|
for (let token: string | undefined; ;) {
|
|
const listed = await s3.send(
|
|
new ListObjectsV2Command({
|
|
Bucket: env.S3_BUCKET,
|
|
Prefix: `${prefix}/tenants/${tenantId}/projects/${project}/`,
|
|
ContinuationToken: token,
|
|
}),
|
|
);
|
|
const keys = (listed.Contents ?? []).map(item => ({ Key: item.Key! }));
|
|
if (keys.length) {
|
|
await s3.send(new DeleteObjectsCommand({ Bucket: env.S3_BUCKET, Delete: { Objects: keys } }));
|
|
objects += keys.length;
|
|
}
|
|
if (!listed.IsTruncated) break;
|
|
token = listed.NextContinuationToken;
|
|
}
|
|
const gitRoot = path.join(env.WF_GIT_ROOT!, tenantId);
|
|
let repositories = 0;
|
|
for (const name of await readdir(gitRoot).catch(() => []))
|
|
if (projects.includes(name.replace(/\.git$/, ''))) {
|
|
await rm(path.join(gitRoot, name), { recursive: true, force: true });
|
|
repositories++;
|
|
}
|
|
// Builds and deployment journals of the releases that are gone.
|
|
for (const project of PROJECTS) await rm(project.stateDirectory, { recursive: true, force: true });
|
|
return { objects, repositories };
|
|
}
|
|
|
|
const store = new PgStore(env.DATABASE_URL!);
|
|
try {
|
|
await store.assertSafeRole();
|
|
// The tenant's owner, as the operator bootstrap created it (db:seed).
|
|
const ownerId = '11111111-1111-4111-8111-111111111111';
|
|
const owner = (await store.read(tenantId, tx => tx.membership(ownerId))) === 'owner' ? ownerId : undefined;
|
|
if (!owner) throw Error('The tenant has no owner to create the projects as');
|
|
const actor: Actor = { tenantId, userId: owner, kind: 'human', role: 'owner' };
|
|
for (const project of PROJECTS) await loadPocProfile(project.profileFile);
|
|
|
|
const deleted = await deleteEverything();
|
|
const files = await deleteFiles(deleted.projects);
|
|
|
|
const control = new ControlPlane(store, env);
|
|
const bindings: LocalPocBinding[] = [];
|
|
const created: { key: string; projectId: string; environmentId: string }[] = [];
|
|
for (const spec of PROJECTS) {
|
|
const profile = await loadPocProfile(spec.profileFile);
|
|
const project = await control.createProject(actor, {
|
|
name: spec.name,
|
|
repositoryUrl: REPOSITORY,
|
|
targetBranch: profile.branch,
|
|
});
|
|
const environment = await new Releases(store, env).createEnvironment(actor, {
|
|
projectId: project.id,
|
|
name: spec.environment,
|
|
type: 'kubernetes',
|
|
production: false,
|
|
// Every release waits for a person's approval.
|
|
autoDeploy: false,
|
|
architecture: 'linux/amd64',
|
|
configVersion: 'adopt-existing-v1',
|
|
secretVersionRefs: [],
|
|
});
|
|
bindings.push(
|
|
localPocBindingSchema.parse({
|
|
schemaVersion: 1,
|
|
tenantId,
|
|
projectId: project.id,
|
|
environmentId: environment.id,
|
|
repositoryUrl: REPOSITORY,
|
|
targetBranch: profile.branch,
|
|
profileFile: spec.profileFile,
|
|
stateDirectory: spec.stateDirectory,
|
|
gitHome: '/home/wishforge',
|
|
}),
|
|
);
|
|
created.push({ key: spec.key, projectId: project.id, environmentId: environment.id });
|
|
}
|
|
// The application addresses are plain HTTP, which only bound environments may use.
|
|
const bindingFile = `/tmp/projects-${randomUUID()}.json`;
|
|
await writeFile(bindingFile, JSON.stringify(bindings), { mode: 0o600 });
|
|
const bound = { ...env, WF_LOCAL_POC_BINDING_FILE: bindingFile };
|
|
const runners: Record<string, { environmentId: string; token: string; publicKeys: unknown }> = {};
|
|
try {
|
|
for (const [index, spec] of PROJECTS.entries()) {
|
|
const profile = await loadPocProfile(spec.profileFile);
|
|
const { projectId, environmentId } = created[index]!;
|
|
const releases = new Releases(store, bound);
|
|
await releases.updateApplicationUrl(actor, environmentId, { applicationUrl: profile.applicationUrl });
|
|
await store.transaction(tenantId, async tx => {
|
|
const project = await tx.get<Project>('project', projectId);
|
|
if (!project) throw Error('Project disappeared');
|
|
project.installCommands = [[...CHECK, 'install']];
|
|
project.verificationCommands = [[...CHECK, 'verify']];
|
|
project.autoDeploy = true;
|
|
project.codeReview = CODE_REVIEW;
|
|
project.memberIds = [...new Set([...project.memberIds, ...spec.members])];
|
|
project.deliveryConfig = {
|
|
environmentId,
|
|
imageRepository: `${profile.registry}/droboticflow-api`,
|
|
architecture: 'linux/amd64',
|
|
dockerfile: 'Dockerfile.api',
|
|
sourceContext: '.',
|
|
containerPort: 3001,
|
|
healthPath: '/api/health',
|
|
smokeChecks: [
|
|
{ path: '/api/ready', expectedStatus: 200, bodyIncludes: '"ready":true' },
|
|
{ path: '/', expectedStatus: 200, bodyIncludes: '<div id="root">' },
|
|
],
|
|
};
|
|
await tx.put('project', touch(project));
|
|
});
|
|
const registration = await new RunnerBroker(store, releases, bound).register(actor, environmentId);
|
|
runners[spec.key] = { environmentId, token: registration.token, publicKeys: registration.publicKeys };
|
|
}
|
|
} finally {
|
|
await rm(bindingFile, { force: true });
|
|
}
|
|
await writeFile(OUTPUT, `${JSON.stringify({ tenantId, bindings, runners }, null, 2)}\n`, { mode: 0o600 });
|
|
await chmod(OUTPUT, 0o600);
|
|
console.log(
|
|
JSON.stringify({
|
|
deleted: {
|
|
projects: deleted.projects.length,
|
|
runs: deleted.runs.length,
|
|
entities: deleted.entities,
|
|
runEvents: deleted.runEvents,
|
|
...files,
|
|
},
|
|
created,
|
|
output: OUTPUT,
|
|
}),
|
|
);
|
|
} finally {
|
|
await store.close();
|
|
}
|