wishforge-ide/tests/skills.test.ts
eust-w 0e4ef42771 ♻️ Unify project lessons into self-evolving project skills
Lessons become skills with a category (workflow / pitfall / convention):
one model, one prompt section, one permission rule. A repaired failure
records or updates a pitfall skill deduplicated by failure signature;
harvest sees every skill and may merge into an existing one; corroborated
skills auto-activate per project (skillAutoActivation, default on).
Runs given a skill are counted (skill_usage); platform-maintained skills
retire after 3 failing runs in a row or 90 days unused. Skills can be
imported from SKILL.md / zip, copied between projects and searched across
projects. Migration 004 converts existing lesson rows to skills.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-04 12:37:36 +08:00

474 lines
21 KiB
TypeScript

/**
* Project skills as the one home of a project's know-how (requirements FR-I2): pitfalls learned
* from repaired failures, harvested proposals merged and activated, effectiveness counted and
* retired, and skills moved between projects as SKILL.md files, zips or copies.
*/
import { describe, expect, it } from 'vitest';
import { crc32, deflateRawSync } from 'node:zlib';
import type { Actor, FailureSummary, Project, Run, Skill } from '@wishforge/contracts';
import {
RETIRE_AFTER_FAILURES,
RETIRE_UNUSED_DAYS,
Skills,
failureSignature,
learnFromRun,
projectSkills,
recordSkillUse,
settleSkillUsage,
} from '../apps/api/src/domain/skills.js';
import { parseSkillMarkdown, readSkillArchive, skillMarkdown, zipFiles } from '../apps/api/src/domain/skill-files.js';
import { MemoryStore } from './helpers/memory-store.js';
const TENANT = 'demo';
const owner: Actor = { tenantId: TENANT, userId: 'owner', role: 'owner', kind: 'human' };
const viewer: Actor = { tenantId: TENANT, userId: 'viewer', role: 'viewer', kind: 'human' };
const stamp = (id: string) => {
const time = new Date().toISOString();
return { id, tenantId: TENANT, version: 1, createdAt: time, updatedAt: time };
};
const failure = (argv = ['npm', 'test'], tail = 'Error: expected 2 to equal 3'): FailureSummary => ({
kind: 'verification',
evidenceId: 'evidence',
sourceCommit: 'a'.repeat(40),
argv,
exitCode: 1,
outputTail: tail,
});
function setup() {
const store = new MemoryStore();
store.seed(TENANT);
const answers: unknown[] = [];
const asked: string[] = [];
const skills = new Skills(store, { WF_TENANT_ID: TENANT } as NodeJS.ProcessEnv, async input => {
asked.push(input.prompt);
return answers.shift() ?? { skills: [] };
});
const project = async (id: string, extra: Partial<Project> = {}) => {
const value = {
...stamp(id),
name: `Project ${id}`,
repositoryStorage: 'platform',
repositoryStatus: 'ready',
targetBranch: 'main',
memberIds: ['viewer'],
verificationCommands: [],
autoDeploy: false,
...extra,
} as Project;
await store.transaction(TENANT, tx => tx.put('project', value));
return value;
};
const run = async (id: string, projectId: string, extra: Partial<Run> = {}) => {
const value = {
...stamp(id),
projectId,
wishId: `wish-${id}`,
goal: 'Change the greeting',
acceptanceCriteria: ['check passes'],
status: 'running',
stage: 'implement',
repairCount: 0,
...extra,
} as unknown as Run;
await store.transaction(TENANT, tx => tx.put('run', value));
return value;
};
/** The run ends: its status changes and, for a success, the control plane learns from it. */
const finish = (value: Run, status: Run['status'], extra: Partial<Run> = {}) =>
store.transaction(TENANT, async tx => {
const next = { ...value, ...extra, status };
await tx.put('run', next);
if (status === 'succeeded') await learnFromRun(tx, next);
return next;
});
const use = (value: Run) => store.transaction(TENANT, tx => recordSkillUse(tx, value));
const all = (projectId: string) => skills.list(owner, projectId);
return { store, skills, answers, asked, project, run, finish, use, all };
}
describe('pitfall skills from repaired failures', () => {
it('turns the same failure in several runs into one pitfall, active right away', async () => {
const { project, run, finish, all, store } = setup();
await project('p1');
const first = await run('r1', 'p1');
await finish(first, 'succeeded', {
repairCount: 1,
lastFailure: failure(),
implementationSummary: 'Built before testing.',
});
const second = await run('r2', 'p1');
await finish(second, 'succeeded', {
repairCount: 1,
lastFailure: failure(['npm', ' test'], 'FAIL other.test.ts'),
implementationSummary: 'Added a fixture.',
});
// Learning from the same run again changes nothing.
await finish(second, 'succeeded', { repairCount: 1, lastFailure: failure(), implementationSummary: 'x' });
const skills = await all('p1');
expect(skills).toHaveLength(1);
const pitfall = skills[0]!;
expect(pitfall).toMatchObject({
name: 'pitfall-npm-test',
category: 'pitfall',
status: 'active',
signature: 'verification:npm test',
sourceRunIds: ['r1', 'r2'],
createdBy: 'wishforge',
});
// Most recent first.
expect(pitfall.content.indexOf('Added a fixture')).toBeLessThan(pitfall.content.indexOf('Built before testing'));
const events = await store.read(TENANT, tx => tx.events('r2'));
expect(events.map(event => event.type)).toEqual(['skill.updated']);
// A different command is a different pitfall; a conflict is identified by its files.
expect(failureSignature(failure(['npm', 'run', 'lint']))).toBe('verification:npm run lint');
expect(
failureSignature({ ...failure(), kind: 'integration_conflict', argv: undefined, outputTail: 'b.ts\na.ts' }),
).toBe('integration_conflict:a.ts b.ts');
});
it('waits for a person where the project does not activate skills itself, and is not shown before', async () => {
const { project, run, finish, all, store } = setup();
await project('p1', { skillAutoActivation: false });
await finish(await run('r1', 'p1'), 'succeeded', { repairCount: 1, lastFailure: failure() });
expect((await all('p1'))[0]).toMatchObject({ status: 'proposed', category: 'pitfall' });
expect(await store.read(TENANT, tx => projectSkills(tx, 'p1'))).toEqual([]);
// A run that needed no repair teaches nothing.
await finish(await run('r2', 'p1'), 'succeeded');
expect(await all('p1')).toHaveLength(1);
});
});
describe('harvested skills', () => {
it('merges a proposal into the skill it updates, and activates one two succeeded runs proposed', async () => {
const { project, run, skills, answers, asked, all } = setup();
await project('p1');
await run('r1', 'p1', { status: 'succeeded' });
await run('r2', 'p1', { status: 'succeeded' });
const written = await skills.create(owner, 'p1', {
name: 'house-style',
description: 'Any change',
content: 'Use tabs.',
category: 'convention',
});
answers.push({
skills: [
{ name: 'add-page', description: 'When adding a page', content: 'Register it in routes.ts.' },
{ name: 'house-style', updates: 'house-style', description: 'Any change', content: 'Use spaces.' },
],
});
expect((await skills.harvest(TENANT, 'r1')).map(skill => skill.name).sort()).toEqual(['add-page', 'house-style']);
expect(JSON.parse(asked[0]!).existingSkills).toEqual([
expect.objectContaining({ name: 'house-style', category: 'convention', status: 'active' }),
]);
let listed = await all('p1');
expect(listed.find(skill => skill.name === 'add-page')).toMatchObject({ status: 'proposed', sourceRunIds: ['r1'] });
// A person's skill keeps its words.
expect(listed.find(skill => skill.id === written.id)).toMatchObject({ content: 'Use tabs.', sourceRunIds: ['r1'] });
// A second succeeded run proposes the same skill again (by name): it is rewritten and activated.
answers.push({
skills: [
{ name: 'add-page', description: 'When adding a page', content: 'Register it in routes.ts; add a test.' },
],
});
await skills.harvest(TENANT, 'r2');
listed = await all('p1');
expect(listed.filter(skill => skill.name.startsWith('add-page'))).toEqual([
expect.objectContaining({
status: 'active',
content: 'Register it in routes.ts; add a test.',
sourceRunIds: ['r1', 'r2'],
}),
]);
});
it('leaves corroborated proposals for a person where the project says so', async () => {
const { project, run, skills, answers, all } = setup();
await project('p1', { skillAutoActivation: false });
for (const id of ['r1', 'r2']) {
await run(id, 'p1', { status: 'succeeded' });
answers.push({ skills: [{ name: 'add-page', description: 'When adding a page', content: `Do it (${id}).` }] });
await skills.harvest(TENANT, id);
}
expect(await all('p1')).toEqual([
expect.objectContaining({ status: 'proposed', content: 'Do it (r2).', sourceRunIds: ['r1', 'r2'] }),
]);
});
});
describe('how well skills work', () => {
it('counts the runs given a skill and retires a platform skill whose runs keep failing', async () => {
const { project, run, finish, use, all, store, skills } = setup();
await project('p1');
await finish(await run('r0', 'p1'), 'succeeded', { repairCount: 1, lastFailure: failure() });
const mine = await skills.create(owner, 'p1', { name: 'mine', description: 'Always', content: 'Be careful.' });
const good = await run('good', 'p1');
await use(good);
await use(good); // a repair round does not count twice
await finish(good, 'succeeded');
let [pitfall, written] = [
(await all('p1')).find(skill => skill.category === 'pitfall')!,
(await all('p1')).find(skill => skill.id === mine.id)!,
];
expect(pitfall).toMatchObject({ useCount: 1, successCount: 1, failureStreak: 0 });
expect(written).toMatchObject({ useCount: 1, successCount: 1 });
// Canceled without a failed check: neither success nor failure.
const changedMind = await run('cancel', 'p1');
await use(changedMind);
await store.transaction(TENANT, async tx => {
await tx.put('run', { ...changedMind, status: 'canceled' });
await settleSkillUsage(tx, { ...changedMind, status: 'canceled' });
});
for (let n = 1; n <= RETIRE_AFTER_FAILURES; n++) {
const failing = await run(`fail-${n}`, 'p1', { repairCount: 2 });
await use(failing);
await store.transaction(TENANT, tx => tx.put('run', { ...failing, status: 'canceled' }));
}
// Failures are found by the background maintenance.
await skills.maintain(TENANT);
[pitfall, written] = [
(await all('p1')).find(skill => skill.category === 'pitfall')!,
(await all('p1')).find(skill => skill.id === mine.id)!,
];
expect(pitfall).toMatchObject({
status: 'disabled',
retiredReason: 'failing',
useCount: 2 + RETIRE_AFTER_FAILURES,
failureCount: RETIRE_AFTER_FAILURES,
});
// A person's skill is only flagged by its counts; theirs to decide.
expect(written).toMatchObject({ status: 'active', failureStreak: RETIRE_AFTER_FAILURES });
const retiredOn = await store.read(TENANT, tx => tx.events(`fail-${RETIRE_AFTER_FAILURES}`));
expect(retiredOn.map(event => event.type)).toEqual(['skill.retired']);
// Enabling it again starts afresh.
const again = await skills.update(owner, pitfall.id, { status: 'active' });
expect(again).toMatchObject({ status: 'active', failureStreak: 0 });
expect(again.retiredReason).toBeUndefined();
expect(again.updatedBy).toBeUndefined();
});
it('retires a platform skill no run was given for a long time while its project kept running', async () => {
const { project, run, finish, all, skills, store } = setup();
await project('p1');
await finish(await run('r0', 'p1'), 'succeeded', { repairCount: 1, lastFailure: failure() });
// Long ago it was used; today's run has only newer skills in its prompt.
const old = new Date(Date.now() - (RETIRE_UNUSED_DAYS + 1) * 86_400_000).toISOString();
await store.transaction(TENANT, async tx => {
const [skill] = await tx.list<Skill>('skill', { projectId: 'p1' });
await tx.put('skill', { ...skill!, lastUsedAt: old, updatedAt: old });
await tx.put('skill', {
...stamp('fresh'),
projectId: 'p1',
name: 'fresh',
description: 'd',
content: 'c',
status: 'active',
sourceRunIds: [],
createdBy: 'owner',
} as Skill);
});
// A project that did not run keeps its skills.
expect(await skills.maintain(TENANT)).toEqual([]);
const today = await run('today', 'p1');
await store.transaction(TENANT, tx =>
tx.put('skill_usage', { ...stamp('today'), runId: today.id, projectId: 'p1', skillIds: ['fresh'] }),
);
const retired = await skills.maintain(TENANT);
expect(retired.map(skill => skill.category)).toEqual(['pitfall']);
expect((await all('p1')).find(skill => skill.category === 'pitfall')).toMatchObject({
status: 'disabled',
retiredReason: 'stale',
});
});
});
/** A zip whose entries are deflated, as zip tools write them. */
function deflatedZip(files: { path: string; content: string }[]) {
const locals: Buffer[] = [];
const centrals: Buffer[] = [];
let offset = 0;
for (const file of files) {
const name = Buffer.from(file.path);
const data = Buffer.from(file.content);
const packed = deflateRawSync(data);
const local = Buffer.alloc(30);
local.writeUInt32LE(0x04034b50, 0);
local.writeUInt16LE(20, 4);
local.writeUInt16LE(8, 8);
local.writeUInt32LE(crc32(data), 14);
local.writeUInt32LE(packed.length, 18);
local.writeUInt32LE(data.length, 22);
local.writeUInt16LE(name.length, 26);
const central = Buffer.alloc(46);
central.writeUInt32LE(0x02014b50, 0);
central.writeUInt16LE(20, 4);
central.writeUInt16LE(20, 6);
central.writeUInt16LE(8, 10);
central.writeUInt32LE(crc32(data), 16);
central.writeUInt32LE(packed.length, 20);
central.writeUInt32LE(data.length, 24);
central.writeUInt16LE(name.length, 28);
central.writeUInt32LE(offset, 42);
locals.push(local, name, packed);
centrals.push(central, name);
offset += 30 + name.length + packed.length;
}
const directory = Buffer.concat(centrals);
const end = Buffer.alloc(22);
end.writeUInt32LE(0x06054b50, 0);
end.writeUInt16LE(files.length, 8);
end.writeUInt16LE(files.length, 10);
end.writeUInt32LE(directory.length, 12);
end.writeUInt32LE(offset, 16);
return Buffer.concat([...locals, directory, end]);
}
describe('skills as files', () => {
it('reads back what export writes, and SKILL.md written elsewhere', () => {
const skill = {
name: 'pitfall-npm-test',
description: 'Before "npm test"',
content: '- It fails.',
category: 'pitfall' as const,
};
expect(parseSkillMarkdown(skillMarkdown(skill), 'x/SKILL.md')).toEqual({ ...skill, source: 'x/SKILL.md' });
expect(
parseSkillMarkdown(
"---\r\nname: 'Lazy Load'\r\ndescription: >\r\n When a page\r\n is slow\r\nallowed-tools: Read\r\n---\r\n# Steps\r\nDo it.\r\n",
'lazy/SKILL.md',
),
).toMatchObject({
name: 'lazy-load',
description: 'When a page is slow',
content: '# Steps\nDo it.',
category: 'workflow',
});
// No front matter: the name comes from the file, the description from the first line.
expect(parseSkillMarkdown('# Run the e2e suite first\n\nnpm run e2e', 'e2e-first.SKILL.md')).toMatchObject({
name: 'e2e-first',
description: 'Run the e2e suite first',
});
expect(() => parseSkillMarkdown('---\nname: x\n', 'a.md')).toThrow(/not closed/);
expect(() => parseSkillMarkdown('---\nname: x\ndescription: y\n---\n', 'a.md')).toThrow(/no instructions/);
expect(() => parseSkillMarkdown('x'.repeat(70 * 1024), 'big.md')).toThrow(/larger than/);
});
it('reads stored and deflated zips, skipping what is not a skill and refusing what is too large', () => {
const files = [
{ path: 'a/SKILL.md', content: '---\nname: a\ndescription: A\n---\n\nDo A.' },
{ path: 'a/scripts/run.sh', content: 'echo' },
{ path: '__MACOSX/a/._SKILL.md', content: 'junk' },
];
for (const archive of [zipFiles(files), deflatedZip(files)])
expect(readSkillArchive(archive)).toEqual({ files: [files[0]], ignored: 2 });
expect(() => readSkillArchive(Buffer.from('not a zip'))).toThrow(/not a zip/);
expect(() => readSkillArchive(Buffer.alloc(600 * 1024))).toThrow(/larger than/);
const bomb = deflatedZip([{ path: 'b/SKILL.md', content: 'x'.repeat(100 * 1024) }]);
expect(() => readSkillArchive(bomb)).toThrow(/larger than 64 KiB/);
const damaged = zipFiles([files[0]!]);
damaged[40] ^= 0xff; // a byte of the file's content
expect(() => readSkillArchive(damaged)).toThrow(/damaged/);
});
});
describe('moving skills between projects', () => {
it('imports an export into another project once, numbering names a different skill holds', async () => {
const { project, skills, all } = setup();
await project('p1');
await project('p2');
await skills.create(owner, 'p1', { name: 'add-page', description: 'Pages', content: 'Register it.' });
await skills.create(owner, 'p1', {
name: 'house-style',
description: 'Always',
content: 'Tabs.',
category: 'convention',
});
await skills.create(owner, 'p2', { name: 'add-page', description: 'Pages', content: 'Something else.' });
const exported = await skills.exportAll(owner, 'p1');
const archive = exported.content.toString('base64');
const result = await skills.importFiles(owner, 'p2', { archive }, 'import-key-0001');
expect(result.created.map(skill => [skill.name, skill.category, skill.status])).toEqual([
['add-page-2', 'workflow', 'active'],
['house-style', 'convention', 'active'],
]);
// The same request again answers the same; the same files again add nothing.
expect(await skills.importFiles(owner, 'p2', { archive }, 'import-key-0001')).toEqual(result);
const again = await skills.importFiles(owner, 'p2', { archive, status: 'proposed' }, 'import-key-0002');
expect(again.created).toEqual([]);
expect(again.skipped.map(item => item.reason)).toEqual(['identical', 'identical']);
expect(await all('p2')).toHaveLength(3);
// Loose files; one unreadable file is reported, the rest imported.
const loose = await skills.importFiles(
owner,
'p2',
{
files: [
{ path: 'review.SKILL.md', content: '---\ndescription: Before review\n---\nCheck it.' },
{ path: 'broken.md', content: '---\nname: x' },
],
status: 'proposed',
},
'import-key-0003',
);
expect(loose.created).toEqual([expect.objectContaining({ name: 'review', status: 'proposed' })]);
expect(loose.skipped).toEqual([expect.objectContaining({ source: 'broken.md', reason: 'invalid' })]);
await expect(
skills.importFiles(owner, 'p2', { files: [{ path: 'broken.md', content: '---\nname: x' }] }, 'import-key-0004'),
).rejects.toMatchObject({ code: 'NO_SKILLS' });
await expect(skills.importFiles(viewer, 'p2', { archive }, 'import-key-0005')).rejects.toMatchObject({
code: 'FORBIDDEN',
});
await expect(skills.importFiles(owner, 'p2', { archive })).rejects.toMatchObject({ code: 'IDEMPOTENCY_REQUIRED' });
});
it('copies skills to a project the person may shape, keeping their kind and state but not their history', async () => {
const { project, run, finish, skills, all } = setup();
await project('p1');
await project('p2');
await project('hidden', { memberIds: [] });
await finish(await run('r1', 'p1'), 'succeeded', { repairCount: 1, lastFailure: failure() });
const [pitfall] = await all('p1');
const copied = await skills.copy(owner, { skillIds: [pitfall!.id], projectId: 'p2' }, 'copy-key-0001');
expect(copied.created).toEqual([
expect.objectContaining({
projectId: 'p2',
name: 'pitfall-npm-test',
category: 'pitfall',
signature: 'verification:npm test',
status: 'active',
sourceRunIds: [],
createdBy: 'owner',
}),
]);
expect((await skills.copy(owner, { skillIds: [pitfall!.id], projectId: 'p2' }, 'copy-key-0002')).skipped).toEqual([
expect.objectContaining({ reason: 'identical' }),
]);
// The copy keeps learning in its new project: the same failure there updates it.
await finish(await run('r2', 'p2'), 'succeeded', { repairCount: 1, lastFailure: failure() });
expect(await all('p2')).toEqual([expect.objectContaining({ id: copied.created[0]!.id, sourceRunIds: ['r2'] })]);
// Someone who may not wish for the target may not copy into it, nor see a project they are not in.
await expect(
skills.copy(viewer, { skillIds: [pitfall!.id], projectId: 'p2' }, 'copy-key-0003'),
).rejects.toMatchObject({
code: 'FORBIDDEN',
});
await expect(
skills.copy(viewer, { skillIds: [pitfall!.id], projectId: 'hidden' }, 'copy-key-0004'),
).rejects.toMatchObject({
code: 'NOT_FOUND',
});
});
it('lists the skills of every project a person may see, filtered', async () => {
const { project, skills } = setup();
await project('p1');
await project('hidden', { memberIds: [] });
await skills.create(owner, 'p1', { name: 'one', description: 'Pages', content: 'Register routes.' });
await skills.create(owner, 'p1', { name: 'two', description: 'Style', content: 'Tabs.', category: 'convention' });
await skills.create(owner, 'hidden', { name: 'secret', description: 'Hidden', content: 'x' });
expect((await skills.search(owner, {})).map(skill => skill.name).sort()).toEqual(['one', 'secret', 'two']);
expect((await skills.search(viewer, {})).map(skill => skill.name).sort()).toEqual(['one', 'two']);
expect((await skills.search(viewer, { category: 'convention' })).map(skill => skill.name)).toEqual(['two']);
expect((await skills.search(viewer, { q: 'ROUTES' })).map(skill => skill.name)).toEqual(['one']);
expect(await skills.search(viewer, { status: 'proposed' })).toEqual([]);
await expect(skills.search(viewer, { projectId: 'hidden' })).rejects.toMatchObject({ code: 'NOT_FOUND' });
});
});