diff --git a/README.md b/README.md index bed4661..bb7fd86 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ EmboFlow is a B/S embodied-data workflow platform for raw asset ingestion, delivery normalization, dataset transformation, workflow execution, preview, and export. +## Current V1 Features + +- Project-scoped workspace shell with a dedicated Projects page and active project selector in the header +- Asset workspace that supports local asset registration, probe summaries, storage connection management, and dataset creation +- Workflow templates as first-class objects, including default project templates and creating project workflows from a template +- Blank workflow creation and a large React Flow editor with drag-and-drop nodes, free canvas movement, edge validation, node runtime presets, and Python code-hook injection +- Workflow-level `Save As Template` so edited graphs can be promoted into reusable project templates +- Mongo-backed run orchestration, worker execution, run history, task detail, logs, stdout/stderr, artifacts, cancel, retry, and task retry +- Runtime shell level Chinese and English switching + ## Bootstrap From the repository root: @@ -68,7 +78,11 @@ You can register that directory from the Assets page or via `POST /api/assets/re The workflow editor currently requires selecting at least one registered asset before a run can be created. The editor now also persists per-node runtime config in workflow versions, including executor overrides, optional artifact title overrides, and Python code-hook source for inspect and transform style nodes. The runtime web shell now exposes a visible `中文 / English` language toggle. The core workspace shell and workflow authoring surface are translated through a lightweight i18n layer. +The shell now also exposes a dedicated Projects page plus an active project selector, so assets, datasets, workflow templates, workflows, and runs all switch together at the project boundary. +The Assets workspace now includes first-class storage connections and datasets. A dataset is distinct from a raw asset and binds project source assets to a selected local or object-storage-backed destination. +The Workflows workspace now includes a template gallery. Projects can start from default or saved templates, or create a blank workflow directly. The workflow editor center panel now uses a real draggable node canvas with zoom, pan, mini-map, dotted background, handle-based edge creation, persisted node positions, and localized validation feedback instead of a static list of node cards. +The workflow editor right panel now also supports saving the current workflow draft as a reusable workflow template, in addition to editing per-node runtime settings and Python hooks. The node library now supports both click-to-append and drag-and-drop placement into the canvas. V1 connection rules block self-edges, duplicate edges, cycles, incoming edges into source nodes, outgoing edges from export nodes, and multiple upstream edges into a single node. The Runs workspace now shows project-scoped run history, run-level aggregated summaries, cancel/retry controls, and run detail views with persisted task summaries, stdout/stderr sections, result previews, and artifact links into Explore. Selected run tasks now expose the frozen node definition id, executor config snapshot, and code-hook metadata that were captured when the run was created. diff --git a/apps/api/src/runtime/mongo-store.ts b/apps/api/src/runtime/mongo-store.ts index 89503f3..b1f85bc 100644 --- a/apps/api/src/runtime/mongo-store.ts +++ b/apps/api/src/runtime/mongo-store.ts @@ -68,6 +68,62 @@ type AssetProbeReportDocument = { createdAt: string; }; +type StorageProvider = "local" | "minio" | "s3" | "bos" | "oss"; + +type StorageConnectionDocument = Timestamped & { + _id: string; + workspaceId: string; + name: string; + slug: string; + provider: StorageProvider; + bucket?: string; + endpoint?: string; + region?: string; + basePath?: string; + rootPath?: string; + status: "active"; + createdBy: string; +}; + +type DatasetDocument = Timestamped & { + _id: string; + workspaceId: string; + projectId: string; + name: string; + slug: string; + description: string; + status: "draft" | "active"; + sourceAssetIds: string[]; + storageConnectionId: string; + storagePath: string; + latestVersionId: string; + latestVersionNumber: number; + createdBy: string; + summary: Record; +}; + +type DatasetVersionDocument = { + _id: string; + datasetId: string; + workspaceId: string; + projectId: string; + versionNumber: number; + sourceAssetIds: string[]; + storageSnapshot: { + storageConnectionId: string; + provider: StorageProvider; + bucket?: string; + endpoint?: string; + region?: string; + basePath?: string; + rootPath?: string; + storagePath: string; + }; + summary: Record; + createdBy: string; + createdAt: string; +}; + type WorkflowDefinitionDocument = Timestamped & { _id: string; workspaceId: string; @@ -149,6 +205,24 @@ type ArtifactDocument = Timestamped & { payload: Record; }; +type WorkflowTemplateDocument = Timestamped & { + _id: string; + workspaceId: string; + projectId?: string; + name: string; + slug: string; + description: string; + status: "active"; + visualGraph: Record; + logicGraph: { + nodes: Array<{ id: string; type: string }>; + edges: Array<{ from: string; to: string }>; + }; + runtimeGraph: Record; + pluginRefs: string[]; + createdBy: string; +}; + type WorkflowRuntimeGraph = Record & { selectedPreset?: string; nodeBindings?: Record; @@ -320,9 +394,105 @@ function collectRetryNodeIds(tasks: RunTaskDocument[], rootNodeId: string) { return collected; } +function createStorageSnapshot( + connection: StorageConnectionDocument, + storagePath: string, +): DatasetVersionDocument["storageSnapshot"] { + return { + storageConnectionId: connection._id, + provider: connection.provider, + bucket: connection.bucket, + endpoint: connection.endpoint, + region: connection.region, + basePath: connection.basePath, + rootPath: connection.rootPath, + storagePath, + }; +} + export class MongoAppStore { constructor(private readonly db: Db) {} + private async ensureDefaultStorageConnection(workspaceId: string, createdBy: string) { + const collection = this.db.collection("storage_connections"); + const existing = await collection.findOne({ + workspaceId, + slug: "local-workspace-storage", + }); + if (existing) { + return existing; + } + + const connection: StorageConnectionDocument = { + _id: `storage-${randomUUID()}`, + workspaceId, + name: "Local Workspace Storage", + slug: "local-workspace-storage", + provider: "local", + rootPath: "/Users/longtaowu/workspace/emboldata", + status: "active", + createdBy, + createdAt: nowIso(), + updatedAt: nowIso(), + }; + await collection.insertOne(connection); + return connection; + } + + private async ensureDefaultWorkflowTemplate( + workspaceId: string, + projectId: string, + createdBy: string, + ) { + const collection = this.db.collection("workflow_templates"); + const existing = await collection.findOne({ + workspaceId, + projectId, + slug: "delivery-normalization-template", + }); + if (existing) { + return existing; + } + + const template: WorkflowTemplateDocument = { + _id: `template-${randomUUID()}`, + workspaceId, + projectId, + name: "Delivery Normalization Template", + slug: "delivery-normalization-template", + description: "Default delivery normalization pipeline with source, validation, and export steps.", + status: "active", + visualGraph: { + viewport: { x: 0, y: 0, zoom: 1 }, + nodePositions: { + "source-asset": { x: 120, y: 120 }, + "validate-structure": { x: 440, y: 240 }, + "export-delivery-package": { x: 780, y: 360 }, + }, + }, + logicGraph: { + nodes: [ + { id: "source-asset", type: "source" }, + { id: "validate-structure", type: "inspect" }, + { id: "export-delivery-package", type: "export" }, + ], + edges: [ + { from: "source-asset", to: "validate-structure" }, + { from: "validate-structure", to: "export-delivery-package" }, + ], + }, + runtimeGraph: { + selectedPreset: "delivery-normalization", + }, + pluginRefs: ["builtin:delivery-nodes"], + createdBy, + createdAt: nowIso(), + updatedAt: nowIso(), + }; + await collection.insertOne(template); + return template; + } + async bootstrapDevContext(input: { userId?: string; workspaceName?: string; @@ -387,6 +557,9 @@ export class MongoAppStore { throw new Error("failed to bootstrap project"); } + await this.ensureDefaultStorageConnection(workspace._id, userId); + await this.ensureDefaultWorkflowTemplate(workspace._id, project._id, userId); + return { userId, workspace: mapDoc(workspace as WithId), @@ -420,6 +593,7 @@ export class MongoAppStore { updatedAt: nowIso(), }; await this.db.collection("projects").insertOne(project); + await this.ensureDefaultWorkflowTemplate(project.workspaceId, project._id, input.createdBy); return project; } @@ -431,6 +605,51 @@ export class MongoAppStore { .toArray(); } + async createStorageConnection(input: { + workspaceId: string; + name: string; + provider: StorageProvider; + bucket?: string; + endpoint?: string; + region?: string; + basePath?: string; + rootPath?: string; + createdBy: string; + }) { + const connection: StorageConnectionDocument = { + _id: `storage-${randomUUID()}`, + workspaceId: input.workspaceId, + name: input.name, + slug: slugify(input.name), + provider: input.provider, + bucket: input.bucket, + endpoint: input.endpoint, + region: input.region, + basePath: input.basePath, + rootPath: input.rootPath, + status: "active", + createdBy: input.createdBy, + createdAt: nowIso(), + updatedAt: nowIso(), + }; + await this.db.collection("storage_connections").insertOne(connection); + return connection; + } + + async listStorageConnections(workspaceId: string) { + return this.db + .collection("storage_connections") + .find({ workspaceId, status: "active" }) + .sort({ createdAt: 1 }) + .toArray(); + } + + async getStorageConnection(storageConnectionId: string) { + return this.db + .collection("storage_connections") + .findOne({ _id: storageConnectionId }); + } + async registerAsset(input: { workspaceId: string; projectId: string; @@ -532,6 +751,158 @@ export class MongoAppStore { .next(); } + async createDataset(input: { + workspaceId: string; + projectId: string; + name: string; + description?: string; + sourceAssetIds: string[]; + storageConnectionId: string; + storagePath: string; + createdBy: string; + }) { + const assetIds = Array.from(new Set((input.sourceAssetIds ?? []).filter(Boolean))); + if (assetIds.length === 0) { + throw new Error("sourceAssetIds must include at least one asset"); + } + + const assets = await this.db + .collection("assets") + .find({ _id: { $in: assetIds } }) + .toArray(); + if (assets.length !== assetIds.length) { + throw new Error("one or more source assets do not exist"); + } + if (assets.some((asset) => asset.projectId !== input.projectId)) { + throw new Error("source assets must belong to the dataset project"); + } + + const storageConnection = await this.getStorageConnection(input.storageConnectionId); + if (!storageConnection) { + throw new Error(`storage connection not found: ${input.storageConnectionId}`); + } + if (storageConnection.workspaceId !== input.workspaceId) { + throw new Error("storage connection must belong to the dataset workspace"); + } + + const createdAt = nowIso(); + const datasetId = `dataset-${randomUUID()}`; + const versionId = `${datasetId}-v1`; + const summary = { + sourceAssetCount: assetIds.length, + storageProvider: storageConnection.provider, + storagePath: input.storagePath, + }; + + const dataset: DatasetDocument = { + _id: datasetId, + workspaceId: input.workspaceId, + projectId: input.projectId, + name: input.name, + slug: slugify(input.name), + description: input.description ?? "", + status: "active", + sourceAssetIds: assetIds, + storageConnectionId: storageConnection._id, + storagePath: input.storagePath, + latestVersionId: versionId, + latestVersionNumber: 1, + createdBy: input.createdBy, + createdAt, + updatedAt: createdAt, + summary, + }; + const version: DatasetVersionDocument = { + _id: versionId, + datasetId, + workspaceId: input.workspaceId, + projectId: input.projectId, + versionNumber: 1, + sourceAssetIds: assetIds, + storageSnapshot: createStorageSnapshot(storageConnection, input.storagePath), + summary, + createdBy: input.createdBy, + createdAt, + }; + + await this.db.collection("datasets").insertOne(dataset); + await this.db.collection("dataset_versions").insertOne(version); + return dataset; + } + + async listDatasets(projectId: string) { + return this.db + .collection("datasets") + .find({ projectId }) + .sort({ createdAt: -1 }) + .toArray(); + } + + async getDataset(datasetId: string) { + return this.db.collection("datasets").findOne({ _id: datasetId }); + } + + async listDatasetVersions(datasetId: string) { + return this.db + .collection("dataset_versions") + .find({ datasetId }) + .sort({ versionNumber: -1 }) + .toArray(); + } + + async createWorkflowTemplate(input: { + workspaceId: string; + projectId?: string; + name: string; + description?: string; + visualGraph: Record; + logicGraph: WorkflowTemplateDocument["logicGraph"]; + runtimeGraph: Record; + pluginRefs: string[]; + createdBy: string; + }) { + const template: WorkflowTemplateDocument = { + _id: `template-${randomUUID()}`, + workspaceId: input.workspaceId, + projectId: input.projectId, + name: input.name, + slug: slugify(input.name), + description: input.description ?? "", + status: "active", + visualGraph: input.visualGraph, + logicGraph: input.logicGraph, + runtimeGraph: input.runtimeGraph, + pluginRefs: input.pluginRefs, + createdBy: input.createdBy, + createdAt: nowIso(), + updatedAt: nowIso(), + }; + await this.db.collection("workflow_templates").insertOne(template); + return template; + } + + async listWorkflowTemplates(input: { workspaceId: string; projectId?: string }) { + const filter: Record = { + workspaceId: input.workspaceId, + status: "active", + }; + if (input.projectId) { + filter.$or = [{ projectId: input.projectId }, { projectId: { $exists: false } }]; + } + + return this.db + .collection("workflow_templates") + .find(filter) + .sort({ createdAt: -1 }) + .toArray(); + } + + async getWorkflowTemplate(templateId: string) { + return this.db + .collection("workflow_templates") + .findOne({ _id: templateId }); + } + async createWorkflowDefinition(input: { workspaceId: string; projectId: string; @@ -555,6 +926,43 @@ export class MongoAppStore { return definition; } + async createWorkflowFromTemplate(input: { + templateId: string; + workspaceId: string; + projectId: string; + name: string; + createdBy: string; + }) { + const template = await this.getWorkflowTemplate(input.templateId); + if (!template) { + throw new Error(`workflow template not found: ${input.templateId}`); + } + if (template.workspaceId !== input.workspaceId) { + throw new Error("workflow template must belong to the target workspace"); + } + if (template.projectId && template.projectId !== input.projectId) { + throw new Error("workflow template must belong to the target project or be workspace-scoped"); + } + + const definition = await this.createWorkflowDefinition({ + workspaceId: input.workspaceId, + projectId: input.projectId, + name: input.name, + createdBy: input.createdBy, + }); + + await this.saveWorkflowVersion({ + workflowDefinitionId: definition._id, + visualGraph: template.visualGraph, + logicGraph: template.logicGraph, + runtimeGraph: template.runtimeGraph, + pluginRefs: template.pluginRefs, + createdBy: input.createdBy, + }); + + return this.getWorkflowDefinition(definition._id); + } + async listWorkflowDefinitions(projectId: string) { return this.db .collection("workflow_definitions") diff --git a/apps/api/src/runtime/server.ts b/apps/api/src/runtime/server.ts index 1b2e123..4470d15 100644 --- a/apps/api/src/runtime/server.ts +++ b/apps/api/src/runtime/server.ts @@ -94,6 +94,34 @@ export async function createApiRuntime(config = resolveApiRuntimeConfig()) { } }); + app.post("/api/storage-connections", async (request, response, next) => { + try { + response.json( + await store.createStorageConnection({ + workspaceId: request.body.workspaceId, + name: request.body.name, + provider: request.body.provider, + bucket: request.body.bucket, + endpoint: request.body.endpoint, + region: request.body.region, + basePath: request.body.basePath, + rootPath: request.body.rootPath, + createdBy: request.body.createdBy ?? "local-user", + }), + ); + } catch (error) { + next(error); + } + }); + + app.get("/api/storage-connections", async (request, response, next) => { + try { + response.json(await store.listStorageConnections(String(request.query.workspaceId))); + } catch (error) { + next(error); + } + }); + app.post("/api/assets/register", async (request, response, next) => { try { const sourcePath = request.body.sourcePath as string | undefined; @@ -158,10 +186,120 @@ export async function createApiRuntime(config = resolveApiRuntimeConfig()) { } }); + app.post("/api/datasets", async (request, response, next) => { + try { + response.json( + await store.createDataset({ + workspaceId: request.body.workspaceId, + projectId: request.body.projectId, + name: request.body.name, + description: request.body.description, + sourceAssetIds: request.body.sourceAssetIds ?? [], + storageConnectionId: request.body.storageConnectionId, + storagePath: request.body.storagePath, + createdBy: request.body.createdBy ?? "local-user", + }), + ); + } catch (error) { + next(error); + } + }); + + app.get("/api/datasets", async (request, response, next) => { + try { + response.json(await store.listDatasets(String(request.query.projectId))); + } catch (error) { + next(error); + } + }); + + app.get("/api/datasets/:datasetId", async (request, response, next) => { + try { + const dataset = await store.getDataset(request.params.datasetId); + if (!dataset) { + response.status(404).json({ message: "dataset not found" }); + return; + } + response.json(dataset); + } catch (error) { + next(error); + } + }); + + app.get("/api/datasets/:datasetId/versions", async (request, response, next) => { + try { + response.json(await store.listDatasetVersions(request.params.datasetId)); + } catch (error) { + next(error); + } + }); + app.get("/api/node-definitions", (_request, response) => { response.json(store.listNodeDefinitions()); }); + app.post("/api/workflow-templates", async (request, response, next) => { + try { + response.json( + await store.createWorkflowTemplate({ + workspaceId: request.body.workspaceId, + projectId: request.body.projectId, + name: request.body.name, + description: request.body.description, + visualGraph: request.body.visualGraph ?? {}, + logicGraph: request.body.logicGraph, + runtimeGraph: request.body.runtimeGraph ?? {}, + pluginRefs: request.body.pluginRefs ?? [], + createdBy: request.body.createdBy ?? "local-user", + }), + ); + } catch (error) { + next(error); + } + }); + + app.get("/api/workflow-templates", async (request, response, next) => { + try { + response.json( + await store.listWorkflowTemplates({ + workspaceId: String(request.query.workspaceId), + projectId: request.query.projectId ? String(request.query.projectId) : undefined, + }), + ); + } catch (error) { + next(error); + } + }); + + app.get("/api/workflow-templates/:templateId", async (request, response, next) => { + try { + const template = await store.getWorkflowTemplate(request.params.templateId); + if (!template) { + response.status(404).json({ message: "workflow template not found" }); + return; + } + response.json(template); + } catch (error) { + next(error); + } + }); + + app.post("/api/workflow-templates/:templateId/workflows", async (request, response, next) => { + try { + response.json( + await store.createWorkflowFromTemplate({ + templateId: request.params.templateId, + workspaceId: request.body.workspaceId, + projectId: request.body.projectId, + name: request.body.name, + createdBy: request.body.createdBy ?? "local-user", + }), + ); + } catch (error) { + next(error); + } + }); + app.post("/api/workflows", async (request, response, next) => { try { response.json( diff --git a/apps/api/test/runtime-http.integration.spec.ts b/apps/api/test/runtime-http.integration.spec.ts index 1e6180b..b202cb3 100644 --- a/apps/api/test/runtime-http.integration.spec.ts +++ b/apps/api/test/runtime-http.integration.spec.ts @@ -96,6 +96,67 @@ test("mongo-backed runtime reuses bootstrapped workspace and project across rest assert.equal(projects[0]?._id, bootstrap.project._id); }); +test("mongo-backed runtime provisions a default workflow template for newly created projects", async (t) => { + const mongod = await MongoMemoryServer.create({ + instance: { + ip: "127.0.0.1", + port: 27217, + }, + }); + t.after(async () => { + await mongod.stop(); + }); + + const server = await startRuntimeServer({ + host: "127.0.0.1", + port: 0, + mongoUri: mongod.getUri(), + database: "emboflow-runtime-project-template", + corsOrigin: "http://127.0.0.1:3000", + }); + t.after(async () => { + await server.close(); + }); + + const bootstrap = await readJson<{ + workspace: { _id: string }; + }>( + await fetch(`${server.baseUrl}/api/dev/bootstrap`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ userId: "project-template-user", projectName: "Seed Project" }), + }), + ); + + const project = await readJson<{ _id: string }>( + await fetch(`${server.baseUrl}/api/projects`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: bootstrap.workspace._id, + name: "Second Project", + description: "Project created after bootstrap", + createdBy: "project-template-user", + }), + }), + ); + + const templates = await readJson>( + await fetch( + `${server.baseUrl}/api/workflow-templates?workspaceId=${encodeURIComponent(bootstrap.workspace._id)}&projectId=${encodeURIComponent(project._id)}`, + ), + ); + + assert.equal( + templates.some((template) => template.projectId === project._id), + true, + ); + assert.equal( + templates.some((template) => template.slug === "delivery-normalization-template"), + true, + ); +}); + test("mongo-backed runtime persists probed assets and workflow runs through the HTTP API", async (t) => { const sourceDir = await mkdtemp(path.join(os.tmpdir(), "emboflow-runtime-")); await mkdir(path.join(sourceDir, "DJI_001")); @@ -826,6 +887,191 @@ test("mongo-backed runtime exposes persisted task execution summaries and logs", }); }); +test("mongo-backed runtime supports storage connections, datasets, workflow templates, and workflow creation from templates", async (t) => { + const sourceDir = await mkdtemp(path.join(os.tmpdir(), "emboflow-runtime-datasets-")); + await mkdir(path.join(sourceDir, "DJI_001")); + await writeFile(path.join(sourceDir, "meta.json"), "{}"); + await writeFile(path.join(sourceDir, "intrinsics.json"), "{}"); + await writeFile(path.join(sourceDir, "video_meta.json"), "{}"); + await writeFile(path.join(sourceDir, "DJI_001", "DJI_001.mp4"), ""); + + const mongod = await MongoMemoryServer.create({ + instance: { + ip: "127.0.0.1", + port: 27125, + }, + }); + t.after(async () => { + await mongod.stop(); + }); + + const server = await startRuntimeServer({ + host: "127.0.0.1", + port: 0, + mongoUri: mongod.getUri(), + database: "emboflow-runtime-datasets-templates", + corsOrigin: "http://127.0.0.1:3000", + }); + t.after(async () => { + await server.close(); + }); + + const bootstrap = await readJson<{ + workspace: { _id: string }; + project: { _id: string }; + }>( + await fetch(`${server.baseUrl}/api/dev/bootstrap`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ userId: "dataset-user", projectName: "Dataset Project" }), + }), + ); + + const connections = await readJson>( + await fetch( + `${server.baseUrl}/api/storage-connections?workspaceId=${encodeURIComponent(bootstrap.workspace._id)}`, + ), + ); + + const cloudConnection = await readJson<{ _id: string; provider: string; bucket: string }>( + await fetch(`${server.baseUrl}/api/storage-connections`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: bootstrap.workspace._id, + name: "Project OSS", + provider: "oss", + bucket: "emboflow-datasets", + endpoint: "oss-cn-hangzhou.aliyuncs.com", + basePath: "datasets/project-a", + }), + }), + ); + + const asset = await readJson<{ _id: string; displayName: string }>( + await fetch(`${server.baseUrl}/api/assets/register`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: bootstrap.workspace._id, + projectId: bootstrap.project._id, + sourcePath: sourceDir, + }), + }), + ); + await readJson(await fetch(`${server.baseUrl}/api/assets/${asset._id}/probe`, { method: "POST" })); + + const dataset = await readJson<{ + _id: string; + latestVersionNumber: number; + storageConnectionId: string; + }>( + await fetch(`${server.baseUrl}/api/datasets`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: bootstrap.workspace._id, + projectId: bootstrap.project._id, + name: "Delivery Dataset", + description: "Dataset derived from the probed delivery asset", + sourceAssetIds: [asset._id], + storageConnectionId: cloudConnection._id, + storagePath: "delivery/dataset-v1", + createdBy: "dataset-user", + }), + }), + ); + + const datasets = await readJson>( + await fetch(`${server.baseUrl}/api/datasets?projectId=${encodeURIComponent(bootstrap.project._id)}`), + ); + const datasetVersions = await readJson>( + await fetch(`${server.baseUrl}/api/datasets/${dataset._id}/versions`), + ); + + const template = await readJson<{ _id: string; name: string }>( + await fetch(`${server.baseUrl}/api/workflow-templates`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: bootstrap.workspace._id, + projectId: bootstrap.project._id, + name: "Delivery Review Template", + description: "Template with inspect and export nodes", + visualGraph: { + viewport: { x: 0, y: 0, zoom: 1 }, + nodePositions: { + "source-asset": { x: 120, y: 120 }, + "validate-structure": { x: 460, y: 220 }, + "export-delivery-package": { x: 820, y: 340 }, + }, + }, + logicGraph: { + nodes: [ + { id: "source-asset", type: "source" }, + { id: "validate-structure", type: "inspect" }, + { id: "export-delivery-package", type: "export" }, + ], + edges: [ + { from: "source-asset", to: "validate-structure" }, + { from: "validate-structure", to: "export-delivery-package" }, + ], + }, + runtimeGraph: { + selectedPreset: "delivery-template", + nodeConfigs: { + "validate-structure": { + executorType: "python", + }, + }, + }, + pluginRefs: ["builtin:delivery-nodes"], + createdBy: "dataset-user", + }), + }), + ); + + const templates = await readJson>( + await fetch( + `${server.baseUrl}/api/workflow-templates?workspaceId=${encodeURIComponent(bootstrap.workspace._id)}&projectId=${encodeURIComponent(bootstrap.project._id)}`, + ), + ); + + const workflowFromTemplate = await readJson<{ _id: string; name: string; latestVersionNumber: number }>( + await fetch(`${server.baseUrl}/api/workflow-templates/${template._id}/workflows`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: bootstrap.workspace._id, + projectId: bootstrap.project._id, + name: "Delivery Review Flow", + createdBy: "dataset-user", + }), + }), + ); + + const workflowVersions = await readJson>( + await fetch(`${server.baseUrl}/api/workflows/${workflowFromTemplate._id}/versions`), + ); + + assert.equal(connections[0]?.provider, "local"); + assert.equal(cloudConnection.provider, "oss"); + assert.equal(cloudConnection.bucket, "emboflow-datasets"); + assert.equal(dataset.storageConnectionId, cloudConnection._id); + assert.equal(dataset.latestVersionNumber, 1); + assert.equal(datasets.length, 1); + assert.equal(datasets[0]?._id, dataset._id); + assert.equal(datasetVersions.length, 1); + assert.equal(datasetVersions[0]?.datasetId, dataset._id); + assert.equal(datasetVersions[0]?.versionNumber, 1); + assert.equal(template.name, "Delivery Review Template"); + assert.equal(templates.some((item) => item._id === template._id), true); + assert.equal(workflowFromTemplate.latestVersionNumber, 1); + assert.equal(workflowVersions.length, 1); + assert.equal(workflowVersions[0]?.versionNumber, 1); + assert.equal(workflowVersions[0]?.runtimeGraph?.selectedPreset, "delivery-template"); +}); + test("mongo-backed runtime can cancel a run, retry a run snapshot, and retry a failed task", async (t) => { const mongod = await MongoMemoryServer.create({ instance: { diff --git a/apps/web/src/runtime/api-client.ts b/apps/web/src/runtime/api-client.ts index bc1fc36..01f1b35 100644 --- a/apps/web/src/runtime/api-client.ts +++ b/apps/web/src/runtime/api-client.ts @@ -24,12 +24,61 @@ export class ApiClient { return readJson(response); } + async listProjects(workspaceId: string) { + return readJson( + await fetch(`${this.baseUrl}/api/projects?workspaceId=${encodeURIComponent(workspaceId)}`), + ); + } + + async createProject(input: { + workspaceId: string; + name: string; + description?: string; + createdBy?: string; + }) { + return readJson( + await fetch(`${this.baseUrl}/api/projects`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(input), + }), + ); + } + async listAssets(projectId: string) { return readJson( await fetch(`${this.baseUrl}/api/assets?projectId=${encodeURIComponent(projectId)}`), ); } + async listStorageConnections(workspaceId: string) { + return readJson( + await fetch( + `${this.baseUrl}/api/storage-connections?workspaceId=${encodeURIComponent(workspaceId)}`, + ), + ); + } + + async createStorageConnection(input: { + workspaceId: string; + name: string; + provider: "local" | "minio" | "s3" | "bos" | "oss"; + bucket?: string; + endpoint?: string; + region?: string; + basePath?: string; + rootPath?: string; + createdBy?: string; + }) { + return readJson( + await fetch(`${this.baseUrl}/api/storage-connections`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(input), + }), + ); + } + async registerLocalAsset(input: { workspaceId: string; projectId: string; @@ -61,6 +110,41 @@ export class ApiClient { ); } + async listDatasets(projectId: string) { + return readJson( + await fetch(`${this.baseUrl}/api/datasets?projectId=${encodeURIComponent(projectId)}`), + ); + } + + async createDataset(input: { + workspaceId: string; + projectId: string; + name: string; + description?: string; + sourceAssetIds: string[]; + storageConnectionId: string; + storagePath: string; + createdBy?: string; + }) { + return readJson( + await fetch(`${this.baseUrl}/api/datasets`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(input), + }), + ); + } + + async getDataset(datasetId: string) { + return readJson(await fetch(`${this.baseUrl}/api/datasets/${datasetId}`)); + } + + async listDatasetVersions(datasetId: string) { + return readJson( + await fetch(`${this.baseUrl}/api/datasets/${datasetId}/versions`), + ); + } + async listWorkflows(projectId: string) { return readJson( await fetch(`${this.baseUrl}/api/workflows?projectId=${encodeURIComponent(projectId)}`), @@ -107,6 +191,60 @@ export class ApiClient { return readJson(await fetch(`${this.baseUrl}/api/node-definitions`)); } + async listWorkflowTemplates(input: { + workspaceId: string; + projectId?: string; + }) { + const search = new URLSearchParams({ workspaceId: input.workspaceId }); + if (input.projectId) { + search.set("projectId", input.projectId); + } + return readJson( + await fetch(`${this.baseUrl}/api/workflow-templates?${search.toString()}`), + ); + } + + async createWorkflowTemplate(input: { + workspaceId: string; + projectId?: string; + name: string; + description?: string; + visualGraph: Record; + logicGraph: Record; + runtimeGraph: Record; + pluginRefs: string[]; + createdBy?: string; + }) { + return readJson( + await fetch(`${this.baseUrl}/api/workflow-templates`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(input), + }), + ); + } + + async createWorkflowFromTemplate(input: { + templateId: string; + workspaceId: string; + projectId: string; + name: string; + createdBy?: string; + }) { + return readJson( + await fetch(`${this.baseUrl}/api/workflow-templates/${input.templateId}/workflows`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + workspaceId: input.workspaceId, + projectId: input.projectId, + name: input.name, + createdBy: input.createdBy, + }), + }), + ); + } + async createRun(input: { workflowDefinitionId: string; workflowVersionId: string; diff --git a/apps/web/src/runtime/app.tsx b/apps/web/src/runtime/app.tsx index b6baa0a..8307cd0 100644 --- a/apps/web/src/runtime/app.tsx +++ b/apps/web/src/runtime/app.tsx @@ -39,6 +39,38 @@ import { } from "./workflow-editor-state.ts"; const NODE_LIBRARY_MIME = "application/x-emboflow-node-definition"; +const ACTIVE_PROJECT_STORAGE_KEY_PREFIX = "emboflow.activeProject"; + +function navigateTo(pathname: string) { + if (typeof window === "undefined") { + return; + } + if (window.location.pathname === pathname) { + return; + } + window.history.pushState({}, "", pathname); + window.dispatchEvent(new PopStateEvent("popstate")); +} + +function getActiveProjectStorageKey(workspaceId: string) { + return `${ACTIVE_PROJECT_STORAGE_KEY_PREFIX}:${workspaceId}`; +} + +function normalizePathnameForProjectSwitch(pathname: string) { + if (pathname.startsWith("/assets/")) { + return "/assets"; + } + if (pathname.startsWith("/workflows/")) { + return "/workflows"; + } + if (pathname.startsWith("/runs/")) { + return "/runs"; + } + if (pathname.startsWith("/explore/")) { + return "/explore"; + } + return pathname === "/" ? "/projects" : pathname; +} function mapConnectionValidationReasonToKey( reason: WorkflowConnectionValidationReason | "missing_connection_endpoint", @@ -63,7 +95,7 @@ function mapConnectionValidationReasonToKey( } } -type NavItem = "Assets" | "Workflows" | "Runs" | "Explore" | "Labels" | "Admin"; +type NavItem = "Projects" | "Assets" | "Workflows" | "Runs" | "Explore" | "Labels" | "Admin"; type BootstrapContext = { userId: string; @@ -71,6 +103,14 @@ type BootstrapContext = { project: { _id: string; name: string }; }; +type ProjectSummary = { + _id: string; + name: string; + description?: string; + status?: string; + createdAt?: string; +}; + type AppProps = { apiBaseUrl: string; }; @@ -132,7 +172,7 @@ function formatExecutorConfigLabel(config?: Record) { function usePathname() { const [pathname, setPathname] = useState( - typeof window === "undefined" ? "/assets" : window.location.pathname || "/assets", + typeof window === "undefined" ? "/projects" : window.location.pathname || "/projects", ); useEffect(() => { @@ -141,17 +181,19 @@ function usePathname() { return () => window.removeEventListener("popstate", handle); }, []); - return pathname === "/" ? "/assets" : pathname; + return pathname === "/" ? "/projects" : pathname; } function AppShell(props: { workspaceName: string; projectName: string; + projectControl?: React.ReactNode; active: NavItem; children: React.ReactNode; }) { const { language, setLanguage, t } = useI18n(); - const navItems: Array<{ label: NavItem; href: string; key: "navAssets" | "navWorkflows" | "navRuns" | "navExplore" | "navLabels" | "navAdmin" }> = [ + const navItems: Array<{ label: NavItem; href: string; key: "navProjects" | "navAssets" | "navWorkflows" | "navRuns" | "navExplore" | "navLabels" | "navAdmin" }> = [ + { label: "Projects", href: "/projects", key: "navProjects" }, { label: "Assets", href: "/assets", key: "navAssets" }, { label: "Workflows", href: "/workflows", key: "navWorkflows" }, { label: "Runs", href: "/runs", key: "navRuns" }, @@ -170,7 +212,7 @@ function AppShell(props: {
{t("project")} - {props.projectName} + {props.projectControl ?? {props.projectName}}
@@ -200,7 +242,14 @@ function AppShell(props: {
    {navItems.map((item) => (
  • - + { + event.preventDefault(); + navigateTo(item.href); + }} + > {t(item.key)}
    • @@ -212,25 +261,163 @@ function AppShell(props: { ); } +function ProjectsPage(props: { + api: ApiClient; + bootstrap: BootstrapContext; + projects: ProjectSummary[]; + activeProjectId: string; + onProjectCreated: (project: ProjectSummary) => Promise | void; + onProjectSelected: (projectId: string, nextPath?: string) => void; +}) { + const { t } = useI18n(); + const [projectName, setProjectName] = useState(""); + const [projectDescription, setProjectDescription] = useState(""); + const [error, setError] = useState(null); + + return ( +
    +
    +

    {t("projectsTitle")}

    +

    {t("projectsDescription")}

    +
    + +