eust-w 43326d6a5e feat(platform): land workflow, data tooling, and docs updates
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
2026-04-24 12:29:46 +08:00

1796 lines
83 KiB
Python

import contextlib
import importlib.util
import io
import json
import os
import pathlib
import tempfile
import unittest
from unittest import mock
MODULE_PATH = pathlib.Path(__file__).with_name("main.py")
SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH)
assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class _FakeHTTPResponse:
def __init__(self, payload: bytes):
self._payload = payload
self._offset = 0
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self, *_args, **_kwargs):
if self._offset >= len(self._payload):
return b""
chunk = self._payload[self._offset:]
self._offset = len(self._payload)
return chunk
class _FakeBosClient:
def __init__(self):
self.uploads = []
self.downloads = []
def put_object_from_file(self, bucket, key, file_name, **kwargs):
self.uploads.append(
{
"bucket": bucket,
"key": key,
"file_name": file_name,
"content_type": kwargs.get("content_type"),
}
)
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}"
def get_object_to_file(self, bucket, key, file_name, **_kwargs):
self.downloads.append(
{
"bucket": bucket,
"key": key,
"file_name": file_name,
}
)
pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8"))
class _FakeBosClientBytesURL(_FakeBosClient):
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8")
class _FakeBosListClient(_FakeBosClient):
def __init__(self, objects, object_payloads):
super().__init__()
self._objects = objects
self._object_payloads = object_payloads
self.downloaded_payload_keys = []
def list_all_objects(self, bucket, prefix=None):
return list(self._objects)
def get_object_as_string(self, bucket, key):
self.downloaded_payload_keys.append((bucket, key))
return self._object_payloads[key]
class SourcePathFromItemTests(unittest.TestCase):
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "node-output"
workspace_root = pathlib.Path(temp_dir) / "workspace"
output_dir.mkdir(parents=True)
workspace_root.mkdir(parents=True)
item = {
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4",
"previewURL": "https://example.com/should-not-download.mp4",
}
}
fake_client = _FakeBosClient()
with mock.patch.dict(
os.environ,
{
"WORKSPACE_ROOT": str(workspace_root),
"BOS_ENDPOINT": "https://bj.bcebos.com",
"BOS_ACCESS_KEY_ID": "ak",
"BOS_SECRET_ACCESS_KEY": "sk",
},
clear=False,
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object(
MODULE.request,
"urlopen",
) as urlopen:
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(first.exists())
self.assertEqual(first, second)
self.assertIn(str(workspace_root), str(first))
self.assertEqual(len(fake_client.downloads), 1)
build_client.assert_called()
urlopen.assert_not_called()
def test_downloads_http_preview_url_to_local_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
item = {
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4",
"previewURL": "https://example.com/demo.mp4?signature=abc",
}
}
with mock.patch.object(
MODULE.request,
"urlopen",
return_value=_FakeHTTPResponse(b"real-video-bytes"),
) as urlopen:
resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(resolved.exists())
self.assertEqual(resolved.read_bytes(), b"real-video-bytes")
urlopen.assert_called_once()
def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "node-output"
workspace_root = pathlib.Path(temp_dir) / "workspace"
output_dir.mkdir(parents=True)
workspace_root.mkdir(parents=True)
item = {
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4",
"previewURL": "https://example.com/demo.mp4?signature=abc",
}
}
with mock.patch.dict(
os.environ,
{
"WORKSPACE_ROOT": str(workspace_root),
"BOS_ENDPOINT": "",
"BOS_ACCESS_KEY_ID": "",
"BOS_SECRET_ACCESS_KEY": "",
},
clear=False,
), mock.patch.object(
MODULE.request,
"urlopen",
return_value=_FakeHTTPResponse(b"real-video-bytes"),
) as urlopen:
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
self.assertTrue(first.exists())
self.assertEqual(first, second)
self.assertEqual(first.read_bytes(), b"real-video-bytes")
self.assertIn(str(workspace_root / "_bos_cache"), str(first))
urlopen.assert_called_once()
def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self):
source = {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4",
"previewURL": "",
}
with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer:
resolved = MODULE.remote_video_url_for_source(source)
self.assertEqual(resolved, "https://signed.example.com/video.mp4")
signer.assert_called_once_with(source["path"])
class RuntimeFileContractTests(unittest.TestCase):
def test_load_input_artifacts_prefers_manifest_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json"
manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8")
with mock.patch.dict(
os.environ,
{
"INPUT_ARTIFACTS_PATH": str(manifest_path),
"INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]',
},
clear=False,
):
loaded = MODULE.load_input_artifacts()
self.assertEqual(len(loaded), 1)
self.assertEqual(loaded[0]["id"], "artifact-from-file")
def test_main_writes_result_and_stage_manifest_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
node_root = pathlib.Path(temp_dir)
output_dir = node_root / "outputs"
output_dir.mkdir(parents=True)
input_artifacts_path = node_root / "input-artifacts.json"
input_artifacts_path.write_text("[]", encoding="utf-8")
result_path = node_root / "result.json"
stage_manifest_path = node_root / "stage-manifest.json"
artifacts = [
{
"id": "artifact-1",
"runId": "run-1",
"nodeId": "node-1",
"name": "report.json",
"type": "json",
"size": "12 B",
"path": str(output_dir / "report.json"),
}
]
with mock.patch.dict(
os.environ,
{
"RUN_ID": "run-1",
"WORKFLOW_ID": "wf-1",
"NODE_ID": "node-1",
"NODE_LABEL": "通用处理",
"OUTPUT_DIR": str(output_dir),
"NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}',
"INPUT_ARTIFACTS_PATH": str(input_artifacts_path),
"RESULT_OUTPUT_PATH": str(result_path),
"STAGE_MANIFEST_PATH": str(stage_manifest_path),
},
clear=False,
), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object(
MODULE,
"resource_series",
return_value=[],
):
stdout = io.StringIO()
with contextlib.redirect_stdout(stdout):
exit_code = MODULE.main()
self.assertEqual(exit_code, 0)
self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue())
result_payload = json.loads(result_path.read_text(encoding="utf-8"))
self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1")
stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8"))
self.assertEqual(stage_manifest["artifactCount"], 1)
self.assertEqual(stage_manifest["nodeLabel"], "通用处理")
def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"dataset-index.json",
{
"datasetType": "x-platform-index",
"collectionCount": 1,
"collections": [{"id": "collection-1"}],
"items": [{"id": "item-1"}, {"id": "item-2"}],
},
)
data_ref = artifact["data"]["dataRef"]
self.assertEqual(data_ref["kind"], "dataset")
self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists())
self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists())
manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8"))
self.assertEqual(manifest["datasetType"], "x-platform-index")
self.assertIn("items", manifest["partitionFiles"])
self.assertIn("collections", manifest["partitionFiles"])
def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"filtered-videos.json",
{
"datasetType": "x-platform-filtered",
"count": 2,
"items": [{"id": "v1"}, {"id": "v2"}],
},
)
loaded = MODULE.load_artifact_payload(artifact)
self.assertEqual(loaded["datasetType"], "x-platform-filtered")
self.assertEqual(len(loaded["items"]), 2)
def test_load_payload_from_file_retries_transient_json_decode_error(self):
with tempfile.TemporaryDirectory() as temp_dir:
payload_path = pathlib.Path(temp_dir) / "payload.json"
payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8")
decode_error = json.JSONDecodeError("transient", "", 0)
with mock.patch.object(
MODULE.json,
"load",
side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}],
) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock:
loaded = MODULE.load_payload_from_file(payload_path)
self.assertEqual(loaded["datasetType"], "x-platform")
self.assertEqual(len(loaded["items"]), 1)
self.assertEqual(json_load.call_count, 2)
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
def test_load_payload_from_file_retries_until_payload_path_exists(self):
with tempfile.TemporaryDirectory() as temp_dir:
payload_path = pathlib.Path(temp_dir) / "payload.json"
payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8")
original_exists = pathlib.Path.exists
seen = {"count": 0}
def fake_exists(path_obj: pathlib.Path):
if path_obj == payload_path and seen["count"] == 0:
seen["count"] += 1
return False
return original_exists(path_obj)
with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object(
MODULE.time,
"sleep",
) as sleep_mock:
loaded = MODULE.load_payload_from_file(payload_path)
self.assertEqual(loaded["datasetType"], "x-platform")
self.assertEqual(len(loaded["items"]), 1)
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
def test_stage_manifest_includes_data_refs(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
artifact = MODULE.write_json_artifact(
"run-1",
"node-1",
output_dir,
"packages-manifest.json",
{
"datasetType": "x-platform-packages",
"packageCount": 1,
"items": [{"packageName": "pkg-1"}],
},
)
manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact])
self.assertEqual(len(manifest["dataRefs"]), 1)
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
class XPlatformIndexingTests(unittest.TestCase):
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "outputs"
output_dir.mkdir(parents=True)
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
manifest_path.write_text("{}", encoding="utf-8")
payload = MODULE.load_xplatform_index_payload(
{},
[
{
"path": str(manifest_path),
"data": {
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
"entries": [
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "工厂",
"place": "绿控传动",
"date": "20260330",
},
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260327132547_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2880",
"frameRate": 29.97,
"duration": 0.6,
}
],
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
},
},
],
},
}
],
output_dir,
)
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["videoCount"], 1)
self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动")
def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir) / "outputs"
output_dir.mkdir(parents=True)
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
manifest_path.write_text("{}", encoding="utf-8")
with self.assertRaisesRegex(RuntimeError, "source-manifest"):
MODULE.load_xplatform_index_payload(
{},
[
{
"name": "source-manifest.json",
"path": str(manifest_path),
"data": {
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
"entriesCount": 59,
"dataRef": {"payloadPath": str(manifest_path)},
},
}
],
output_dir,
)
def test_manifest_entries_support_video_metadata_alias(self):
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "工厂",
"place": "绿控传动",
"date": "20260330",
},
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260327132547_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2880",
"frameRate": 29.97,
"duration": 0.6,
}
],
},
{
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
},
},
],
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertEqual(payload["items"][0]["codec"], "hevc")
def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "algo" / "X-platform-test"
collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = collection / "DJI_001"
video_dir.mkdir(parents=True)
meta_path = collection / "meta.json"
video_meta_path = collection / "video_meta.json"
video_path = video_dir / "DJI_0001_D.mp4"
meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8")
video_meta_path.write_text(
'[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
encoding="utf-8",
)
video_path.write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
"sizeBytes": meta_path.stat().st_size,
"mimeType": "application/json",
"source": {"path": str(meta_path)},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
"sizeBytes": video_meta_path.stat().st_size,
"mimeType": "application/json",
"source": {"path": str(video_meta_path)},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
"sizeBytes": video_path.stat().st_size,
"mimeType": "video/mp4",
"source": {"path": str(video_path)},
},
],
str(root),
)
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
self.assertEqual(payload["items"][0]["codec"], "hevc")
def test_manifest_entries_support_root_level_meta_and_video_meta(self):
payload = MODULE.scan_xplatform_manifest_entries(
[
{
"path": "meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {
"type": "家庭",
"place": "蓟门里小区",
"date": "20260319",
},
},
{
"path": "video_meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
}
],
},
{
"path": "DJI_001/DJI_20260318112834_0001_D.MP4",
"sizeBytes": 123,
"mimeType": "video/mp4",
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/video.mp4",
},
},
],
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(collection["relativePath"], "")
self.assertEqual(collection["videoCount"], 1)
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertEqual(collection["meta"]["place"], "蓟门里小区")
item = payload["items"][0]
self.assertEqual(item["collectionName"], collection["name"])
self.assertEqual(item["collectionRelativePath"], "")
self.assertEqual(item["videoFolder"], "DJI_001")
self.assertEqual(item["codec"], "hevc")
self.assertEqual(item["meta"]["type"], "家庭")
def test_local_dataset_root_with_meta_is_treated_as_collection_root(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
video_dir = root / "DJI_001"
video_dir.mkdir(parents=True)
(root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8")
(root / "video_meta.json").write_text(
'[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
encoding="utf-8",
)
(video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_dataset(root)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["name"], root.name)
self.assertEqual(collection["relativePath"], "")
self.assertEqual(collection["missingRequiredFiles"], [])
item = payload["items"][0]
self.assertEqual(item["collectionName"], root.name)
self.assertEqual(item["collectionRelativePath"], "")
self.assertEqual(item["codec"], "hevc")
def test_local_dataset_root_supports_video_metadata_alias(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11"
video_dir = root / "DJI_001"
video_dir.mkdir(parents=True)
(root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8")
(root / "video_metadata.json").write_text(
'[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]',
encoding="utf-8",
)
(video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video")
payload = MODULE.scan_xplatform_dataset(root)
self.assertEqual(payload["collectionCount"], 1)
collection = payload["collections"][0]
self.assertEqual(collection["missingRequiredFiles"], [])
self.assertTrue(collection["hasVideoMeta"])
self.assertEqual(payload["items"][0]["codec"], "hevc")
class SourceOperatorTests(unittest.TestCase):
def test_builtin_source_defaults_to_inline_when_source_json_is_present(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-默认内联",
"hyperparams": [
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["dataSource"], "INLINE")
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
def test_builtin_source_prefers_source_json_over_bos_env(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-JSON",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
def test_builtin_source_prefers_source_text_over_bos_env(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "真实数据源-文本",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
],
}
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
self.assertEqual(payload["listedCount"], 1)
self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma")
def test_directory_incremental_source_filter_emits_full_new_directory(self):
entries = [
{"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"},
{"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"},
{"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"},
{"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"},
]
emitted, checkpoint = MODULE.apply_incremental_source_filter(
entries,
"BOS",
"wjzh-data-migration",
{
"SOURCE_STATE_JSON": json.dumps(
{
"seenFingerprints": ["batch-a"],
"incrementalUnit": "DIRECTORY",
"directoryDepth": 1,
}
),
"SOURCE_FINGERPRINT": "fp-directory",
"INCREMENTAL_UNIT": "DIRECTORY",
"INCREMENTAL_DIRECTORY_DEPTH": "1",
},
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
"wjzh-0330-Data-migration",
"",
)
self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"])
self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"])
self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY")
self.assertEqual(checkpoint["directoryDepth"], 1)
self.assertEqual(checkpoint["newCount"], 2)
self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"])
def test_list_bos_entries_skips_large_non_metadata_json_payloads(self):
object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})()
object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})()
fake_client = _FakeBosListClient(
[object_meta, object_imu],
{
"wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}',
"wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}',
},
)
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
entries = MODULE.list_bos_entries(
"wjzh-data-migration",
"wjzh-0330-Data-migration",
"",
"https://bj.bcebos.com",
"ak",
"sk",
)
self.assertEqual(len(entries), 2)
meta_entry = next(item for item in entries if item["path"].endswith("meta.json"))
imu_entry = next(item for item in entries if item["path"].endswith("_imu.json"))
self.assertIn("data", meta_entry)
self.assertNotIn("data", imu_entry)
self.assertEqual(
fake_client.downloaded_payload_keys,
[("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")],
)
def test_bos_source_parses_bucket_and_prefix_from_source_path(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "BOS 增量扫描",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
{"key": "SCAN_MODE", "value": "FULL"},
],
}
fake_entries = [
{
"path": "SZ_000_0005/meta.json",
"storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json",
"sizeBytes": 10,
"mimeType": "application/json",
"data": {"type": "工厂"},
"source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"},
}
]
with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object(
MODULE,
"source_entry_artifact",
return_value={
"id": "artifact-1",
"runId": "run-1",
"nodeId": "source",
"name": "meta.json",
"type": "json",
"path": str(output_dir / "meta.json"),
},
):
artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, [])
self.assertGreaterEqual(len(artifacts), 1)
list_bos_entries.assert_called_once_with(
"wjzh-data-migration",
"wjzh-0330-Data-migration",
"",
"https://bj.bcebos.com",
"ak",
"sk",
)
class QualityMetricHelperTests(unittest.TestCase):
def test_build_skin_mask_detects_skin_tone_patch(self):
_, np = MODULE.require_cv2()
frame = np.zeros((80, 80, 3), dtype=np.uint8)
frame[20:60, 20:60] = (90, 130, 190)
mask = MODULE.build_skin_mask(frame)
self.assertGreater(int(mask.sum()), 0)
def test_build_skin_mask_rejects_neutral_background(self):
_, np = MODULE.require_cv2()
frame = np.full((80, 80, 3), 128, dtype=np.uint8)
mask = MODULE.build_skin_mask(frame)
self.assertEqual(int(mask.sum()), 0)
def test_motion_jitter_ratio_stays_low_for_smooth_motion(self):
smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)]
jitter = MODULE.motion_jitter_ratio(smooth, 100.0)
self.assertLess(jitter, 0.01)
def test_motion_jitter_ratio_increases_for_erratic_motion(self):
smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0)
erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0)
self.assertGreater(erratic, smooth)
self.assertGreater(erratic, 0.05)
def test_largest_skin_component_ratio_ignores_small_noise(self):
_, np = MODULE.require_cv2()
mask = np.zeros((100, 100), dtype=np.uint8)
mask[10:13, 10:13] = 255
ratio = MODULE.largest_skin_component_ratio(mask)
self.assertEqual(ratio, 0.0)
def test_largest_skin_component_ratio_returns_large_component_area(self):
_, np = MODULE.require_cv2()
mask = np.zeros((100, 100), dtype=np.uint8)
mask[20:50, 30:70] = 255
ratio = MODULE.largest_skin_component_ratio(mask)
self.assertGreater(ratio, 0.1)
self.assertLess(ratio, 0.13)
class QualityOperatorTests(unittest.TestCase):
def test_video_quality_accepts_zero_hand_occlusion(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "视频质检",
"hyperparams": [
{"key": "MIN_CLARITY", "value": "0.8"},
{"key": "MAX_SHAKE", "value": "0.1"},
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
],
}
inputs = [
{
"data": {
"items": [
{
"filename": "demo.mp4",
"source": {"path": "/tmp/demo.mp4"},
}
]
}
}
]
with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object(
MODULE,
"analyze_video_quality_metrics",
return_value={
"clarity": 1.0,
"shake": 0.0,
"handOcclusion": 0.0,
"qualityScore": 1.0,
},
):
artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["passedCount"], 1)
self.assertTrue(payload["items"][0]["quality"]["passed"])
def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
node_data = {
"label": "数据过滤",
"hyperparams": [
{"key": "DATE_START", "value": "2026-03-01"},
{"key": "DATE_END", "value": "2026-03-31"},
{"key": "PLACE", "value": "蓟门里小区"},
{"key": "TYPE", "value": "家庭"},
{"key": "MIN_CLARITY", "value": "0.8"},
{"key": "MAX_SHAKE", "value": "0.1"},
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
],
}
inputs = [
{
"data": {
"items": [
{
"meta": {
"date": "20260319",
"place": "蓟门里小区",
"type": "家庭",
},
"quality": {
"clarity": 0.95,
"shake": 0.0,
"handOcclusion": 0.0,
"passed": True,
},
}
]
}
}
]
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
payload = MODULE.load_artifact_payload(artifacts[0])
self.assertEqual(payload["count"], 1)
self.assertEqual(len(payload["items"]), 1)
class UploadOperatorTests(unittest.TestCase):
def test_build_artifact_skips_sha256_for_large_files(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
clip_path = root / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
MODULE,
"sha256_file",
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
):
artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video")
metadata = artifact.get("metadata") or {}
self.assertEqual(metadata.get("sha256", ""), "")
self.assertTrue(metadata.get("sha256Skipped"))
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
def test_uploads_nested_package_outputs_to_bos(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
package_root = root / "package"
package_dir = package_root / "packages" / "clip001_package"
package_dir.mkdir(parents=True)
manifest_path = package_root / "packages-manifest.json"
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
clip_path = package_dir / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
annotation_path = package_dir / "clip001_annotation.json"
annotation_path.write_text('{"label":"walking"}', encoding="utf-8")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
MODULE.build_artifact("run-1", "package", clip_path, "video"),
MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}),
]
fake_client = _FakeBosClient()
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
uploaded_names = {item["name"] for item in artifacts}
uploaded_keys = {item["key"] for item in fake_client.uploads}
self.assertEqual(len(fake_client.uploads), 3)
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
self.assertEqual(uploaded_summary_payload["uploadedCount"], 3)
self.assertEqual(uploaded_summary_payload["bucket"], "datastorage")
self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo")
def test_upload_skips_sha256_for_large_remote_artifacts(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
clip_path = root / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")]
fake_client = _FakeBosClient()
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
MODULE,
"sha256_file",
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://"))
metadata = remote_video.get("metadata") or {}
self.assertEqual(metadata.get("sha256", ""), "")
self.assertTrue(metadata.get("sha256Skipped"))
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
class _FakeInsertManyResult:
def __init__(self, inserted_ids):
self.inserted_ids = inserted_ids
class _FakeUpdateResult:
def __init__(self, matched_count=0, modified_count=0, upserted_id=None):
self.matched_count = matched_count
self.modified_count = modified_count
self.upserted_id = upserted_id
class _FakeMongoCollection:
def __init__(self):
self.insert_many_calls = []
self.replace_one_calls = []
def insert_many(self, documents, ordered=False):
self.insert_many_calls.append({"documents": documents, "ordered": ordered})
return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))])
def replace_one(self, filter_document, replacement, upsert=False):
self.replace_one_calls.append(
{
"filter": filter_document,
"replacement": replacement,
"upsert": upsert,
}
)
return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None)
class _FakeMongoDatabase:
def __init__(self, collection):
self._collection = collection
self.requested_collections = []
def __getitem__(self, name):
self.requested_collections.append(name)
return self._collection
class _FakeMongoClient:
def __init__(self, database):
self._database = database
self.requested_databases = []
self.closed = False
def __getitem__(self, name):
self.requested_databases.append(name)
return self._database
def close(self):
self.closed = True
class MongoWriteOperatorTests(unittest.TestCase):
def test_inserts_documents_into_default_business_database(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
{"key": "MONGO_MODE", "value": "insert_many"},
],
}
inputs = [
{
"data": {
"items": [
{"clipId": "clip-001", "qualityScore": 0.91},
{"clipId": "clip-002", "qualityScore": 0.87},
]
}
}
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(database.requested_collections, ["qualityReports"])
self.assertEqual(client.requested_databases, ["business-data"])
self.assertEqual(len(collection.insert_many_calls), 1)
self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001")
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["collection"], "qualityReports")
self.assertEqual(summary["data"]["mode"], "insert_many")
self.assertEqual(summary["data"]["target"], "business")
self.assertEqual(summary["data"]["insertedCount"], 2)
self.assertTrue(client.closed)
def test_upserts_documents_by_key(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "annotations"},
{"key": "MONGO_MODE", "value": "upsert"},
{"key": "UPSERT_KEY", "value": "clipId"},
],
}
inputs = [
{"data": {"clipId": "clip-001", "label": "walking"}},
{"data": {"clipId": "clip-002", "label": "standing"}},
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(len(collection.replace_one_calls), 2)
self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"})
self.assertTrue(collection.replace_one_calls[0]["upsert"])
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["mode"], "upsert")
self.assertEqual(summary["data"]["upsertKey"], "clipId")
self.assertEqual(summary["data"]["target"], "business")
self.assertEqual(summary["data"]["upsertedCount"], 2)
def test_unwraps_single_value_dict_documents_for_business_writes(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "businessWriteValidation"},
{"key": "MONGO_MODE", "value": "insert_many"},
],
}
inputs = [
{
"data": {
"items": [
{
"value": {
"runLabel": "business-mongo-test",
"source": "workflow",
"createdAt": "2026-04-14T00:00:00Z",
"value": 1,
}
}
]
}
}
]
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
self.assertEqual(len(collection.insert_many_calls), 1)
inserted = collection.insert_many_calls[0]["documents"][0]
self.assertEqual(inserted["runLabel"], "business-mongo-test")
self.assertEqual(inserted["source"], "workflow")
self.assertEqual(inserted["value"], 1)
self.assertNotEqual(set(inserted.keys()), {"value"})
def test_can_target_platform_database_explicitly(self):
collection = _FakeMongoCollection()
database = _FakeMongoDatabase(collection)
client = _FakeMongoClient(database)
node_data = {
"label": "MongoDB 写入",
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"hyperparams": [
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
{"key": "MONGO_TARGET", "value": "platform"},
],
}
with tempfile.TemporaryDirectory() as temp_dir:
with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False):
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}])
self.assertEqual(client.requested_databases, ["droboticflow"])
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
self.assertEqual(summary["data"]["target"], "platform")
def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
package_manifest = {
"packageCount": 1,
"items": [
{
"packageName": "clip001_package",
"clipId": "clip001",
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"startSecond": 0,
"endSecond": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
"quality": {"clarity": 0.9, "passed": True},
"annotation": {"objects": ["人"], "minConfidence": 0.92},
}
],
}
inputs = [
{"name": "packages-manifest.json", "type": "json", "data": package_manifest},
{
"name": "clip001.mp4",
"type": "video",
"path": "/tmp/non-existent-clip001.mp4",
"data": {"preview": {"kind": "video", "format": "mp4"}},
},
{
"name": "clip001_annotation.json",
"type": "json",
"path": "/tmp/non-existent-clip001_annotation.json",
"data": {"objects": ["人"], "minConfidence": 0.92},
},
{
"name": "clip001_quality.json",
"type": "json",
"path": "/tmp/non-existent-clip001_quality.json",
"data": {"clarity": 0.9, "passed": True},
},
{
"name": "clip001_config.json",
"type": "json",
"path": "/tmp/non-existent-clip001_config.json",
"data": {"collectionId": "collection-1"},
},
]
fake_client = _FakeBosClient()
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
uploaded_keys = {item["key"] for item in fake_client.uploads}
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys)
self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys)
self.assertFalse(
(output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists()
)
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
self.assertEqual(uploaded_summary_payload["uploadedCount"], 5)
def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
package_root = root / "package"
package_dir = package_root / "packages" / "clip001_package"
package_dir.mkdir(parents=True)
manifest_path = package_root / "packages-manifest.json"
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
clip_path = package_dir / "clip001.mp4"
clip_path.write_bytes(b"video-bytes")
output_dir = root / "upload"
node_data = {
"label": "结果上传",
"bucket": "datastorage",
"prefix": "factory/output/demo",
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
],
}
inputs = [
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
MODULE.build_artifact("run-1", "package", clip_path, "video"),
]
with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")):
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
json.dumps({"artifacts": artifacts}, ensure_ascii=False)
class XPlatformClipHandoffTests(unittest.TestCase):
def test_video_slicing_manifest_preserves_source_reference(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 41.8,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
clip = manifest_payload["items"][0]
self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4")
def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 365.0,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_write_video_segments(_source_path, segments):
outputs = []
for segment in segments:
destination = pathlib.Path(segment["destination"])
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8"))
outputs.append(
{
**segment,
"duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3),
"frameRate": 29.97,
"resolution": "3840x2160",
}
)
return outputs
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(
MODULE,
"write_video_segments",
side_effect=fake_write_video_segments,
) as multi_writer, mock.patch.object(
MODULE,
"write_video_segment",
) as single_writer, mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["clipCount"], 2)
self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [
"DJI_20260318112834_0001_D_clip001.mp4",
"DJI_20260318112834_0001_D_clip002.mp4",
])
multi_writer.assert_called_once()
single_writer.assert_not_called()
def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
destination = root / "clips" / "clip001.mp4"
def fake_run(args, stdout=None, stderr=None, text=None):
self.assertEqual(args[0], "ffmpeg")
self.assertIn(str(source_path), args)
self.assertEqual(args[-1], str(destination))
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-video")
return mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE.subprocess,
"run",
side_effect=fake_run,
) as run_mock, mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
):
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
self.assertEqual(metrics["duration"], 21.5)
self.assertEqual(metrics["frameRate"], 29.97)
self.assertEqual(metrics["resolution"], "3840x2160")
run_args = run_mock.call_args.args[0]
self.assertIn("-ss", run_args)
self.assertIn("-t", run_args)
self.assertIn("-c", run_args)
self.assertIn("copy", run_args)
self.assertNotIn("-movflags", run_args)
self.assertNotIn("+faststart", run_args)
def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = pathlib.Path(temp_dir)
source_path = root / "source.mp4"
source_path.write_bytes(b"source-video")
destination = root / "clips" / "clip001.mp4"
calls = []
def fake_run(args, stdout=None, stderr=None, text=None):
calls.append(list(args))
if len(calls) == 1:
return mock.Mock(returncode=1, stdout="", stderr="copy failed")
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-video")
return mock.Mock(returncode=0, stdout="", stderr="")
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE.subprocess,
"run",
side_effect=fake_run,
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
):
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
self.assertEqual(metrics["duration"], 21.5)
self.assertEqual(len(calls), 2)
self.assertIn("copy", calls[0])
self.assertIn("libx264", calls[1])
self.assertIn("aac", calls[1])
def test_video_slicing_prefers_ffmpeg_when_available(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "filtered-videos.json",
"type": "json",
"data": {
"count": 1,
"items": [
{
"filename": "DJI_20260318112834_0001_D.MP4",
"duration": 365.0,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"passed": True},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频切片",
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
}
def fake_ffmpeg_writer(_source_path, destination, start_second, end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8"))
return {
"duration": round(float(end_second) - float(start_second), 3),
"frameRate": 29.97,
"resolution": "3840x2160",
}
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
MODULE, "source_path_from_item", return_value=source_path
), mock.patch.object(
MODULE,
"inspect_video_file",
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
), mock.patch.object(
MODULE,
"write_video_segment_with_ffmpeg",
side_effect=fake_ffmpeg_writer,
) as ffmpeg_writer, mock.patch.object(
MODULE,
"write_video_segments",
) as multi_writer, mock.patch.object(
MODULE,
"write_video_segment",
) as single_writer, mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["clipCount"], 2)
self.assertEqual(ffmpeg_writer.call_count, 2)
multi_writer.assert_not_called()
single_writer.assert_not_called()
def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
shared_clip_path = output_dir / "shared" / "clip001.mp4"
shared_clip_path.parent.mkdir(parents=True, exist_ok=True)
shared_clip_path.write_bytes(b"clip-bytes")
inputs = [
{
"name": "clips-manifest.json",
"type": "json",
"data": {
"clipCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"clipPath": str(shared_clip_path),
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"duration": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "视频标注",
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}],
}
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
MODULE,
"annotate_video_clip",
return_value={
"duration": 41.8,
"actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}],
"objects": ["人"],
"minConfidence": 0.92,
},
), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
self.assertEqual(manifest_payload["annotationCount"], 1)
self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path))
self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts))
def test_materialize_clip_from_source_prefers_remote_segment_generation(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
item = {
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
},
}
def fake_remote_segment(_source_url, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8}
with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object(
MODULE,
"write_video_segment_from_remote_url",
side_effect=fake_remote_segment,
) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer:
resolved = MODULE.materialize_clip_from_source(item, output_dir)
self.assertIsNotNone(resolved)
self.assertTrue(resolved.exists())
self.assertEqual(resolved.read_bytes(), b"clip-bytes")
remote_writer.assert_called_once()
local_materializer.assert_not_called()
def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
source_path = output_dir / "source.mp4"
source_path.write_bytes(b"source-video")
inputs = [
{
"name": "annotations-manifest.json",
"type": "json",
"data": {
"annotationCount": 1,
"items": [
{
"clipId": "clip001",
"clipName": "clip001.mp4",
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
"startSecond": 0,
"endSecond": 41.8,
"duration": 41.8,
"collectionId": "collection-1",
"meta": {"type": "家庭"},
"quality": {"clarity": 0.9, "passed": True},
"annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92},
"source": {
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
"previewURL": "https://example.com/source.mp4",
},
}
],
},
}
]
node_data = {
"label": "成品打包",
"hyperparams": [],
}
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(b"clip-bytes")
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
MODULE,
"write_video_segment",
side_effect=fake_write_video_segment,
), mock.patch.object(
MODULE,
"extract_preview_data",
return_value={"preview": {"kind": "video", "format": "mp4"}},
):
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
manifest_payload = MODULE.load_artifact_payload(manifest)
package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4")
self.assertEqual(manifest_payload["packageCount"], 1)
manifest_item = manifest_payload["items"][0]
self.assertEqual(manifest_item["clipName"], "clip001.mp4")
self.assertEqual(manifest_item["collectionId"], "collection-1")
self.assertEqual(manifest_item["startSecond"], 0)
self.assertEqual(manifest_item["endSecond"], 41.8)
self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
self.assertEqual(package_clip["name"], "clip001.mp4")
if __name__ == "__main__":
unittest.main()