forked from longtao.wu/droboticflow
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
1796 lines
83 KiB
Python
1796 lines
83 KiB
Python
import contextlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
MODULE_PATH = pathlib.Path(__file__).with_name("main.py")
|
|
SPEC = importlib.util.spec_from_file_location("operator_runner_main", MODULE_PATH)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
MODULE = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(MODULE)
|
|
|
|
|
|
class _FakeHTTPResponse:
|
|
def __init__(self, payload: bytes):
|
|
self._payload = payload
|
|
self._offset = 0
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self, *_args, **_kwargs):
|
|
if self._offset >= len(self._payload):
|
|
return b""
|
|
chunk = self._payload[self._offset:]
|
|
self._offset = len(self._payload)
|
|
return chunk
|
|
|
|
|
|
class _FakeBosClient:
|
|
def __init__(self):
|
|
self.uploads = []
|
|
self.downloads = []
|
|
|
|
def put_object_from_file(self, bucket, key, file_name, **kwargs):
|
|
self.uploads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"file_name": file_name,
|
|
"content_type": kwargs.get("content_type"),
|
|
}
|
|
)
|
|
|
|
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
|
|
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}"
|
|
|
|
def get_object_to_file(self, bucket, key, file_name, **_kwargs):
|
|
self.downloads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"key": key,
|
|
"file_name": file_name,
|
|
}
|
|
)
|
|
pathlib.Path(file_name).write_bytes(f"{bucket}/{key}".encode("utf-8"))
|
|
|
|
|
|
class _FakeBosClientBytesURL(_FakeBosClient):
|
|
def generate_pre_signed_url(self, bucket, key, expiration_in_seconds=3600, protocol="https", **_kwargs):
|
|
return f"{protocol}://{bucket}.example.com/{key}?expires={expiration_in_seconds}".encode("utf-8")
|
|
|
|
|
|
class _FakeBosListClient(_FakeBosClient):
|
|
def __init__(self, objects, object_payloads):
|
|
super().__init__()
|
|
self._objects = objects
|
|
self._object_payloads = object_payloads
|
|
self.downloaded_payload_keys = []
|
|
|
|
def list_all_objects(self, bucket, prefix=None):
|
|
return list(self._objects)
|
|
|
|
def get_object_as_string(self, bucket, key):
|
|
self.downloaded_payload_keys.append((bucket, key))
|
|
return self._object_payloads[key]
|
|
|
|
|
|
class SourcePathFromItemTests(unittest.TestCase):
|
|
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "node-output"
|
|
workspace_root = pathlib.Path(temp_dir) / "workspace"
|
|
output_dir.mkdir(parents=True)
|
|
workspace_root.mkdir(parents=True)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"previewURL": "https://example.com/should-not-download.mp4",
|
|
}
|
|
}
|
|
fake_client = _FakeBosClient()
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"WORKSPACE_ROOT": str(workspace_root),
|
|
"BOS_ENDPOINT": "https://bj.bcebos.com",
|
|
"BOS_ACCESS_KEY_ID": "ak",
|
|
"BOS_SECRET_ACCESS_KEY": "sk",
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")) as build_client, mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
) as urlopen:
|
|
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
|
|
self.assertTrue(first.exists())
|
|
self.assertEqual(first, second)
|
|
self.assertIn(str(workspace_root), str(first))
|
|
self.assertEqual(len(fake_client.downloads), 1)
|
|
build_client.assert_called()
|
|
urlopen.assert_not_called()
|
|
|
|
def test_downloads_http_preview_url_to_local_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/demo.mp4",
|
|
"previewURL": "https://example.com/demo.mp4?signature=abc",
|
|
}
|
|
}
|
|
with mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
return_value=_FakeHTTPResponse(b"real-video-bytes"),
|
|
) as urlopen:
|
|
resolved = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
self.assertTrue(resolved.exists())
|
|
self.assertEqual(resolved.read_bytes(), b"real-video-bytes")
|
|
urlopen.assert_called_once()
|
|
|
|
def test_prefers_workspace_cache_path_for_preview_url_when_bos_credentials_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "node-output"
|
|
workspace_root = pathlib.Path(temp_dir) / "workspace"
|
|
output_dir.mkdir(parents=True)
|
|
workspace_root.mkdir(parents=True)
|
|
item = {
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0000/DJI_001/demo.mp4",
|
|
"previewURL": "https://example.com/demo.mp4?signature=abc",
|
|
}
|
|
}
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"WORKSPACE_ROOT": str(workspace_root),
|
|
"BOS_ENDPOINT": "",
|
|
"BOS_ACCESS_KEY_ID": "",
|
|
"BOS_SECRET_ACCESS_KEY": "",
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(
|
|
MODULE.request,
|
|
"urlopen",
|
|
return_value=_FakeHTTPResponse(b"real-video-bytes"),
|
|
) as urlopen:
|
|
first = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
second = MODULE.source_path_from_item(item, "demo.mp4", output_dir)
|
|
self.assertTrue(first.exists())
|
|
self.assertEqual(first, second)
|
|
self.assertEqual(first.read_bytes(), b"real-video-bytes")
|
|
self.assertIn(str(workspace_root / "_bos_cache"), str(first))
|
|
urlopen.assert_called_once()
|
|
|
|
def test_remote_video_url_for_source_signs_bos_path_when_preview_missing(self):
|
|
source = {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_0001_D.MP4",
|
|
"previewURL": "",
|
|
}
|
|
with mock.patch.object(MODULE, "signed_bos_source_url", return_value="https://signed.example.com/video.mp4") as signer:
|
|
resolved = MODULE.remote_video_url_for_source(source)
|
|
self.assertEqual(resolved, "https://signed.example.com/video.mp4")
|
|
signer.assert_called_once_with(source["path"])
|
|
|
|
|
|
class RuntimeFileContractTests(unittest.TestCase):
|
|
def test_load_input_artifacts_prefers_manifest_file(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
manifest_path = pathlib.Path(temp_dir) / "input-artifacts.json"
|
|
manifest_path.write_text('[{"id":"artifact-from-file"}]', encoding="utf-8")
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"INPUT_ARTIFACTS_PATH": str(manifest_path),
|
|
"INPUT_ARTIFACTS_JSON": '[{"id":"artifact-inline"}]',
|
|
},
|
|
clear=False,
|
|
):
|
|
loaded = MODULE.load_input_artifacts()
|
|
|
|
self.assertEqual(len(loaded), 1)
|
|
self.assertEqual(loaded[0]["id"], "artifact-from-file")
|
|
|
|
def test_main_writes_result_and_stage_manifest_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
node_root = pathlib.Path(temp_dir)
|
|
output_dir = node_root / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
input_artifacts_path = node_root / "input-artifacts.json"
|
|
input_artifacts_path.write_text("[]", encoding="utf-8")
|
|
result_path = node_root / "result.json"
|
|
stage_manifest_path = node_root / "stage-manifest.json"
|
|
artifacts = [
|
|
{
|
|
"id": "artifact-1",
|
|
"runId": "run-1",
|
|
"nodeId": "node-1",
|
|
"name": "report.json",
|
|
"type": "json",
|
|
"size": "12 B",
|
|
"path": str(output_dir / "report.json"),
|
|
}
|
|
]
|
|
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{
|
|
"RUN_ID": "run-1",
|
|
"WORKFLOW_ID": "wf-1",
|
|
"NODE_ID": "node-1",
|
|
"NODE_LABEL": "通用处理",
|
|
"OUTPUT_DIR": str(output_dir),
|
|
"NODE_DATA_JSON": '{"label":"通用处理","hyperparams":[]}',
|
|
"INPUT_ARTIFACTS_PATH": str(input_artifacts_path),
|
|
"RESULT_OUTPUT_PATH": str(result_path),
|
|
"STAGE_MANIFEST_PATH": str(stage_manifest_path),
|
|
},
|
|
clear=False,
|
|
), mock.patch.object(MODULE, "builtin_generic", return_value=artifacts), mock.patch.object(
|
|
MODULE,
|
|
"resource_series",
|
|
return_value=[],
|
|
):
|
|
stdout = io.StringIO()
|
|
with contextlib.redirect_stdout(stdout):
|
|
exit_code = MODULE.main()
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertIn(MODULE.RESULT_PATH_PREFIX + str(result_path), stdout.getvalue())
|
|
result_payload = json.loads(result_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(result_payload["artifacts"][0]["id"], "artifact-1")
|
|
stage_manifest = json.loads(stage_manifest_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(stage_manifest["artifactCount"], 1)
|
|
self.assertEqual(stage_manifest["nodeLabel"], "通用处理")
|
|
|
|
def test_write_json_artifact_externalizes_dataset_payload_to_data_ref(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"dataset-index.json",
|
|
{
|
|
"datasetType": "x-platform-index",
|
|
"collectionCount": 1,
|
|
"collections": [{"id": "collection-1"}],
|
|
"items": [{"id": "item-1"}, {"id": "item-2"}],
|
|
},
|
|
)
|
|
|
|
data_ref = artifact["data"]["dataRef"]
|
|
self.assertEqual(data_ref["kind"], "dataset")
|
|
self.assertTrue(pathlib.Path(data_ref["manifestPath"]).exists())
|
|
self.assertTrue(pathlib.Path(data_ref["payloadPath"]).exists())
|
|
manifest = json.loads(pathlib.Path(data_ref["manifestPath"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["datasetType"], "x-platform-index")
|
|
self.assertIn("items", manifest["partitionFiles"])
|
|
self.assertIn("collections", manifest["partitionFiles"])
|
|
|
|
def test_load_artifact_payload_reads_externalized_payload_from_data_ref(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"filtered-videos.json",
|
|
{
|
|
"datasetType": "x-platform-filtered",
|
|
"count": 2,
|
|
"items": [{"id": "v1"}, {"id": "v2"}],
|
|
},
|
|
)
|
|
|
|
loaded = MODULE.load_artifact_payload(artifact)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform-filtered")
|
|
self.assertEqual(len(loaded["items"]), 2)
|
|
|
|
def test_load_payload_from_file_retries_transient_json_decode_error(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
payload_path = pathlib.Path(temp_dir) / "payload.json"
|
|
payload_path.write_text('{"items":[{"id":"v1"}]}', encoding="utf-8")
|
|
decode_error = json.JSONDecodeError("transient", "", 0)
|
|
|
|
with mock.patch.object(
|
|
MODULE.json,
|
|
"load",
|
|
side_effect=[decode_error, {"datasetType": "x-platform", "items": [{"id": "v1"}]}],
|
|
) as json_load, mock.patch.object(MODULE.time, "sleep") as sleep_mock:
|
|
loaded = MODULE.load_payload_from_file(payload_path)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform")
|
|
self.assertEqual(len(loaded["items"]), 1)
|
|
self.assertEqual(json_load.call_count, 2)
|
|
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
|
|
|
|
def test_load_payload_from_file_retries_until_payload_path_exists(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
payload_path = pathlib.Path(temp_dir) / "payload.json"
|
|
payload_path.write_text('{"datasetType":"x-platform","items":[{"id":"v1"}]}', encoding="utf-8")
|
|
original_exists = pathlib.Path.exists
|
|
seen = {"count": 0}
|
|
|
|
def fake_exists(path_obj: pathlib.Path):
|
|
if path_obj == payload_path and seen["count"] == 0:
|
|
seen["count"] += 1
|
|
return False
|
|
return original_exists(path_obj)
|
|
|
|
with mock.patch.object(pathlib.Path, "exists", autospec=True, side_effect=fake_exists), mock.patch.object(
|
|
MODULE.time,
|
|
"sleep",
|
|
) as sleep_mock:
|
|
loaded = MODULE.load_payload_from_file(payload_path)
|
|
|
|
self.assertEqual(loaded["datasetType"], "x-platform")
|
|
self.assertEqual(len(loaded["items"]), 1)
|
|
sleep_mock.assert_called_once_with(MODULE.PAYLOAD_FILE_READ_DELAY_SECONDS)
|
|
|
|
def test_stage_manifest_includes_data_refs(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
artifact = MODULE.write_json_artifact(
|
|
"run-1",
|
|
"node-1",
|
|
output_dir,
|
|
"packages-manifest.json",
|
|
{
|
|
"datasetType": "x-platform-packages",
|
|
"packageCount": 1,
|
|
"items": [{"packageName": "pkg-1"}],
|
|
},
|
|
)
|
|
|
|
manifest = MODULE.stage_manifest_payload("run-1", "wf-1", "node-1", "成品打包", output_dir, [artifact])
|
|
|
|
self.assertEqual(len(manifest["dataRefs"]), 1)
|
|
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
|
|
|
|
|
|
class XPlatformIndexingTests(unittest.TestCase):
|
|
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
|
|
manifest_path.write_text("{}", encoding="utf-8")
|
|
|
|
payload = MODULE.load_xplatform_index_payload(
|
|
{},
|
|
[
|
|
{
|
|
"path": str(manifest_path),
|
|
"data": {
|
|
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"entries": [
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "工厂",
|
|
"place": "绿控传动",
|
|
"date": "20260330",
|
|
},
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132547_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2880",
|
|
"frameRate": 29.97,
|
|
"duration": 0.6,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
}
|
|
],
|
|
output_dir,
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["videoCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "绿控传动")
|
|
|
|
def test_load_index_payload_raises_when_source_manifest_summary_has_no_entries(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir) / "outputs"
|
|
output_dir.mkdir(parents=True)
|
|
manifest_path = pathlib.Path(temp_dir) / "source-manifest.json"
|
|
manifest_path.write_text("{}", encoding="utf-8")
|
|
|
|
with self.assertRaisesRegex(RuntimeError, "source-manifest"):
|
|
MODULE.load_xplatform_index_payload(
|
|
{},
|
|
[
|
|
{
|
|
"name": "source-manifest.json",
|
|
"path": str(manifest_path),
|
|
"data": {
|
|
"sourcePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"entriesCount": 59,
|
|
"dataRef": {"payloadPath": str(manifest_path)},
|
|
},
|
|
}
|
|
],
|
|
output_dir,
|
|
)
|
|
|
|
def test_manifest_entries_support_video_metadata_alias(self):
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "工厂",
|
|
"place": "绿控传动",
|
|
"date": "20260330",
|
|
},
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/video_metadata.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132547_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2880",
|
|
"frameRate": 29.97,
|
|
"duration": 0.6,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005_OsmoNano_2026-03-30_16-15-11/DJI_001/DJI_20260327132547_0003_D.MP4",
|
|
},
|
|
},
|
|
],
|
|
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
def test_manifest_entries_load_local_meta_when_inline_data_is_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "algo" / "X-platform-test"
|
|
collection = root / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
|
|
video_dir = collection / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
meta_path = collection / "meta.json"
|
|
video_meta_path = collection / "video_meta.json"
|
|
video_path = video_dir / "DJI_0001_D.mp4"
|
|
meta_path.write_text('{"type":"家庭","place":"蓟门里小区","date":"20260319"}', encoding="utf-8")
|
|
video_meta_path.write_text(
|
|
'[{"filename":"DJI_0001_D.mp4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
|
|
encoding="utf-8",
|
|
)
|
|
video_path.write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
|
|
"sizeBytes": meta_path.stat().st_size,
|
|
"mimeType": "application/json",
|
|
"source": {"path": str(meta_path)},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
|
|
"sizeBytes": video_meta_path.stat().st_size,
|
|
"mimeType": "application/json",
|
|
"source": {"path": str(video_meta_path)},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": video_path.stat().st_size,
|
|
"mimeType": "video/mp4",
|
|
"source": {"path": str(video_path)},
|
|
},
|
|
],
|
|
str(root),
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
|
|
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
def test_manifest_entries_support_root_level_meta_and_video_meta(self):
|
|
payload = MODULE.scan_xplatform_manifest_entries(
|
|
[
|
|
{
|
|
"path": "meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {
|
|
"type": "家庭",
|
|
"place": "蓟门里小区",
|
|
"date": "20260319",
|
|
},
|
|
},
|
|
{
|
|
"path": "video_meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"duration": 41.8,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"sizeBytes": 123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/video.mp4",
|
|
},
|
|
},
|
|
],
|
|
"bos://datastorage/algo/X-platform-test/raw/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(collection["relativePath"], "")
|
|
self.assertEqual(collection["videoCount"], 1)
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertEqual(collection["meta"]["place"], "蓟门里小区")
|
|
|
|
item = payload["items"][0]
|
|
self.assertEqual(item["collectionName"], collection["name"])
|
|
self.assertEqual(item["collectionRelativePath"], "")
|
|
self.assertEqual(item["videoFolder"], "DJI_001")
|
|
self.assertEqual(item["codec"], "hevc")
|
|
self.assertEqual(item["meta"]["type"], "家庭")
|
|
|
|
def test_local_dataset_root_with_meta_is_treated_as_collection_root(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "BJ_001_0001_OsmoNano_2026-03-19_14-51-43"
|
|
video_dir = root / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
(root / "meta.json").write_text('{"type":"家庭","place":"蓟门里小区"}', encoding="utf-8")
|
|
(root / "video_meta.json").write_text(
|
|
'[{"filename":"DJI_20260318112834_0001_D.MP4","codec":"hevc","resolution":"3840x2160","frameRate":29.97,"duration":41.8}]',
|
|
encoding="utf-8",
|
|
)
|
|
(video_dir / "DJI_20260318112834_0001_D.MP4").write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_dataset(root)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["name"], root.name)
|
|
self.assertEqual(collection["relativePath"], "")
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
item = payload["items"][0]
|
|
self.assertEqual(item["collectionName"], root.name)
|
|
self.assertEqual(item["collectionRelativePath"], "")
|
|
self.assertEqual(item["codec"], "hevc")
|
|
|
|
def test_local_dataset_root_supports_video_metadata_alias(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir) / "SZ_000_0005_OsmoNano_2026-03-30_16-15-11"
|
|
video_dir = root / "DJI_001"
|
|
video_dir.mkdir(parents=True)
|
|
(root / "meta.json").write_text('{"type":"工厂","place":"绿控传动"}', encoding="utf-8")
|
|
(root / "video_metadata.json").write_text(
|
|
'[{"filename":"DJI_20260327132547_0003_D.MP4","codec":"hevc","resolution":"3840x2880","frameRate":29.97,"duration":0.6}]',
|
|
encoding="utf-8",
|
|
)
|
|
(video_dir / "DJI_20260327132547_0003_D.MP4").write_bytes(b"fake-video")
|
|
|
|
payload = MODULE.scan_xplatform_dataset(root)
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
collection = payload["collections"][0]
|
|
self.assertEqual(collection["missingRequiredFiles"], [])
|
|
self.assertTrue(collection["hasVideoMeta"])
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
|
|
|
|
class SourceOperatorTests(unittest.TestCase):
|
|
def test_builtin_source_defaults_to_inline_when_source_json_is_present(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-默认内联",
|
|
"hyperparams": [
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-inline-default", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["dataSource"], "INLINE")
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
|
|
|
|
def test_builtin_source_prefers_source_json_over_bos_env(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-JSON",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_JSON", "value": json.dumps({"items": [{"id": 1, "kind": "synthetic"}]}, ensure_ascii=False)},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-json", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["value"]["items"][0]["kind"], "synthetic")
|
|
|
|
def test_builtin_source_prefers_source_text_over_bos_env(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "真实数据源-文本",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_TEXT", "value": "alpha\nbeta\ngamma"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
],
|
|
}
|
|
with mock.patch.object(MODULE, "list_bos_entries", side_effect=AssertionError("should not hit BOS")):
|
|
artifacts = MODULE.builtin_source("run-test", "source-text", output_dir, node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
payload = json.loads(pathlib.Path(manifest["path"]).read_text(encoding="utf-8"))
|
|
self.assertEqual(payload["listedCount"], 1)
|
|
self.assertEqual(payload["entries"][0]["text"], "alpha\nbeta\ngamma")
|
|
|
|
def test_directory_incremental_source_filter_emits_full_new_directory(self):
|
|
entries = [
|
|
{"path": "batch-a/meta.json", "sizeBytes": 10, "sha256": "sha-a-meta"},
|
|
{"path": "batch-a/DJI_001/clip-a.mp4", "sizeBytes": 20, "sha256": "sha-a-clip"},
|
|
{"path": "batch-b/meta.json", "sizeBytes": 30, "sha256": "sha-b-meta"},
|
|
{"path": "batch-b/DJI_001/clip-b.mp4", "sizeBytes": 40, "sha256": "sha-b-clip"},
|
|
]
|
|
emitted, checkpoint = MODULE.apply_incremental_source_filter(
|
|
entries,
|
|
"BOS",
|
|
"wjzh-data-migration",
|
|
{
|
|
"SOURCE_STATE_JSON": json.dumps(
|
|
{
|
|
"seenFingerprints": ["batch-a"],
|
|
"incrementalUnit": "DIRECTORY",
|
|
"directoryDepth": 1,
|
|
}
|
|
),
|
|
"SOURCE_FINGERPRINT": "fp-directory",
|
|
"INCREMENTAL_UNIT": "DIRECTORY",
|
|
"INCREMENTAL_DIRECTORY_DEPTH": "1",
|
|
},
|
|
"bos://wjzh-data-migration/wjzh-0330-Data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
)
|
|
|
|
self.assertEqual([item["path"] for item in emitted], ["batch-b/meta.json", "batch-b/DJI_001/clip-b.mp4"])
|
|
self.assertEqual([item["incrementalKey"] for item in emitted], ["batch-b", "batch-b"])
|
|
self.assertEqual(checkpoint["incrementalUnit"], "DIRECTORY")
|
|
self.assertEqual(checkpoint["directoryDepth"], 1)
|
|
self.assertEqual(checkpoint["newCount"], 2)
|
|
self.assertEqual(sorted(checkpoint["seenFingerprints"]), ["batch-a", "batch-b"])
|
|
|
|
def test_list_bos_entries_skips_large_non_metadata_json_payloads(self):
|
|
object_meta = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/meta.json", "size": 128, "e_tag": "meta"})()
|
|
object_imu = type("ObjectMeta", (), {"key": "wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json", "size": MODULE.PREVIEW_INLINE_BINARY_LIMIT + 1, "e_tag": "imu"})()
|
|
fake_client = _FakeBosListClient(
|
|
[object_meta, object_imu],
|
|
{
|
|
"wjzh-0330-Data-migration/SZ_000_0005/meta.json": b'{"type":"\xe5\xb7\xa5\xe5\x8e\x82"}',
|
|
"wjzh-0330-Data-migration/SZ_000_0005/DJI_001/DJI_20260327132547_0003_D_imu.json": b'{"data_points":[1,2,3]}',
|
|
},
|
|
)
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
entries = MODULE.list_bos_entries(
|
|
"wjzh-data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
"https://bj.bcebos.com",
|
|
"ak",
|
|
"sk",
|
|
)
|
|
|
|
self.assertEqual(len(entries), 2)
|
|
meta_entry = next(item for item in entries if item["path"].endswith("meta.json"))
|
|
imu_entry = next(item for item in entries if item["path"].endswith("_imu.json"))
|
|
self.assertIn("data", meta_entry)
|
|
self.assertNotIn("data", imu_entry)
|
|
self.assertEqual(
|
|
fake_client.downloaded_payload_keys,
|
|
[("wjzh-data-migration", "wjzh-0330-Data-migration/SZ_000_0005/meta.json")],
|
|
)
|
|
|
|
def test_bos_source_parses_bucket_and_prefix_from_source_path(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "BOS 增量扫描",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://wjzh-data-migration/wjzh-0330-Data-migration"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
{"key": "SCAN_MODE", "value": "FULL"},
|
|
],
|
|
}
|
|
fake_entries = [
|
|
{
|
|
"path": "SZ_000_0005/meta.json",
|
|
"storagePath": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json",
|
|
"sizeBytes": 10,
|
|
"mimeType": "application/json",
|
|
"data": {"type": "工厂"},
|
|
"source": {"path": "bos://wjzh-data-migration/wjzh-0330-Data-migration/SZ_000_0005/meta.json"},
|
|
}
|
|
]
|
|
with mock.patch.object(MODULE, "list_bos_entries", return_value=fake_entries) as list_bos_entries, mock.patch.object(
|
|
MODULE,
|
|
"source_entry_artifact",
|
|
return_value={
|
|
"id": "artifact-1",
|
|
"runId": "run-1",
|
|
"nodeId": "source",
|
|
"name": "meta.json",
|
|
"type": "json",
|
|
"path": str(output_dir / "meta.json"),
|
|
},
|
|
):
|
|
artifacts = MODULE.builtin_source("run-1", "source", output_dir, node_data, [])
|
|
|
|
self.assertGreaterEqual(len(artifacts), 1)
|
|
list_bos_entries.assert_called_once_with(
|
|
"wjzh-data-migration",
|
|
"wjzh-0330-Data-migration",
|
|
"",
|
|
"https://bj.bcebos.com",
|
|
"ak",
|
|
"sk",
|
|
)
|
|
|
|
|
|
class QualityMetricHelperTests(unittest.TestCase):
|
|
def test_build_skin_mask_detects_skin_tone_patch(self):
|
|
_, np = MODULE.require_cv2()
|
|
frame = np.zeros((80, 80, 3), dtype=np.uint8)
|
|
frame[20:60, 20:60] = (90, 130, 190)
|
|
mask = MODULE.build_skin_mask(frame)
|
|
self.assertGreater(int(mask.sum()), 0)
|
|
|
|
def test_build_skin_mask_rejects_neutral_background(self):
|
|
_, np = MODULE.require_cv2()
|
|
frame = np.full((80, 80, 3), 128, dtype=np.uint8)
|
|
mask = MODULE.build_skin_mask(frame)
|
|
self.assertEqual(int(mask.sum()), 0)
|
|
|
|
def test_motion_jitter_ratio_stays_low_for_smooth_motion(self):
|
|
smooth = [(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)]
|
|
jitter = MODULE.motion_jitter_ratio(smooth, 100.0)
|
|
self.assertLess(jitter, 0.01)
|
|
|
|
def test_motion_jitter_ratio_increases_for_erratic_motion(self):
|
|
smooth = MODULE.motion_jitter_ratio([(4.0, 0.0), (4.2, 0.1), (3.9, -0.1), (4.1, 0.0)], 100.0)
|
|
erratic = MODULE.motion_jitter_ratio([(0.0, 0.0), (12.0, -4.0), (-10.0, 8.0), (15.0, -6.0)], 100.0)
|
|
self.assertGreater(erratic, smooth)
|
|
self.assertGreater(erratic, 0.05)
|
|
|
|
def test_largest_skin_component_ratio_ignores_small_noise(self):
|
|
_, np = MODULE.require_cv2()
|
|
mask = np.zeros((100, 100), dtype=np.uint8)
|
|
mask[10:13, 10:13] = 255
|
|
ratio = MODULE.largest_skin_component_ratio(mask)
|
|
self.assertEqual(ratio, 0.0)
|
|
|
|
def test_largest_skin_component_ratio_returns_large_component_area(self):
|
|
_, np = MODULE.require_cv2()
|
|
mask = np.zeros((100, 100), dtype=np.uint8)
|
|
mask[20:50, 30:70] = 255
|
|
ratio = MODULE.largest_skin_component_ratio(mask)
|
|
self.assertGreater(ratio, 0.1)
|
|
self.assertLess(ratio, 0.13)
|
|
|
|
|
|
class QualityOperatorTests(unittest.TestCase):
|
|
def test_video_quality_accepts_zero_hand_occlusion(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "视频质检",
|
|
"hyperparams": [
|
|
{"key": "MIN_CLARITY", "value": "0.8"},
|
|
{"key": "MAX_SHAKE", "value": "0.1"},
|
|
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"filename": "demo.mp4",
|
|
"source": {"path": "/tmp/demo.mp4"},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=pathlib.Path("/tmp/demo.mp4")), mock.patch.object(
|
|
MODULE,
|
|
"analyze_video_quality_metrics",
|
|
return_value={
|
|
"clarity": 1.0,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"qualityScore": 1.0,
|
|
},
|
|
):
|
|
artifacts = MODULE.builtin_video_quality("run-1", "quality", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["passedCount"], 1)
|
|
self.assertTrue(payload["items"][0]["quality"]["passed"])
|
|
|
|
def test_dataset_filter_keeps_zero_shake_and_hand_occlusion(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
node_data = {
|
|
"label": "数据过滤",
|
|
"hyperparams": [
|
|
{"key": "DATE_START", "value": "2026-03-01"},
|
|
{"key": "DATE_END", "value": "2026-03-31"},
|
|
{"key": "PLACE", "value": "蓟门里小区"},
|
|
{"key": "TYPE", "value": "家庭"},
|
|
{"key": "MIN_CLARITY", "value": "0.8"},
|
|
{"key": "MAX_SHAKE", "value": "0.1"},
|
|
{"key": "MAX_HAND_OCCLUSION", "value": "0.1"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"meta": {
|
|
"date": "20260319",
|
|
"place": "蓟门里小区",
|
|
"type": "家庭",
|
|
},
|
|
"quality": {
|
|
"clarity": 0.95,
|
|
"shake": 0.0,
|
|
"handOcclusion": 0.0,
|
|
"passed": True,
|
|
},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
artifacts = MODULE.builtin_dataset_filter("run-1", "filter", output_dir, node_data, inputs)
|
|
|
|
payload = MODULE.load_artifact_payload(artifacts[0])
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(len(payload["items"]), 1)
|
|
|
|
|
|
class UploadOperatorTests(unittest.TestCase):
|
|
def test_build_artifact_skips_sha256_for_large_files(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
clip_path = root / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
|
|
MODULE,
|
|
"sha256_file",
|
|
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
|
|
):
|
|
artifact = MODULE.build_artifact("run-1", "slice", clip_path, "video")
|
|
|
|
metadata = artifact.get("metadata") or {}
|
|
self.assertEqual(metadata.get("sha256", ""), "")
|
|
self.assertTrue(metadata.get("sha256Skipped"))
|
|
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
|
|
|
|
def test_uploads_nested_package_outputs_to_bos(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
package_root = root / "package"
|
|
package_dir = package_root / "packages" / "clip001_package"
|
|
package_dir.mkdir(parents=True)
|
|
|
|
manifest_path = package_root / "packages-manifest.json"
|
|
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
|
|
clip_path = package_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
annotation_path = package_dir / "clip001_annotation.json"
|
|
annotation_path.write_text('{"label":"walking"}', encoding="utf-8")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [
|
|
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
|
|
MODULE.build_artifact("run-1", "package", clip_path, "video"),
|
|
MODULE.build_artifact("run-1", "package", annotation_path, "json", data={"label": "walking"}),
|
|
]
|
|
fake_client = _FakeBosClient()
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
|
|
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
|
|
uploaded_names = {item["name"] for item in artifacts}
|
|
uploaded_keys = {item["key"] for item in fake_client.uploads}
|
|
|
|
self.assertEqual(len(fake_client.uploads), 3)
|
|
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
|
|
self.assertEqual(uploaded_summary_payload["uploadedCount"], 3)
|
|
self.assertEqual(uploaded_summary_payload["bucket"], "datastorage")
|
|
self.assertEqual(uploaded_summary_payload["prefix"], "factory/output/demo")
|
|
|
|
def test_upload_skips_sha256_for_large_remote_artifacts(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
clip_path = root / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [MODULE.build_artifact("run-1", "slice", clip_path, "video")]
|
|
fake_client = _FakeBosClient()
|
|
|
|
with mock.patch.object(MODULE, "MAX_SHA256_FILE_BYTES", 1), mock.patch.object(
|
|
MODULE,
|
|
"sha256_file",
|
|
side_effect=AssertionError("sha256_file should not be called for files above size limit"),
|
|
), mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
remote_video = next(item for item in artifacts if item.get("type") == "video" and str(item.get("path", "")).startswith("bos://"))
|
|
metadata = remote_video.get("metadata") or {}
|
|
self.assertEqual(metadata.get("sha256", ""), "")
|
|
self.assertTrue(metadata.get("sha256Skipped"))
|
|
self.assertEqual(metadata.get("sha256SkipReason"), "size-limit")
|
|
|
|
|
|
class _FakeInsertManyResult:
|
|
def __init__(self, inserted_ids):
|
|
self.inserted_ids = inserted_ids
|
|
|
|
|
|
class _FakeUpdateResult:
|
|
def __init__(self, matched_count=0, modified_count=0, upserted_id=None):
|
|
self.matched_count = matched_count
|
|
self.modified_count = modified_count
|
|
self.upserted_id = upserted_id
|
|
|
|
|
|
class _FakeMongoCollection:
|
|
def __init__(self):
|
|
self.insert_many_calls = []
|
|
self.replace_one_calls = []
|
|
|
|
def insert_many(self, documents, ordered=False):
|
|
self.insert_many_calls.append({"documents": documents, "ordered": ordered})
|
|
return _FakeInsertManyResult([f"id-{index + 1}" for index in range(len(documents))])
|
|
|
|
def replace_one(self, filter_document, replacement, upsert=False):
|
|
self.replace_one_calls.append(
|
|
{
|
|
"filter": filter_document,
|
|
"replacement": replacement,
|
|
"upsert": upsert,
|
|
}
|
|
)
|
|
return _FakeUpdateResult(matched_count=1, modified_count=1, upserted_id=None)
|
|
|
|
|
|
class _FakeMongoDatabase:
|
|
def __init__(self, collection):
|
|
self._collection = collection
|
|
self.requested_collections = []
|
|
|
|
def __getitem__(self, name):
|
|
self.requested_collections.append(name)
|
|
return self._collection
|
|
|
|
|
|
class _FakeMongoClient:
|
|
def __init__(self, database):
|
|
self._database = database
|
|
self.requested_databases = []
|
|
self.closed = False
|
|
|
|
def __getitem__(self, name):
|
|
self.requested_databases.append(name)
|
|
return self._database
|
|
|
|
def close(self):
|
|
self.closed = True
|
|
|
|
|
|
class MongoWriteOperatorTests(unittest.TestCase):
|
|
def test_inserts_documents_into_default_business_database(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
|
|
{"key": "MONGO_MODE", "value": "insert_many"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{"clipId": "clip-001", "qualityScore": 0.91},
|
|
{"clipId": "clip-002", "qualityScore": 0.87},
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-1", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(database.requested_collections, ["qualityReports"])
|
|
self.assertEqual(client.requested_databases, ["business-data"])
|
|
self.assertEqual(len(collection.insert_many_calls), 1)
|
|
self.assertEqual(collection.insert_many_calls[0]["documents"][0]["clipId"], "clip-001")
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["collection"], "qualityReports")
|
|
self.assertEqual(summary["data"]["mode"], "insert_many")
|
|
self.assertEqual(summary["data"]["target"], "business")
|
|
self.assertEqual(summary["data"]["insertedCount"], 2)
|
|
self.assertTrue(client.closed)
|
|
|
|
def test_upserts_documents_by_key(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "annotations"},
|
|
{"key": "MONGO_MODE", "value": "upsert"},
|
|
{"key": "UPSERT_KEY", "value": "clipId"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{"data": {"clipId": "clip-001", "label": "walking"}},
|
|
{"data": {"clipId": "clip-002", "label": "standing"}},
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-2", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(len(collection.replace_one_calls), 2)
|
|
self.assertEqual(collection.replace_one_calls[0]["filter"], {"clipId": "clip-001"})
|
|
self.assertTrue(collection.replace_one_calls[0]["upsert"])
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["mode"], "upsert")
|
|
self.assertEqual(summary["data"]["upsertKey"], "clipId")
|
|
self.assertEqual(summary["data"]["target"], "business")
|
|
self.assertEqual(summary["data"]["upsertedCount"], 2)
|
|
|
|
def test_unwraps_single_value_dict_documents_for_business_writes(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "businessWriteValidation"},
|
|
{"key": "MONGO_MODE", "value": "insert_many"},
|
|
],
|
|
}
|
|
inputs = [
|
|
{
|
|
"data": {
|
|
"items": [
|
|
{
|
|
"value": {
|
|
"runLabel": "business-mongo-test",
|
|
"source": "workflow",
|
|
"createdAt": "2026-04-14T00:00:00Z",
|
|
"value": 1,
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"BUSINESS_MONGO_URI": "mongodb://business-mongo:27017", "BUSINESS_MONGO_DB": "business-data"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
MODULE.builtin_mongodb_write("run-unwrap", "mongo-node", pathlib.Path(temp_dir), node_data, inputs)
|
|
|
|
self.assertEqual(len(collection.insert_many_calls), 1)
|
|
inserted = collection.insert_many_calls[0]["documents"][0]
|
|
self.assertEqual(inserted["runLabel"], "business-mongo-test")
|
|
self.assertEqual(inserted["source"], "workflow")
|
|
self.assertEqual(inserted["value"], 1)
|
|
self.assertNotEqual(set(inserted.keys()), {"value"})
|
|
|
|
def test_can_target_platform_database_explicitly(self):
|
|
collection = _FakeMongoCollection()
|
|
database = _FakeMongoDatabase(collection)
|
|
client = _FakeMongoClient(database)
|
|
node_data = {
|
|
"label": "MongoDB 写入",
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "MONGO_COLLECTION", "value": "qualityReports"},
|
|
{"key": "MONGO_TARGET", "value": "platform"},
|
|
],
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
with mock.patch.dict(MODULE.os.environ, {"PLATFORM_MONGO_URI": "mongodb://platform-mongo:27017", "PLATFORM_MONGO_DB": "droboticflow"}, clear=False):
|
|
with mock.patch.object(MODULE, "build_mongo_client", return_value=client):
|
|
artifacts = MODULE.builtin_mongodb_write("run-3", "mongo-node", pathlib.Path(temp_dir), node_data, [{"data": {"clipId": "clip-001"}}])
|
|
|
|
self.assertEqual(client.requested_databases, ["droboticflow"])
|
|
summary = next(item for item in artifacts if item["name"] == "mongodb-write-result.json")
|
|
self.assertEqual(summary["data"]["target"], "platform")
|
|
|
|
def test_rebuilds_package_files_from_manifest_when_temp_paths_are_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
package_manifest = {
|
|
"packageCount": 1,
|
|
"items": [
|
|
{
|
|
"packageName": "clip001_package",
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
"quality": {"clarity": 0.9, "passed": True},
|
|
"annotation": {"objects": ["人"], "minConfidence": 0.92},
|
|
}
|
|
],
|
|
}
|
|
inputs = [
|
|
{"name": "packages-manifest.json", "type": "json", "data": package_manifest},
|
|
{
|
|
"name": "clip001.mp4",
|
|
"type": "video",
|
|
"path": "/tmp/non-existent-clip001.mp4",
|
|
"data": {"preview": {"kind": "video", "format": "mp4"}},
|
|
},
|
|
{
|
|
"name": "clip001_annotation.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_annotation.json",
|
|
"data": {"objects": ["人"], "minConfidence": 0.92},
|
|
},
|
|
{
|
|
"name": "clip001_quality.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_quality.json",
|
|
"data": {"clarity": 0.9, "passed": True},
|
|
},
|
|
{
|
|
"name": "clip001_config.json",
|
|
"type": "json",
|
|
"path": "/tmp/non-existent-clip001_config.json",
|
|
"data": {"collectionId": "collection-1"},
|
|
},
|
|
]
|
|
fake_client = _FakeBosClient()
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(fake_client, "https")), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
uploaded_keys = {item["key"] for item in fake_client.uploads}
|
|
self.assertIn("factory/output/demo/packages-manifest.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001.mp4", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_annotation.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_quality.json", uploaded_keys)
|
|
self.assertIn("factory/output/demo/packages/clip001_package/clip001_config.json", uploaded_keys)
|
|
self.assertFalse(
|
|
(output_dir / "uploaded" / "_materialized_upload_inputs" / "packages" / "clip001_package" / "clip001.mp4").exists()
|
|
)
|
|
uploaded_summary = next(item for item in artifacts if item["name"] == "uploaded-results.json")
|
|
uploaded_summary_payload = MODULE.load_artifact_payload(uploaded_summary)
|
|
self.assertEqual(uploaded_summary_payload["uploadedCount"], 5)
|
|
|
|
def test_upload_artifacts_are_json_serializable_when_preview_url_is_bytes(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
package_root = root / "package"
|
|
package_dir = package_root / "packages" / "clip001_package"
|
|
package_dir.mkdir(parents=True)
|
|
|
|
manifest_path = package_root / "packages-manifest.json"
|
|
manifest_path.write_text('{"packageCount":1}', encoding="utf-8")
|
|
clip_path = package_dir / "clip001.mp4"
|
|
clip_path.write_bytes(b"video-bytes")
|
|
|
|
output_dir = root / "upload"
|
|
node_data = {
|
|
"label": "结果上传",
|
|
"bucket": "datastorage",
|
|
"prefix": "factory/output/demo",
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/factory/output/demo"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak-demo"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk-demo"},
|
|
],
|
|
}
|
|
inputs = [
|
|
MODULE.build_artifact("run-1", "package", manifest_path, "json", data={"packageCount": 1}),
|
|
MODULE.build_artifact("run-1", "package", clip_path, "video"),
|
|
]
|
|
|
|
with mock.patch.object(MODULE, "build_bos_client", return_value=(_FakeBosClientBytesURL(), "https")):
|
|
artifacts = MODULE.builtin_upload("run-1", "upload", output_dir, node_data, inputs)
|
|
|
|
json.dumps({"artifacts": artifacts}, ensure_ascii=False)
|
|
|
|
|
|
class XPlatformClipHandoffTests(unittest.TestCase):
|
|
def test_video_slicing_manifest_preserves_source_reference(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 41.8,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(MODULE, "write_video_segment", side_effect=fake_write_video_segment), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
clip = manifest_payload["items"][0]
|
|
self.assertEqual(clip["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
|
|
self.assertEqual(clip["clipName"], "DJI_20260318112834_0001_D_clip001.mp4")
|
|
|
|
def test_video_slicing_uses_single_pass_writer_for_multi_clip_video(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 365.0,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_write_video_segments(_source_path, segments):
|
|
outputs = []
|
|
for segment in segments:
|
|
destination = pathlib.Path(segment["destination"])
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(f"clip-{segment['clipIndex']}".encode("utf-8"))
|
|
outputs.append(
|
|
{
|
|
**segment,
|
|
"duration": round(float(segment["endSecond"]) - float(segment["startSecond"]), 3),
|
|
"frameRate": 29.97,
|
|
"resolution": "3840x2160",
|
|
}
|
|
)
|
|
return outputs
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=False), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segments",
|
|
side_effect=fake_write_video_segments,
|
|
) as multi_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
) as single_writer, mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["clipCount"], 2)
|
|
self.assertEqual([item["clipName"] for item in manifest_payload["items"]], [
|
|
"DJI_20260318112834_0001_D_clip001.mp4",
|
|
"DJI_20260318112834_0001_D_clip002.mp4",
|
|
])
|
|
multi_writer.assert_called_once()
|
|
single_writer.assert_not_called()
|
|
|
|
def test_write_video_segment_with_ffmpeg_prefers_stream_copy_and_uses_inspected_metrics(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
destination = root / "clips" / "clip001.mp4"
|
|
|
|
def fake_run(args, stdout=None, stderr=None, text=None):
|
|
self.assertEqual(args[0], "ffmpeg")
|
|
self.assertIn(str(source_path), args)
|
|
self.assertEqual(args[-1], str(destination))
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-video")
|
|
return mock.Mock(returncode=0, stdout="", stderr="")
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE.subprocess,
|
|
"run",
|
|
side_effect=fake_run,
|
|
) as run_mock, mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
|
|
):
|
|
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
|
|
|
|
self.assertEqual(metrics["duration"], 21.5)
|
|
self.assertEqual(metrics["frameRate"], 29.97)
|
|
self.assertEqual(metrics["resolution"], "3840x2160")
|
|
run_args = run_mock.call_args.args[0]
|
|
self.assertIn("-ss", run_args)
|
|
self.assertIn("-t", run_args)
|
|
self.assertIn("-c", run_args)
|
|
self.assertIn("copy", run_args)
|
|
self.assertNotIn("-movflags", run_args)
|
|
self.assertNotIn("+faststart", run_args)
|
|
|
|
def test_write_video_segment_with_ffmpeg_falls_back_to_reencode_when_copy_fails(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = pathlib.Path(temp_dir)
|
|
source_path = root / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
destination = root / "clips" / "clip001.mp4"
|
|
|
|
calls = []
|
|
|
|
def fake_run(args, stdout=None, stderr=None, text=None):
|
|
calls.append(list(args))
|
|
if len(calls) == 1:
|
|
return mock.Mock(returncode=1, stdout="", stderr="copy failed")
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-video")
|
|
return mock.Mock(returncode=0, stdout="", stderr="")
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE.subprocess,
|
|
"run",
|
|
side_effect=fake_run,
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 21.5, "frameRate": 29.97, "frameCount": 644, "resolution": "3840x2160"},
|
|
):
|
|
metrics = MODULE.write_video_segment_with_ffmpeg(source_path, destination, 12.5, 34.0)
|
|
|
|
self.assertEqual(metrics["duration"], 21.5)
|
|
self.assertEqual(len(calls), 2)
|
|
self.assertIn("copy", calls[0])
|
|
self.assertIn("libx264", calls[1])
|
|
self.assertIn("aac", calls[1])
|
|
|
|
def test_video_slicing_prefers_ffmpeg_when_available(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "filtered-videos.json",
|
|
"type": "json",
|
|
"data": {
|
|
"count": 1,
|
|
"items": [
|
|
{
|
|
"filename": "DJI_20260318112834_0001_D.MP4",
|
|
"duration": 365.0,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"passed": True},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频切片",
|
|
"hyperparams": [{"key": "SEGMENT_DURATION_SECONDS", "value": "300"}],
|
|
}
|
|
|
|
def fake_ffmpeg_writer(_source_path, destination, start_second, end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(f"{start_second}-{end_second}".encode("utf-8"))
|
|
return {
|
|
"duration": round(float(end_second) - float(start_second), 3),
|
|
"frameRate": 29.97,
|
|
"resolution": "3840x2160",
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "ffmpeg_available", return_value=True), mock.patch.object(
|
|
MODULE, "source_path_from_item", return_value=source_path
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"inspect_video_file",
|
|
return_value={"duration": 365.0, "frameRate": 29.97, "resolution": "3840x2160"},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment_with_ffmpeg",
|
|
side_effect=fake_ffmpeg_writer,
|
|
) as ffmpeg_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segments",
|
|
) as multi_writer, mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
) as single_writer, mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_slicing("run-1", "slice", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "clips-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["clipCount"], 2)
|
|
self.assertEqual(ffmpeg_writer.call_count, 2)
|
|
multi_writer.assert_not_called()
|
|
single_writer.assert_not_called()
|
|
|
|
def test_video_annotation_reuses_shared_clip_path_without_copying_clip_artifact(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
shared_clip_path = output_dir / "shared" / "clip001.mp4"
|
|
shared_clip_path.parent.mkdir(parents=True, exist_ok=True)
|
|
shared_clip_path.write_bytes(b"clip-bytes")
|
|
inputs = [
|
|
{
|
|
"name": "clips-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"clipCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"clipPath": str(shared_clip_path),
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"duration": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "视频标注",
|
|
"hyperparams": [{"key": "FRAME_INTERVAL_SECONDS", "value": "1.0"}],
|
|
}
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
|
|
MODULE,
|
|
"annotate_video_clip",
|
|
return_value={
|
|
"duration": 41.8,
|
|
"actions": [{"label": "walking", "confidence": 0.92, "start": 0, "end": 41.8}],
|
|
"objects": ["人"],
|
|
"minConfidence": 0.92,
|
|
},
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_video_annotation("run-1", "annotate", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "annotations-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
self.assertEqual(manifest_payload["annotationCount"], 1)
|
|
self.assertEqual(manifest_payload["items"][0]["clipPath"], str(shared_clip_path))
|
|
self.assertFalse(any(item["name"] == "clip001.mp4" for item in artifacts))
|
|
|
|
def test_materialize_clip_from_source_prefers_remote_segment_generation(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
item = {
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
},
|
|
}
|
|
|
|
def fake_remote_segment(_source_url, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8}
|
|
|
|
with mock.patch.object(MODULE, "remote_video_url_for_source", return_value="https://signed.example.com/video.mp4"), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment_from_remote_url",
|
|
side_effect=fake_remote_segment,
|
|
) as remote_writer, mock.patch.object(MODULE, "source_path_from_item") as local_materializer:
|
|
resolved = MODULE.materialize_clip_from_source(item, output_dir)
|
|
|
|
self.assertIsNotNone(resolved)
|
|
self.assertTrue(resolved.exists())
|
|
self.assertEqual(resolved.read_bytes(), b"clip-bytes")
|
|
remote_writer.assert_called_once()
|
|
local_materializer.assert_not_called()
|
|
|
|
def test_result_package_rebuilds_clip_from_source_when_clip_artifact_is_missing(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
output_dir = pathlib.Path(temp_dir)
|
|
source_path = output_dir / "source.mp4"
|
|
source_path.write_bytes(b"source-video")
|
|
inputs = [
|
|
{
|
|
"name": "annotations-manifest.json",
|
|
"type": "json",
|
|
"data": {
|
|
"annotationCount": 1,
|
|
"items": [
|
|
{
|
|
"clipId": "clip001",
|
|
"clipName": "clip001.mp4",
|
|
"sourceVideo": "DJI_20260318112834_0001_D.MP4",
|
|
"startSecond": 0,
|
|
"endSecond": 41.8,
|
|
"duration": 41.8,
|
|
"collectionId": "collection-1",
|
|
"meta": {"type": "家庭"},
|
|
"quality": {"clarity": 0.9, "passed": True},
|
|
"annotation": {"actions": [], "objects": ["人"], "minConfidence": 0.92},
|
|
"source": {
|
|
"path": "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4",
|
|
"previewURL": "https://example.com/source.mp4",
|
|
},
|
|
}
|
|
],
|
|
},
|
|
}
|
|
]
|
|
node_data = {
|
|
"label": "成品打包",
|
|
"hyperparams": [],
|
|
}
|
|
|
|
def fake_write_video_segment(_source_path, destination, _start_second, _end_second):
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(b"clip-bytes")
|
|
return {"duration": 41.8, "frameRate": 29.97, "resolution": "3840x2160"}
|
|
|
|
with mock.patch.object(MODULE, "source_path_from_item", return_value=source_path), mock.patch.object(
|
|
MODULE,
|
|
"write_video_segment",
|
|
side_effect=fake_write_video_segment,
|
|
), mock.patch.object(
|
|
MODULE,
|
|
"extract_preview_data",
|
|
return_value={"preview": {"kind": "video", "format": "mp4"}},
|
|
):
|
|
artifacts = MODULE.builtin_result_package("run-1", "package", output_dir, node_data, inputs)
|
|
|
|
manifest = next(item for item in artifacts if item["name"] == "packages-manifest.json")
|
|
manifest_payload = MODULE.load_artifact_payload(manifest)
|
|
package_clip = next(item for item in artifacts if item["name"] == "clip001.mp4")
|
|
self.assertEqual(manifest_payload["packageCount"], 1)
|
|
manifest_item = manifest_payload["items"][0]
|
|
self.assertEqual(manifest_item["clipName"], "clip001.mp4")
|
|
self.assertEqual(manifest_item["collectionId"], "collection-1")
|
|
self.assertEqual(manifest_item["startSecond"], 0)
|
|
self.assertEqual(manifest_item["endSecond"], 41.8)
|
|
self.assertEqual(manifest_item["source"]["path"], "bos://datastorage/algo/X-platform-test/raw/BJ_001/DJI_001/DJI_20260318112834_0001_D.MP4")
|
|
self.assertEqual(package_clip["name"], "clip001.mp4")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|