forked from longtao.wu/droboticflow
344 lines
16 KiB
Python
344 lines
16 KiB
Python
import importlib.util
|
|
import json
|
|
import pathlib
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
|
|
def load_operator_runner_module():
|
|
module_path = pathlib.Path(__file__).resolve().parents[2] / "backend" / "operator-runner" / "main.py"
|
|
spec = importlib.util.spec_from_file_location("operator_runner_main", module_path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
assert spec.loader is not None
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
class OperatorRunnerBOSTest(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.runner = load_operator_runner_module()
|
|
|
|
def test_xplatform_index_uses_remote_source_manifest_entries(self):
|
|
manifest = {
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test/",
|
|
"entries": [
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
|
|
"sizeBytes": 256,
|
|
"data": {
|
|
"type": "家庭",
|
|
"place": "蓟门里小区",
|
|
"mission": "无",
|
|
"project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
"date": "20260319",
|
|
"device_id": "C00000",
|
|
},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
|
|
"sizeBytes": 512,
|
|
"data": [
|
|
{
|
|
"filename": "DJI_0001_D.mp4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"bitrate": 56570718,
|
|
"duration": 41.8,
|
|
"startTime": 1773904218,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/intrinsics.json",
|
|
"sizeBytes": 128,
|
|
"data": {"fx": 510.2, "fy": 509.8},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": 2500000000,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"previewURL": "https://example.com/DJI_0001_D.mp4",
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
|
|
},
|
|
},
|
|
],
|
|
}
|
|
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["videoCount"], 1)
|
|
self.assertEqual(payload["collections"][0]["id"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(payload["collections"][0]["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
|
|
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
|
|
self.assertTrue(payload["collections"][0]["hasIntrinsics"])
|
|
self.assertEqual(len(payload["entries"]), 4)
|
|
self.assertEqual(payload["items"][0]["collectionId"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(payload["items"][0]["collectionName"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(payload["items"][0]["codec"], "hevc")
|
|
self.assertEqual(payload["items"][0]["resolution"], "3840x2160")
|
|
self.assertEqual(payload["items"][0]["duration"], 41.8)
|
|
self.assertEqual(payload["items"][0]["source"]["previewURL"], "https://example.com/DJI_0001_D.mp4")
|
|
meta_entry = next(item for item in payload["entries"] if item["relativePath"].endswith("meta.json"))
|
|
self.assertEqual(meta_entry["collectionId"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(meta_entry["data"]["place"], "蓟门里小区")
|
|
video_entry = next(item for item in payload["entries"] if item["relativePath"].endswith("DJI_0001_D.mp4"))
|
|
self.assertEqual(video_entry["source"]["previewURL"], "https://example.com/DJI_0001_D.mp4")
|
|
|
|
def test_xplatform_index_keeps_root_level_orbbec_videos_without_xplatform_metadata(self):
|
|
manifest = {
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700102_061040",
|
|
"entries": [
|
|
{
|
|
"path": "Orbbec_Ego_AZER76400GG_19700102_061040_camera_left_part0001.mp4",
|
|
"sizeBytes": 65734123,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700102_061040/Orbbec_Ego_AZER76400GG_19700102_061040_camera_left_part0001.mp4",
|
|
"previewURL": "https://example.com/left.mp4",
|
|
},
|
|
},
|
|
{
|
|
"path": "Orbbec_Ego_AZER76400GG_19700102_061040_camera_right_part0001.mp4",
|
|
"sizeBytes": 65830435,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"path": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700102_061040/Orbbec_Ego_AZER76400GG_19700102_061040_camera_right_part0001.mp4",
|
|
"previewURL": "https://example.com/right.mp4",
|
|
},
|
|
},
|
|
{
|
|
"path": "Orbbec_Ego_AZER76400GG_19700102_061040_imu.csv",
|
|
"sizeBytes": 2829917,
|
|
"mimeType": "text/csv",
|
|
},
|
|
],
|
|
}
|
|
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
|
|
|
|
self.assertEqual(payload["collectionCount"], 1)
|
|
self.assertEqual(payload["videoCount"], 2)
|
|
self.assertEqual(payload["collections"][0]["relativePath"], "")
|
|
self.assertEqual(payload["collections"][0]["name"], "Orbbec_Ego_AZER76400GG_19700102_061040")
|
|
self.assertEqual(len(payload["entries"]), 3)
|
|
items_by_name = {item["filename"]: item for item in payload["items"]}
|
|
right = items_by_name["Orbbec_Ego_AZER76400GG_19700102_061040_camera_right_part0001.mp4"]
|
|
self.assertEqual(right["sizeBytes"], 65830435)
|
|
self.assertEqual(right["source"]["path"], manifest["entries"][1]["source"]["path"])
|
|
self.assertEqual(right["source"]["previewURL"], "https://example.com/right.mp4")
|
|
|
|
def test_xplatform_index_detects_nested_collection_roots_from_meta(self):
|
|
manifest = {
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test/",
|
|
"entries": [
|
|
{
|
|
"path": "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
|
|
"data": {
|
|
"type": "家庭",
|
|
"place": "蓟门里小区",
|
|
"project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
},
|
|
},
|
|
{
|
|
"path": "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
|
|
"data": [
|
|
{
|
|
"filename": "DJI_0001_D.mp4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"duration": 41.8,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": 2500000000,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"previewURL": "https://example.com/DJI_0001_D.mp4",
|
|
},
|
|
},
|
|
{
|
|
"path": "processed/test_pipeline/20260319/report.json",
|
|
"data": {"ok": True},
|
|
},
|
|
],
|
|
}
|
|
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
|
|
|
|
self.assertEqual(payload["collectionCount"], 2)
|
|
collections_by_id = {item["id"]: item for item in payload["collections"]}
|
|
self.assertEqual(
|
|
collections_by_id["raw-20260319-BJ_001_0001_OsmoNano_2026-03-19_14-51-43"]["name"],
|
|
"BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
)
|
|
self.assertEqual(
|
|
collections_by_id["raw-20260319-BJ_001_0001_OsmoNano_2026-03-19_14-51-43"]["relativePath"],
|
|
"raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
)
|
|
self.assertEqual(collections_by_id["processed"]["fileCount"], 1)
|
|
self.assertEqual(len(payload["entries"]), 4)
|
|
self.assertEqual(payload["items"][0]["collectionId"], "raw-20260319-BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(payload["items"][0]["collectionName"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
|
|
self.assertEqual(payload["items"][0]["relativePath"], "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4")
|
|
|
|
def test_xplatform_index_keeps_incremental_entries_separate_from_full_catalog(self):
|
|
manifest = {
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test/",
|
|
"scanMode": "INCREMENTAL",
|
|
"listedCount": 4,
|
|
"newCount": 0,
|
|
"entries": [],
|
|
"catalogEntries": [
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
|
|
"sizeBytes": 256,
|
|
"data": {
|
|
"project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
"date": "20260319",
|
|
},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
|
|
"sizeBytes": 512,
|
|
"data": [
|
|
{
|
|
"filename": "DJI_20260327132729_0003_D.MP4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"duration": 41.8,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260327132729_0003_D.MP4",
|
|
"sizeBytes": 16004000000,
|
|
"mimeType": "video/mp4",
|
|
"source": {
|
|
"previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4",
|
|
},
|
|
},
|
|
{
|
|
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260327132729_0003_D_imu.json",
|
|
"sizeBytes": 36080000,
|
|
"mimeType": "application/json",
|
|
},
|
|
],
|
|
}
|
|
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
|
|
|
|
self.assertEqual(payload["collectionCount"], 0)
|
|
self.assertEqual(payload["videoCount"], 0)
|
|
self.assertEqual(payload["items"], [])
|
|
self.assertEqual(payload["entries"], [])
|
|
self.assertEqual(payload["catalogCollectionCount"], 1)
|
|
self.assertEqual(payload["catalogVideoCount"], 1)
|
|
self.assertEqual(payload["catalogCollections"][0]["fileCount"], 4)
|
|
self.assertEqual(payload["catalogCollections"][0]["totalBytes"], 16040080768)
|
|
self.assertEqual(len(payload["catalogEntries"]), 4)
|
|
self.assertEqual(payload["dataSource"], "BOS")
|
|
self.assertEqual(payload["sourcePath"], "bos://datastorage/algo/X-platform-test/")
|
|
self.assertEqual(payload["newCount"], 0)
|
|
|
|
def test_clean_metadata_decodes_nested_bytes_values(self):
|
|
payload = self.runner.clean_metadata(
|
|
{
|
|
"path": "bos://datastorage/algo/X-platform-test/video.mp4",
|
|
"source": {
|
|
"previewURL": b"https://example.com/video.mp4",
|
|
"path": b"bos://datastorage/algo/X-platform-test/video.mp4",
|
|
},
|
|
}
|
|
)
|
|
self.assertEqual(payload["source"]["previewURL"], "https://example.com/video.mp4")
|
|
self.assertEqual(payload["source"]["path"], "bos://datastorage/algo/X-platform-test/video.mp4")
|
|
|
|
def test_builtin_source_incremental_mode_only_emits_new_entries_and_checkpoint(self):
|
|
listed_entries = [
|
|
{
|
|
"path": "BJ_001/DJI_0001_D.mp4",
|
|
"sizeBytes": 12,
|
|
"storagePath": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_0001_D.mp4",
|
|
"etag": "etag-1",
|
|
"lastModified": "2026-03-19T14:51:43Z",
|
|
"source": {"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_0001_D.mp4"},
|
|
},
|
|
{
|
|
"path": "BJ_001/meta.json",
|
|
"sizeBytes": 32,
|
|
"storagePath": "bos://datastorage/algo/X-platform-test/BJ_001/meta.json",
|
|
"etag": "etag-2",
|
|
"lastModified": "2026-03-19T14:51:44Z",
|
|
"data": {"project_id": "BJ_001"},
|
|
"source": {"path": "bos://datastorage/algo/X-platform-test/BJ_001/meta.json"},
|
|
},
|
|
]
|
|
seen_fingerprint = self.runner.source_entry_fingerprint("BOS", "datastorage", listed_entries[0])
|
|
node_data = {
|
|
"label": "BOS 增量扫描",
|
|
"outputs": [{"id": "out-1", "name": "manifest", "type": "json"}],
|
|
"hyperparams": [
|
|
{"key": "DATA_SOURCE", "value": "BOS"},
|
|
{"key": "SOURCE_PATH", "value": "bos://datastorage/algo/X-platform-test"},
|
|
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
|
|
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
|
|
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
|
|
{"key": "SCAN_MODE", "value": "INCREMENTAL"},
|
|
{
|
|
"key": "SOURCE_STATE_JSON",
|
|
"value": json.dumps(
|
|
{
|
|
"sourceFingerprint": "fp-1",
|
|
"seenFingerprints": [seen_fingerprint],
|
|
}
|
|
),
|
|
},
|
|
],
|
|
"bucket": "datastorage",
|
|
"prefix": "algo/X-platform-test",
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
with mock.patch.object(self.runner, "list_bos_entries", return_value=listed_entries):
|
|
artifacts = self.runner.builtin_source("run-1", "node-source", pathlib.Path(tempdir), node_data, [])
|
|
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
|
|
checkpoint = next(item for item in artifacts if item["name"] == "source-checkpoint.json")
|
|
manifest_payload = self.runner.load_artifact_payload(manifest)
|
|
checkpoint_payload = self.runner.load_artifact_payload(checkpoint)
|
|
emitted_paths = sorted(item["path"] for item in manifest_payload["entries"])
|
|
self.assertEqual(emitted_paths, ["BJ_001/meta.json"])
|
|
catalog_paths = sorted(item["path"] for item in manifest_payload["catalogEntries"])
|
|
self.assertEqual(catalog_paths, ["BJ_001/DJI_0001_D.mp4", "BJ_001/meta.json"])
|
|
self.assertEqual(manifest_payload["listedCount"], 2)
|
|
self.assertEqual(manifest_payload["newCount"], 1)
|
|
self.assertEqual(checkpoint_payload["listedCount"], 2)
|
|
self.assertEqual(checkpoint_payload["newCount"], 1)
|
|
self.assertEqual(len(checkpoint_payload["seenFingerprints"]), 2)
|
|
self.assertEqual(checkpoint.get("portId"), None)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|