droboticflow/tests/backend/operator_runner_bos_test.py
2026-08-18 01:46:53 +08:00

344 lines
16 KiB
Python

import importlib.util
import json
import pathlib
import tempfile
import unittest
from unittest import mock
def load_operator_runner_module():
module_path = pathlib.Path(__file__).resolve().parents[2] / "backend" / "operator-runner" / "main.py"
spec = importlib.util.spec_from_file_location("operator_runner_main", module_path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
class OperatorRunnerBOSTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.runner = load_operator_runner_module()
def test_xplatform_index_uses_remote_source_manifest_entries(self):
manifest = {
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test/",
"entries": [
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
"sizeBytes": 256,
"data": {
"type": "家庭",
"place": "蓟门里小区",
"mission": "无",
"project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
"date": "20260319",
"device_id": "C00000",
},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
"sizeBytes": 512,
"data": [
{
"filename": "DJI_0001_D.mp4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"bitrate": 56570718,
"duration": 41.8,
"startTime": 1773904218,
}
],
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/intrinsics.json",
"sizeBytes": 128,
"data": {"fx": 510.2, "fy": 509.8},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
"sizeBytes": 2500000000,
"mimeType": "video/mp4",
"source": {
"previewURL": "https://example.com/DJI_0001_D.mp4",
"path": "bos://datastorage/algo/X-platform-test/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
},
},
],
}
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
with tempfile.TemporaryDirectory() as tempdir:
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["videoCount"], 1)
self.assertEqual(payload["collections"][0]["id"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(payload["collections"][0]["name"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(payload["collections"][0]["meta"]["place"], "蓟门里小区")
self.assertTrue(payload["collections"][0]["hasVideoMeta"])
self.assertTrue(payload["collections"][0]["hasIntrinsics"])
self.assertEqual(len(payload["entries"]), 4)
self.assertEqual(payload["items"][0]["collectionId"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(payload["items"][0]["collectionName"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(payload["items"][0]["codec"], "hevc")
self.assertEqual(payload["items"][0]["resolution"], "3840x2160")
self.assertEqual(payload["items"][0]["duration"], 41.8)
self.assertEqual(payload["items"][0]["source"]["previewURL"], "https://example.com/DJI_0001_D.mp4")
meta_entry = next(item for item in payload["entries"] if item["relativePath"].endswith("meta.json"))
self.assertEqual(meta_entry["collectionId"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(meta_entry["data"]["place"], "蓟门里小区")
video_entry = next(item for item in payload["entries"] if item["relativePath"].endswith("DJI_0001_D.mp4"))
self.assertEqual(video_entry["source"]["previewURL"], "https://example.com/DJI_0001_D.mp4")
def test_xplatform_index_keeps_root_level_orbbec_videos_without_xplatform_metadata(self):
manifest = {
"dataSource": "BOS",
"sourcePath": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700102_061040",
"entries": [
{
"path": "Orbbec_Ego_AZER76400GG_19700102_061040_camera_left_part0001.mp4",
"sizeBytes": 65734123,
"mimeType": "video/mp4",
"source": {
"path": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700102_061040/Orbbec_Ego_AZER76400GG_19700102_061040_camera_left_part0001.mp4",
"previewURL": "https://example.com/left.mp4",
},
},
{
"path": "Orbbec_Ego_AZER76400GG_19700102_061040_camera_right_part0001.mp4",
"sizeBytes": 65830435,
"mimeType": "video/mp4",
"source": {
"path": "bos://testxyz/fanyu_test/Orbbec_Ego_AZER76400GG_19700102_061040/Orbbec_Ego_AZER76400GG_19700102_061040_camera_right_part0001.mp4",
"previewURL": "https://example.com/right.mp4",
},
},
{
"path": "Orbbec_Ego_AZER76400GG_19700102_061040_imu.csv",
"sizeBytes": 2829917,
"mimeType": "text/csv",
},
],
}
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
with tempfile.TemporaryDirectory() as tempdir:
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
self.assertEqual(payload["collectionCount"], 1)
self.assertEqual(payload["videoCount"], 2)
self.assertEqual(payload["collections"][0]["relativePath"], "")
self.assertEqual(payload["collections"][0]["name"], "Orbbec_Ego_AZER76400GG_19700102_061040")
self.assertEqual(len(payload["entries"]), 3)
items_by_name = {item["filename"]: item for item in payload["items"]}
right = items_by_name["Orbbec_Ego_AZER76400GG_19700102_061040_camera_right_part0001.mp4"]
self.assertEqual(right["sizeBytes"], 65830435)
self.assertEqual(right["source"]["path"], manifest["entries"][1]["source"]["path"])
self.assertEqual(right["source"]["previewURL"], "https://example.com/right.mp4")
def test_xplatform_index_detects_nested_collection_roots_from_meta(self):
manifest = {
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test/",
"entries": [
{
"path": "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
"data": {
"type": "家庭",
"place": "蓟门里小区",
"project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
},
},
{
"path": "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
"data": [
{
"filename": "DJI_0001_D.mp4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
}
],
},
{
"path": "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4",
"sizeBytes": 2500000000,
"mimeType": "video/mp4",
"source": {
"previewURL": "https://example.com/DJI_0001_D.mp4",
},
},
{
"path": "processed/test_pipeline/20260319/report.json",
"data": {"ok": True},
},
],
}
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
with tempfile.TemporaryDirectory() as tempdir:
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
self.assertEqual(payload["collectionCount"], 2)
collections_by_id = {item["id"]: item for item in payload["collections"]}
self.assertEqual(
collections_by_id["raw-20260319-BJ_001_0001_OsmoNano_2026-03-19_14-51-43"]["name"],
"BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
)
self.assertEqual(
collections_by_id["raw-20260319-BJ_001_0001_OsmoNano_2026-03-19_14-51-43"]["relativePath"],
"raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
)
self.assertEqual(collections_by_id["processed"]["fileCount"], 1)
self.assertEqual(len(payload["entries"]), 4)
self.assertEqual(payload["items"][0]["collectionId"], "raw-20260319-BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(payload["items"][0]["collectionName"], "BJ_001_0001_OsmoNano_2026-03-19_14-51-43")
self.assertEqual(payload["items"][0]["relativePath"], "raw/20260319/BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_0001_D.mp4")
def test_xplatform_index_keeps_incremental_entries_separate_from_full_catalog(self):
manifest = {
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test/",
"scanMode": "INCREMENTAL",
"listedCount": 4,
"newCount": 0,
"entries": [],
"catalogEntries": [
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/meta.json",
"sizeBytes": 256,
"data": {
"project_id": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
"date": "20260319",
},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/video_meta.json",
"sizeBytes": 512,
"data": [
{
"filename": "DJI_20260327132729_0003_D.MP4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
}
],
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260327132729_0003_D.MP4",
"sizeBytes": 16004000000,
"mimeType": "video/mp4",
"source": {
"previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4",
},
},
{
"path": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43/DJI_001/DJI_20260327132729_0003_D_imu.json",
"sizeBytes": 36080000,
"mimeType": "application/json",
},
],
}
inputs = [{"name": "source-manifest.json", "type": "json", "data": manifest}]
with tempfile.TemporaryDirectory() as tempdir:
payload = self.runner.load_xplatform_index_payload({}, inputs, pathlib.Path(tempdir))
self.assertEqual(payload["collectionCount"], 0)
self.assertEqual(payload["videoCount"], 0)
self.assertEqual(payload["items"], [])
self.assertEqual(payload["entries"], [])
self.assertEqual(payload["catalogCollectionCount"], 1)
self.assertEqual(payload["catalogVideoCount"], 1)
self.assertEqual(payload["catalogCollections"][0]["fileCount"], 4)
self.assertEqual(payload["catalogCollections"][0]["totalBytes"], 16040080768)
self.assertEqual(len(payload["catalogEntries"]), 4)
self.assertEqual(payload["dataSource"], "BOS")
self.assertEqual(payload["sourcePath"], "bos://datastorage/algo/X-platform-test/")
self.assertEqual(payload["newCount"], 0)
def test_clean_metadata_decodes_nested_bytes_values(self):
payload = self.runner.clean_metadata(
{
"path": "bos://datastorage/algo/X-platform-test/video.mp4",
"source": {
"previewURL": b"https://example.com/video.mp4",
"path": b"bos://datastorage/algo/X-platform-test/video.mp4",
},
}
)
self.assertEqual(payload["source"]["previewURL"], "https://example.com/video.mp4")
self.assertEqual(payload["source"]["path"], "bos://datastorage/algo/X-platform-test/video.mp4")
def test_builtin_source_incremental_mode_only_emits_new_entries_and_checkpoint(self):
listed_entries = [
{
"path": "BJ_001/DJI_0001_D.mp4",
"sizeBytes": 12,
"storagePath": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_0001_D.mp4",
"etag": "etag-1",
"lastModified": "2026-03-19T14:51:43Z",
"source": {"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_0001_D.mp4"},
},
{
"path": "BJ_001/meta.json",
"sizeBytes": 32,
"storagePath": "bos://datastorage/algo/X-platform-test/BJ_001/meta.json",
"etag": "etag-2",
"lastModified": "2026-03-19T14:51:44Z",
"data": {"project_id": "BJ_001"},
"source": {"path": "bos://datastorage/algo/X-platform-test/BJ_001/meta.json"},
},
]
seen_fingerprint = self.runner.source_entry_fingerprint("BOS", "datastorage", listed_entries[0])
node_data = {
"label": "BOS 增量扫描",
"outputs": [{"id": "out-1", "name": "manifest", "type": "json"}],
"hyperparams": [
{"key": "DATA_SOURCE", "value": "BOS"},
{"key": "SOURCE_PATH", "value": "bos://datastorage/algo/X-platform-test"},
{"key": "BOS_ENDPOINT", "value": "https://bos.example.test"},
{"key": "BOS_ACCESS_KEY_ID", "value": "ak"},
{"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"},
{"key": "SCAN_MODE", "value": "INCREMENTAL"},
{
"key": "SOURCE_STATE_JSON",
"value": json.dumps(
{
"sourceFingerprint": "fp-1",
"seenFingerprints": [seen_fingerprint],
}
),
},
],
"bucket": "datastorage",
"prefix": "algo/X-platform-test",
}
with tempfile.TemporaryDirectory() as tempdir:
with mock.patch.object(self.runner, "list_bos_entries", return_value=listed_entries):
artifacts = self.runner.builtin_source("run-1", "node-source", pathlib.Path(tempdir), node_data, [])
manifest = next(item for item in artifacts if item["name"] == "source-manifest.json")
checkpoint = next(item for item in artifacts if item["name"] == "source-checkpoint.json")
manifest_payload = self.runner.load_artifact_payload(manifest)
checkpoint_payload = self.runner.load_artifact_payload(checkpoint)
emitted_paths = sorted(item["path"] for item in manifest_payload["entries"])
self.assertEqual(emitted_paths, ["BJ_001/meta.json"])
catalog_paths = sorted(item["path"] for item in manifest_payload["catalogEntries"])
self.assertEqual(catalog_paths, ["BJ_001/DJI_0001_D.mp4", "BJ_001/meta.json"])
self.assertEqual(manifest_payload["listedCount"], 2)
self.assertEqual(manifest_payload["newCount"], 1)
self.assertEqual(checkpoint_payload["listedCount"], 2)
self.assertEqual(checkpoint_payload["newCount"], 1)
self.assertEqual(len(checkpoint_payload["seenFingerprints"]), 2)
self.assertEqual(checkpoint.get("portId"), None)
if __name__ == "__main__":
unittest.main()