forked from longtao.wu/droboticflow
213 lines
8.8 KiB
Python
213 lines
8.8 KiB
Python
from __future__ import annotations
|
|
|
|
import mimetypes
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
from typing import Any, Callable
|
|
|
|
|
|
HelperMap = dict[str, Callable[..., Any]]
|
|
|
|
|
|
def run(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip()
|
|
bucket = str(node_data.get("bucket", "")).strip()
|
|
prefix = str(node_data.get("prefix", "")).strip()
|
|
parsed_bucket, parsed_prefix = helpers["parse_bos_path"](upload_path)
|
|
if parsed_bucket:
|
|
bucket = parsed_bucket
|
|
if parsed_prefix or upload_path.endswith("/"):
|
|
prefix = parsed_prefix
|
|
data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper()
|
|
bos_endpoint = params.get("BOS_ENDPOINT", "").strip()
|
|
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip()
|
|
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip()
|
|
copied_paths: list[str] = []
|
|
bundle_inputs: list[dict[str, Any]] = []
|
|
staging_dir = output_dir / "uploaded"
|
|
staging_dir.mkdir(parents=True, exist_ok=True)
|
|
package_manifest_payload = helpers["first_payload_with_keys"](inputs, "packageCount", "items")
|
|
upload_inputs = list(inputs)
|
|
if package_manifest_payload is not None:
|
|
package_names = helpers["package_related_artifact_names"](package_manifest_payload)
|
|
upload_inputs = [
|
|
artifact
|
|
for artifact in inputs
|
|
if str(artifact.get("name", "")).strip() not in package_names
|
|
]
|
|
upload_inputs.extend(
|
|
helpers["materialize_package_artifacts_from_manifest"](
|
|
run_id,
|
|
node_id,
|
|
staging_dir,
|
|
package_manifest_payload,
|
|
node_data,
|
|
)
|
|
)
|
|
|
|
if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key:
|
|
client, scheme = helpers["build_bos_client"](bos_endpoint, bos_access_key_id, bos_secret_access_key)
|
|
temp_paths = [
|
|
str(item.get("path", "")).strip()
|
|
for item in upload_inputs
|
|
if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip()
|
|
]
|
|
input_root = helpers["common_input_root"](temp_paths)
|
|
uploaded_objects: list[dict[str, Any]] = []
|
|
remote_artifacts: list[dict[str, Any]] = []
|
|
|
|
for index, artifact in enumerate(upload_inputs):
|
|
payload = helpers["load_artifact_payload"](artifact)
|
|
bundle_inputs.append(payload)
|
|
fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}"
|
|
relative_path = helpers["upload_relative_path"](artifact, input_root, fallback_name)
|
|
temp_source = helpers["materialized_upload_source"](artifact, staging_dir, relative_path)
|
|
if temp_source is None or not temp_source.exists():
|
|
continue
|
|
|
|
mime_type = (
|
|
str((artifact.get("metadata") or {}).get("mimeType", "")).strip()
|
|
if isinstance(artifact.get("metadata"), dict)
|
|
else ""
|
|
) or mimetypes.guess_type(temp_source.name)[0] or ""
|
|
object_key = helpers["bos_key_join"](prefix, relative_path)
|
|
put_kwargs = {"content_type": mime_type} if mime_type else {}
|
|
try:
|
|
size_bytes = temp_source.stat().st_size
|
|
except OSError:
|
|
size_bytes = 0
|
|
helpers["log"](
|
|
f"[operator-runner][info] upload starts: {temp_source} -> {helpers['canonical_bos_path'](bucket, object_key)} "
|
|
f"({size_bytes} bytes)"
|
|
)
|
|
client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs)
|
|
remote_path = helpers["canonical_bos_path"](bucket, object_key)
|
|
helpers["log"](
|
|
f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} "
|
|
f"({size_bytes} bytes)"
|
|
)
|
|
preview_url = ""
|
|
if pathlib.PurePosixPath(relative_path).suffix.lower() in helpers["VIDEO_FILE_SUFFIXES"] or mime_type.startswith("video/"):
|
|
preview_url = helpers["text_value"](
|
|
client.generate_pre_signed_url(
|
|
bucket,
|
|
object_key,
|
|
expiration_in_seconds=3600,
|
|
protocol=scheme,
|
|
)
|
|
).strip()
|
|
|
|
uploaded_objects.append(
|
|
helpers["clean_metadata"](
|
|
{
|
|
"name": pathlib.PurePosixPath(relative_path).name,
|
|
"relativePath": relative_path,
|
|
"path": remote_path,
|
|
"previewURL": preview_url,
|
|
"sizeBytes": temp_source.stat().st_size,
|
|
"mimeType": mime_type,
|
|
}
|
|
)
|
|
or {}
|
|
)
|
|
|
|
artifact_type = str(artifact.get("type", "")).strip() or helpers["infer_port_type"](temp_source)
|
|
artifact_payload: dict[str, Any] | None = None
|
|
if artifact_type == "json":
|
|
loaded = helpers["load_payload_from_file"](temp_source)
|
|
if loaded is not None:
|
|
artifact_payload = helpers["normalize_json_payload"](loaded)
|
|
else:
|
|
artifact_payload = helpers["preview_payload_for_remote_entry"](relative_path, mime_type, preview_url)
|
|
if artifact_payload is None:
|
|
artifact_payload = helpers["extract_preview_data"](temp_source, artifact_type, mime_type)
|
|
if artifact_payload is not None:
|
|
artifact_payload["relativePath"] = relative_path
|
|
artifact_payload["source"] = helpers["clean_metadata"](
|
|
{
|
|
"path": remote_path,
|
|
"previewURL": preview_url,
|
|
"relativePath": relative_path,
|
|
"mimeType": mime_type,
|
|
}
|
|
) or {}
|
|
|
|
source_size_bytes = temp_source.stat().st_size
|
|
source_sha256, source_sha256_metadata = helpers["artifact_sha256"](temp_source, source_size_bytes)
|
|
|
|
remote_artifacts.append(
|
|
helpers["build_virtual_artifact"](
|
|
run_id,
|
|
node_id,
|
|
pathlib.PurePosixPath(relative_path).name,
|
|
remote_path,
|
|
artifact_type,
|
|
source_size_bytes,
|
|
data=artifact_payload,
|
|
metadata={
|
|
**source_sha256_metadata,
|
|
"sha256": source_sha256,
|
|
"mimeType": mime_type,
|
|
},
|
|
output=helpers["output_binding"](node_data),
|
|
)
|
|
)
|
|
|
|
payload = {
|
|
"copiedPaths": copied_paths,
|
|
"uploadedCount": len(uploaded_objects),
|
|
"uploadedObjects": uploaded_objects,
|
|
"items": bundle_inputs,
|
|
"inputCount": len(inputs),
|
|
"dataSource": "BOS",
|
|
"bucket": bucket,
|
|
"prefix": prefix,
|
|
"destinationPath": helpers["canonical_bos_path"](bucket, prefix),
|
|
}
|
|
artifacts = [
|
|
helpers["write_json_artifact"](
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"uploaded-results.json",
|
|
payload,
|
|
output=helpers["output_binding"](node_data),
|
|
)
|
|
]
|
|
artifacts.extend(remote_artifacts)
|
|
return artifacts
|
|
|
|
for artifact in upload_inputs:
|
|
payload = helpers["load_artifact_payload"](artifact)
|
|
bundle_inputs.append(payload)
|
|
source_path = artifact.get("path")
|
|
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
|
|
source = pathlib.Path(source_path)
|
|
destination = staging_dir / source.name
|
|
if source.resolve() != destination.resolve():
|
|
shutil.copy2(source, destination)
|
|
copied_paths.append(str(destination))
|
|
|
|
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
|
|
artifacts = [
|
|
helpers["write_json_artifact"](
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
"uploaded-results.json",
|
|
payload,
|
|
output=helpers["output_binding"](node_data),
|
|
)
|
|
]
|
|
artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, staging_dir))
|
|
return artifacts
|