213 lines
8.8 KiB
Python

from __future__ import annotations
import mimetypes
import os
import pathlib
import shutil
from typing import Any, Callable
HelperMap = dict[str, Callable[..., Any]]
def run(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
upload_path = params.get("OUTPUT_PATH", "").strip() or params.get("SOURCE_PATH", "").strip()
bucket = str(node_data.get("bucket", "")).strip()
prefix = str(node_data.get("prefix", "")).strip()
parsed_bucket, parsed_prefix = helpers["parse_bos_path"](upload_path)
if parsed_bucket:
bucket = parsed_bucket
if parsed_prefix or upload_path.endswith("/"):
prefix = parsed_prefix
data_source = (params.get("DATA_SOURCE", "") or ("BOS" if bucket and upload_path.startswith("bos://") else "")).strip().upper()
bos_endpoint = params.get("BOS_ENDPOINT", "").strip()
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY", "").strip()
copied_paths: list[str] = []
bundle_inputs: list[dict[str, Any]] = []
staging_dir = output_dir / "uploaded"
staging_dir.mkdir(parents=True, exist_ok=True)
package_manifest_payload = helpers["first_payload_with_keys"](inputs, "packageCount", "items")
upload_inputs = list(inputs)
if package_manifest_payload is not None:
package_names = helpers["package_related_artifact_names"](package_manifest_payload)
upload_inputs = [
artifact
for artifact in inputs
if str(artifact.get("name", "")).strip() not in package_names
]
upload_inputs.extend(
helpers["materialize_package_artifacts_from_manifest"](
run_id,
node_id,
staging_dir,
package_manifest_payload,
node_data,
)
)
if data_source == "BOS" and bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key:
client, scheme = helpers["build_bos_client"](bos_endpoint, bos_access_key_id, bos_secret_access_key)
temp_paths = [
str(item.get("path", "")).strip()
for item in upload_inputs
if str(item.get("path", "")).strip() and "://" not in str(item.get("path", "")).strip()
]
input_root = helpers["common_input_root"](temp_paths)
uploaded_objects: list[dict[str, Any]] = []
remote_artifacts: list[dict[str, Any]] = []
for index, artifact in enumerate(upload_inputs):
payload = helpers["load_artifact_payload"](artifact)
bundle_inputs.append(payload)
fallback_name = str(artifact.get("name", "")).strip() or f"artifact-{index}"
relative_path = helpers["upload_relative_path"](artifact, input_root, fallback_name)
temp_source = helpers["materialized_upload_source"](artifact, staging_dir, relative_path)
if temp_source is None or not temp_source.exists():
continue
mime_type = (
str((artifact.get("metadata") or {}).get("mimeType", "")).strip()
if isinstance(artifact.get("metadata"), dict)
else ""
) or mimetypes.guess_type(temp_source.name)[0] or ""
object_key = helpers["bos_key_join"](prefix, relative_path)
put_kwargs = {"content_type": mime_type} if mime_type else {}
try:
size_bytes = temp_source.stat().st_size
except OSError:
size_bytes = 0
helpers["log"](
f"[operator-runner][info] upload starts: {temp_source} -> {helpers['canonical_bos_path'](bucket, object_key)} "
f"({size_bytes} bytes)"
)
client.put_object_from_file(bucket, object_key, str(temp_source), **put_kwargs)
remote_path = helpers["canonical_bos_path"](bucket, object_key)
helpers["log"](
f"[operator-runner][info] upload finished: {temp_source} -> {remote_path} "
f"({size_bytes} bytes)"
)
preview_url = ""
if pathlib.PurePosixPath(relative_path).suffix.lower() in helpers["VIDEO_FILE_SUFFIXES"] or mime_type.startswith("video/"):
preview_url = helpers["text_value"](
client.generate_pre_signed_url(
bucket,
object_key,
expiration_in_seconds=3600,
protocol=scheme,
)
).strip()
uploaded_objects.append(
helpers["clean_metadata"](
{
"name": pathlib.PurePosixPath(relative_path).name,
"relativePath": relative_path,
"path": remote_path,
"previewURL": preview_url,
"sizeBytes": temp_source.stat().st_size,
"mimeType": mime_type,
}
)
or {}
)
artifact_type = str(artifact.get("type", "")).strip() or helpers["infer_port_type"](temp_source)
artifact_payload: dict[str, Any] | None = None
if artifact_type == "json":
loaded = helpers["load_payload_from_file"](temp_source)
if loaded is not None:
artifact_payload = helpers["normalize_json_payload"](loaded)
else:
artifact_payload = helpers["preview_payload_for_remote_entry"](relative_path, mime_type, preview_url)
if artifact_payload is None:
artifact_payload = helpers["extract_preview_data"](temp_source, artifact_type, mime_type)
if artifact_payload is not None:
artifact_payload["relativePath"] = relative_path
artifact_payload["source"] = helpers["clean_metadata"](
{
"path": remote_path,
"previewURL": preview_url,
"relativePath": relative_path,
"mimeType": mime_type,
}
) or {}
source_size_bytes = temp_source.stat().st_size
source_sha256, source_sha256_metadata = helpers["artifact_sha256"](temp_source, source_size_bytes)
remote_artifacts.append(
helpers["build_virtual_artifact"](
run_id,
node_id,
pathlib.PurePosixPath(relative_path).name,
remote_path,
artifact_type,
source_size_bytes,
data=artifact_payload,
metadata={
**source_sha256_metadata,
"sha256": source_sha256,
"mimeType": mime_type,
},
output=helpers["output_binding"](node_data),
)
)
payload = {
"copiedPaths": copied_paths,
"uploadedCount": len(uploaded_objects),
"uploadedObjects": uploaded_objects,
"items": bundle_inputs,
"inputCount": len(inputs),
"dataSource": "BOS",
"bucket": bucket,
"prefix": prefix,
"destinationPath": helpers["canonical_bos_path"](bucket, prefix),
}
artifacts = [
helpers["write_json_artifact"](
run_id,
node_id,
output_dir,
"uploaded-results.json",
payload,
output=helpers["output_binding"](node_data),
)
]
artifacts.extend(remote_artifacts)
return artifacts
for artifact in upload_inputs:
payload = helpers["load_artifact_payload"](artifact)
bundle_inputs.append(payload)
source_path = artifact.get("path")
if isinstance(source_path, str) and source_path and os.path.exists(source_path):
source = pathlib.Path(source_path)
destination = staging_dir / source.name
if source.resolve() != destination.resolve():
shutil.copy2(source, destination)
copied_paths.append(str(destination))
payload = {"copiedPaths": copied_paths, "items": bundle_inputs, "inputCount": len(inputs)}
artifacts = [
helpers["write_json_artifact"](
run_id,
node_id,
output_dir,
"uploaded-results.json",
payload,
output=helpers["output_binding"](node_data),
)
]
artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, staging_dir))
return artifacts