forked from longtao.wu/droboticflow
209 lines
9.0 KiB
Python
209 lines
9.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import posixpath
|
|
import shutil
|
|
from typing import Any, Callable
|
|
|
|
|
|
HelperMap = dict[str, Callable[..., Any]]
|
|
|
|
|
|
def safe_inline_output_name(value: str, default: str) -> str:
|
|
name = str(value or "").strip() or default
|
|
normalized = posixpath.normpath(name)
|
|
if normalized.startswith("../") or normalized == ".." or posixpath.isabs(normalized):
|
|
raise RuntimeError("OUTPUT_NAME must be a relative file name without parent directory traversal")
|
|
if normalized in {"", "."}:
|
|
raise RuntimeError("OUTPUT_NAME must be a non-empty file name")
|
|
return normalized
|
|
|
|
|
|
def run(
|
|
run_id: str,
|
|
node_id: str,
|
|
output_dir: Any,
|
|
node_data: dict[str, Any],
|
|
inputs: list[dict[str, Any]],
|
|
helpers: HelperMap,
|
|
) -> list[dict[str, Any]]:
|
|
params = helpers["hyperparams"](node_data)
|
|
inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT"))
|
|
data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper()
|
|
if data_source and data_source not in helpers["VALID_SOURCE_DATA_SOURCES"]:
|
|
raise RuntimeError("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
|
|
source_path = (params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "")).strip()
|
|
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
|
|
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
|
|
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
|
|
bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip()
|
|
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip()
|
|
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
|
|
scan_mode = (params.get("SCAN_MODE") or "").strip().upper()
|
|
if scan_mode and scan_mode not in {"FULL", "INCREMENTAL"}:
|
|
raise RuntimeError("SCAN_MODE must be FULL or INCREMENTAL")
|
|
unit = helpers["incremental_unit"](params)
|
|
depth = helpers["incremental_directory_depth"](params)
|
|
source_bucket, source_prefix = helpers["parse_bos_path"](source_path)
|
|
if not bucket and source_bucket:
|
|
bucket = source_bucket
|
|
if not prefix and source_prefix:
|
|
prefix = source_prefix
|
|
local_source_path = pathlib.Path(source_path) if source_path and not source_bucket else None
|
|
local_prefix_path = pathlib.Path(prefix) if prefix and "://" not in prefix else None
|
|
has_bos_connection = bool(bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key)
|
|
|
|
if inline_source:
|
|
data_source = "INLINE"
|
|
elif not data_source:
|
|
if source_bucket or has_bos_connection:
|
|
data_source = "BOS"
|
|
elif local_source_path is not None:
|
|
data_source = "LOCAL"
|
|
elif local_prefix_path is not None and local_prefix_path.exists():
|
|
data_source = "LOCAL"
|
|
elif inputs:
|
|
data_source = "UPSTREAM"
|
|
|
|
if data_source == "INLINE" and not inline_source:
|
|
raise RuntimeError("真实数据源 DATA_SOURCE=INLINE requires SOURCE_TEXT or SOURCE_JSON")
|
|
if data_source == "BOS":
|
|
missing = []
|
|
if not bucket:
|
|
missing.append("bucket or bos:// SOURCE_PATH")
|
|
if not bos_endpoint:
|
|
missing.append("BOS_ENDPOINT")
|
|
if not bos_access_key_id:
|
|
missing.append("BOS_ACCESS_KEY_ID")
|
|
if not bos_secret_access_key:
|
|
missing.append("BOS_SECRET_ACCESS_KEY")
|
|
if missing:
|
|
raise RuntimeError("真实数据源 DATA_SOURCE=BOS requires " + ", ".join(missing))
|
|
if data_source == "LOCAL":
|
|
if source_bucket:
|
|
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL cannot read a bos:// SOURCE_PATH")
|
|
if local_source_path is not None and not local_source_path.exists():
|
|
raise RuntimeError(f"真实数据源 SOURCE_PATH does not exist inside operator-runner: {source_path}")
|
|
if local_source_path is None and (local_prefix_path is None or not local_prefix_path.exists()):
|
|
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL requires SOURCE_PATH or an existing STORAGE_PREFIX path")
|
|
if not data_source:
|
|
raise RuntimeError(
|
|
"真实数据源 requires SOURCE_TEXT, SOURCE_JSON, SOURCE_PATH, a configured BOS source, "
|
|
"an existing local STORAGE_PREFIX path, or upstream input artifacts"
|
|
)
|
|
|
|
entries: list[dict[str, Any]] = []
|
|
resolved_source_path = source_path
|
|
inline_payload: Any = None
|
|
inline_payload_kind = ""
|
|
if params.get("SOURCE_JSON"):
|
|
parsed = json.loads(params["SOURCE_JSON"])
|
|
inline_payload = parsed
|
|
inline_payload_kind = "json"
|
|
if isinstance(parsed, list):
|
|
entries = [{"value": item} for item in parsed]
|
|
else:
|
|
entries = [{"value": parsed}]
|
|
elif params.get("SOURCE_TEXT"):
|
|
inline_payload = params["SOURCE_TEXT"]
|
|
inline_payload_kind = "text"
|
|
entries = [{"text": inline_payload}]
|
|
elif data_source == "BOS":
|
|
cleaned_prefix = helpers["bos_prefix"](prefix)
|
|
entries = helpers["list_bos_entries"](
|
|
bucket,
|
|
cleaned_prefix,
|
|
pattern,
|
|
bos_endpoint,
|
|
bos_access_key_id,
|
|
bos_secret_access_key,
|
|
)
|
|
resolved_source_path = helpers["canonical_bos_path"](bucket, cleaned_prefix)
|
|
elif local_source_path is not None:
|
|
entries = helpers["list_source_entries"](local_source_path, pattern)
|
|
elif local_prefix_path is not None and local_prefix_path.exists():
|
|
entries = helpers["list_source_entries"](local_prefix_path, pattern)
|
|
elif inputs:
|
|
entries = [helpers["load_artifact_payload"](item) for item in inputs]
|
|
|
|
listed_entries = list(entries)
|
|
checkpoint_payload = None
|
|
if scan_mode == "INCREMENTAL":
|
|
entries, checkpoint_payload = helpers["apply_incremental_source_filter"](
|
|
listed_entries,
|
|
data_source,
|
|
bucket,
|
|
params,
|
|
resolved_source_path,
|
|
prefix,
|
|
pattern,
|
|
)
|
|
|
|
payload = {
|
|
"dataSource": data_source,
|
|
"bucket": bucket,
|
|
"prefix": prefix,
|
|
"endpoint": bos_endpoint,
|
|
"pattern": pattern,
|
|
"scanMode": scan_mode or "FULL",
|
|
"incrementalUnit": unit,
|
|
"directoryDepth": depth,
|
|
"sourcePath": resolved_source_path,
|
|
"sourceFingerprint": str((checkpoint_payload or {}).get("sourceFingerprint") or params.get("SOURCE_FINGERPRINT", "")).strip(),
|
|
"listedCount": len(listed_entries),
|
|
"newCount": len(entries) if checkpoint_payload is not None else len(entries),
|
|
"entries": entries,
|
|
"catalogEntries": listed_entries,
|
|
}
|
|
binding = helpers["output_binding"](node_data)
|
|
artifacts = [
|
|
helpers["write_json_artifact"](run_id, node_id, output_dir, "source-manifest.json", payload, output=binding)
|
|
]
|
|
payload_output_id = str(params.get("PAYLOAD_OUTPUT_ID", "")).strip()
|
|
if payload_output_id and inline_payload_kind == "json":
|
|
artifacts.append(
|
|
helpers["write_json_artifact"](
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
safe_inline_output_name(params.get("OUTPUT_NAME", ""), "source-payload.json"),
|
|
inline_payload,
|
|
output=helpers["output_binding"](node_data, name=payload_output_id),
|
|
)
|
|
)
|
|
elif payload_output_id and inline_payload_kind == "text":
|
|
artifacts.append(
|
|
helpers["write_text_artifact"](
|
|
run_id,
|
|
node_id,
|
|
output_dir,
|
|
safe_inline_output_name(params.get("OUTPUT_NAME", ""), "source-text.txt"),
|
|
inline_payload,
|
|
output=helpers["output_binding"](node_data, name=payload_output_id),
|
|
)
|
|
)
|
|
if checkpoint_payload is not None:
|
|
artifacts.append(helpers["write_json_artifact"](run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload))
|
|
|
|
if data_source == "BOS":
|
|
for entry in entries:
|
|
artifacts.append(helpers["source_entry_artifact"](run_id, node_id, entry, resolved_source_path, output=binding))
|
|
elif source_path and pathlib.Path(source_path).exists():
|
|
source_root = pathlib.Path(source_path)
|
|
staged_root = output_dir / "_source_tree"
|
|
for entry in entries:
|
|
relative_path = str(entry.get("path", "")).strip()
|
|
if not relative_path:
|
|
continue
|
|
candidate = source_root / relative_path if source_root.is_dir() else source_root
|
|
if not candidate.exists() or not candidate.is_file():
|
|
continue
|
|
destination = staged_root / relative_path
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(candidate, destination)
|
|
artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, staged_root, node_data))
|
|
|
|
return artifacts
|