2026-08-19 11:42:35 +08:00

209 lines
9.0 KiB
Python

from __future__ import annotations
import json
import os
import pathlib
import posixpath
import shutil
from typing import Any, Callable
HelperMap = dict[str, Callable[..., Any]]
def safe_inline_output_name(value: str, default: str) -> str:
name = str(value or "").strip() or default
normalized = posixpath.normpath(name)
if normalized.startswith("../") or normalized == ".." or posixpath.isabs(normalized):
raise RuntimeError("OUTPUT_NAME must be a relative file name without parent directory traversal")
if normalized in {"", "."}:
raise RuntimeError("OUTPUT_NAME must be a non-empty file name")
return normalized
def run(
run_id: str,
node_id: str,
output_dir: Any,
node_data: dict[str, Any],
inputs: list[dict[str, Any]],
helpers: HelperMap,
) -> list[dict[str, Any]]:
params = helpers["hyperparams"](node_data)
inline_source = bool(params.get("SOURCE_JSON") or params.get("SOURCE_TEXT"))
data_source = (params.get("DATA_SOURCE") or os.getenv("DATA_SOURCE", "")).strip().upper()
if data_source and data_source not in helpers["VALID_SOURCE_DATA_SOURCES"]:
raise RuntimeError("DATA_SOURCE must be INLINE, LOCAL, BOS, or UPSTREAM")
source_path = (params.get("SOURCE_PATH") or os.getenv("SOURCE_PATH", "")).strip()
prefix = str(node_data.get("prefix", "") or os.getenv("STORAGE_PREFIX", "")).strip()
bucket = str(node_data.get("bucket", "") or os.getenv("STORAGE_BUCKET", "")).strip()
pattern = str(node_data.get("pattern", "") or os.getenv("STORAGE_PATTERN", "")).strip()
bos_endpoint = params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "").strip()
bos_access_key_id = params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "").strip()
bos_secret_access_key = params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "").strip()
scan_mode = (params.get("SCAN_MODE") or "").strip().upper()
if scan_mode and scan_mode not in {"FULL", "INCREMENTAL"}:
raise RuntimeError("SCAN_MODE must be FULL or INCREMENTAL")
unit = helpers["incremental_unit"](params)
depth = helpers["incremental_directory_depth"](params)
source_bucket, source_prefix = helpers["parse_bos_path"](source_path)
if not bucket and source_bucket:
bucket = source_bucket
if not prefix and source_prefix:
prefix = source_prefix
local_source_path = pathlib.Path(source_path) if source_path and not source_bucket else None
local_prefix_path = pathlib.Path(prefix) if prefix and "://" not in prefix else None
has_bos_connection = bool(bucket and bos_endpoint and bos_access_key_id and bos_secret_access_key)
if inline_source:
data_source = "INLINE"
elif not data_source:
if source_bucket or has_bos_connection:
data_source = "BOS"
elif local_source_path is not None:
data_source = "LOCAL"
elif local_prefix_path is not None and local_prefix_path.exists():
data_source = "LOCAL"
elif inputs:
data_source = "UPSTREAM"
if data_source == "INLINE" and not inline_source:
raise RuntimeError("真实数据源 DATA_SOURCE=INLINE requires SOURCE_TEXT or SOURCE_JSON")
if data_source == "BOS":
missing = []
if not bucket:
missing.append("bucket or bos:// SOURCE_PATH")
if not bos_endpoint:
missing.append("BOS_ENDPOINT")
if not bos_access_key_id:
missing.append("BOS_ACCESS_KEY_ID")
if not bos_secret_access_key:
missing.append("BOS_SECRET_ACCESS_KEY")
if missing:
raise RuntimeError("真实数据源 DATA_SOURCE=BOS requires " + ", ".join(missing))
if data_source == "LOCAL":
if source_bucket:
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL cannot read a bos:// SOURCE_PATH")
if local_source_path is not None and not local_source_path.exists():
raise RuntimeError(f"真实数据源 SOURCE_PATH does not exist inside operator-runner: {source_path}")
if local_source_path is None and (local_prefix_path is None or not local_prefix_path.exists()):
raise RuntimeError("真实数据源 DATA_SOURCE=LOCAL requires SOURCE_PATH or an existing STORAGE_PREFIX path")
if not data_source:
raise RuntimeError(
"真实数据源 requires SOURCE_TEXT, SOURCE_JSON, SOURCE_PATH, a configured BOS source, "
"an existing local STORAGE_PREFIX path, or upstream input artifacts"
)
entries: list[dict[str, Any]] = []
resolved_source_path = source_path
inline_payload: Any = None
inline_payload_kind = ""
if params.get("SOURCE_JSON"):
parsed = json.loads(params["SOURCE_JSON"])
inline_payload = parsed
inline_payload_kind = "json"
if isinstance(parsed, list):
entries = [{"value": item} for item in parsed]
else:
entries = [{"value": parsed}]
elif params.get("SOURCE_TEXT"):
inline_payload = params["SOURCE_TEXT"]
inline_payload_kind = "text"
entries = [{"text": inline_payload}]
elif data_source == "BOS":
cleaned_prefix = helpers["bos_prefix"](prefix)
entries = helpers["list_bos_entries"](
bucket,
cleaned_prefix,
pattern,
bos_endpoint,
bos_access_key_id,
bos_secret_access_key,
)
resolved_source_path = helpers["canonical_bos_path"](bucket, cleaned_prefix)
elif local_source_path is not None:
entries = helpers["list_source_entries"](local_source_path, pattern)
elif local_prefix_path is not None and local_prefix_path.exists():
entries = helpers["list_source_entries"](local_prefix_path, pattern)
elif inputs:
entries = [helpers["load_artifact_payload"](item) for item in inputs]
listed_entries = list(entries)
checkpoint_payload = None
if scan_mode == "INCREMENTAL":
entries, checkpoint_payload = helpers["apply_incremental_source_filter"](
listed_entries,
data_source,
bucket,
params,
resolved_source_path,
prefix,
pattern,
)
payload = {
"dataSource": data_source,
"bucket": bucket,
"prefix": prefix,
"endpoint": bos_endpoint,
"pattern": pattern,
"scanMode": scan_mode or "FULL",
"incrementalUnit": unit,
"directoryDepth": depth,
"sourcePath": resolved_source_path,
"sourceFingerprint": str((checkpoint_payload or {}).get("sourceFingerprint") or params.get("SOURCE_FINGERPRINT", "")).strip(),
"listedCount": len(listed_entries),
"newCount": len(entries) if checkpoint_payload is not None else len(entries),
"entries": entries,
"catalogEntries": listed_entries,
}
binding = helpers["output_binding"](node_data)
artifacts = [
helpers["write_json_artifact"](run_id, node_id, output_dir, "source-manifest.json", payload, output=binding)
]
payload_output_id = str(params.get("PAYLOAD_OUTPUT_ID", "")).strip()
if payload_output_id and inline_payload_kind == "json":
artifacts.append(
helpers["write_json_artifact"](
run_id,
node_id,
output_dir,
safe_inline_output_name(params.get("OUTPUT_NAME", ""), "source-payload.json"),
inline_payload,
output=helpers["output_binding"](node_data, name=payload_output_id),
)
)
elif payload_output_id and inline_payload_kind == "text":
artifacts.append(
helpers["write_text_artifact"](
run_id,
node_id,
output_dir,
safe_inline_output_name(params.get("OUTPUT_NAME", ""), "source-text.txt"),
inline_payload,
output=helpers["output_binding"](node_data, name=payload_output_id),
)
)
if checkpoint_payload is not None:
artifacts.append(helpers["write_json_artifact"](run_id, node_id, output_dir, "source-checkpoint.json", checkpoint_payload))
if data_source == "BOS":
for entry in entries:
artifacts.append(helpers["source_entry_artifact"](run_id, node_id, entry, resolved_source_path, output=binding))
elif source_path and pathlib.Path(source_path).exists():
source_root = pathlib.Path(source_path)
staged_root = output_dir / "_source_tree"
for entry in entries:
relative_path = str(entry.get("path", "")).strip()
if not relative_path:
continue
candidate = source_root / relative_path if source_root.is_dir() else source_root
if not candidate.exists() or not candidate.is_file():
continue
destination = staged_root / relative_path
destination.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(candidate, destination)
artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, staged_root, node_data))
return artifacts