forked from longtao.wu/droboticflow
- scripts/local/env-local.sh + run-operator.sh (verified working again) - local-samples for json_to_csv + video_slice - docs/local-dev-guide.zh-CN.md + docs/release-checklist.zh-CN.md
132 lines
5.1 KiB
Bash
Executable File
132 lines
5.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 单算子本地一键调试:模拟 worker 在 Kubernetes Job 里调用 operator-runner 的同一套方式。
|
||
#
|
||
# 用法:
|
||
# scripts/local/run-operator.sh <slug> [输入文件或目录]
|
||
#
|
||
# 示例:
|
||
# scripts/local/run-operator.sh json_to_csv local-samples/json_to_csv/input.json
|
||
# scripts/local/run-operator.sh video_slice local-samples/video_slice/input.json
|
||
#
|
||
# 说明:
|
||
# - <slug> 是 operator-repos/builtin/ 下的目录名(如 video_slice)
|
||
# - 输入文件是 JSON Artifact 列表(形如 [{"name":..., "path":..., "type":...}, ...]);
|
||
# 若省略则使用 local-samples/<slug>/input.json(存在时),否则用空输入
|
||
# - 产物固定写到 ./local-runs/<slug>/<时间戳>/,包含 outputs/、result.json、stage-manifest.json
|
||
# - 调试 operator-repos 源码副本改动时无需任何额外参数:脚本默认用
|
||
# DROBOTICFLOW_OPERATOR_SOURCE 直接指向 operator-repos/builtin/<slug>/src/operator.py
|
||
# - 附加超参:OPERATOR_EXTRA_HYPERPARAMS='[{"key":"K","value":"V"}]' 变量传入
|
||
# - 使用仓库 .venv 里的 Python(含 numpy/opencv 等算子依赖);.venv 不存在时回退 python3
|
||
|
||
set -euo pipefail
|
||
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
cd "$REPO_ROOT"
|
||
|
||
slug="${1:-}"
|
||
if [[ -z "$slug" ]]; then
|
||
echo "用法: scripts/local/run-operator.sh <slug> [输入artifact列表JSON文件]" >&2
|
||
echo "示例: scripts/local/run-operator.sh json_to_csv local-samples/json_to_csv/input.json" >&2
|
||
exit 1
|
||
fi
|
||
|
||
op_dir="operator-repos/builtin/$slug"
|
||
if [[ ! -f "$op_dir/operator.json" ]]; then
|
||
echo "错误: 找不到算子 $slug($op_dir/operator.json 不存在)" >&2
|
||
echo "可用算子: ls operator-repos/builtin/" >&2
|
||
exit 1
|
||
fi
|
||
|
||
# 从 operator.json / runtime_contract.json 读取算子名和 handler
|
||
op_name="$(python3 -c "import json; print(json.load(open('$op_dir/operator.json'))['name'])")"
|
||
handler="$(python3 -c "import json; print(json.load(open('$op_dir/runtime_contract.json'))['handler'])")"
|
||
|
||
# 输入 artifact 列表:命令行 > local-samples 默认 > 空列表
|
||
input_file="${2:-}"
|
||
if [[ -z "$input_file" && -f "local-samples/$slug/input.json" ]]; then
|
||
input_file="local-samples/$slug/input.json"
|
||
fi
|
||
if [[ -n "$input_file" && ! -f "$input_file" ]]; then
|
||
echo "错误: 输入文件不存在: $input_file" >&2
|
||
exit 1
|
||
fi
|
||
|
||
run_dir="$REPO_ROOT/local-runs/$slug/$(date +%Y%m%d-%H%M%S)"
|
||
mkdir -p "$run_dir/outputs"
|
||
|
||
# 组装 NODE_DATA_JSON:label 用于日志显示,outputs 占位一个端口
|
||
extra_hyperparams="${OPERATOR_EXTRA_HYPERPARAMS:-[]}"
|
||
node_data_json="$(slug_label="$op_name" extra_hyperparams="$extra_hyperparams" python3 - <<'PY'
|
||
import json, os
|
||
print(json.dumps({
|
||
"label": os.environ["slug_label"],
|
||
"hyperparams": json.loads(os.environ["extra_hyperparams"]),
|
||
"outputs": [{"id": "out-1", "name": "output", "type": "json"}],
|
||
}, ensure_ascii=False))
|
||
PY
|
||
)"
|
||
|
||
# 输入 artifact:文件形式给 INPUT_ARTIFACTS_PATH(>64KiB 或含路径引用时更贴近真实)
|
||
if [[ -n "$input_file" ]]; then
|
||
cp "$input_file" "$run_dir/input-artifacts.json"
|
||
input_env=("INPUT_ARTIFACTS_PATH=$run_dir/input-artifacts.json")
|
||
else
|
||
input_env=("INPUT_ARTIFACTS_JSON=[]")
|
||
fi
|
||
|
||
# Python 解释器:优先仓库 .venv(有 numpy/opencv/h5py 等依赖)
|
||
if [[ -x ".venv/bin/python" ]]; then
|
||
PY=".venv/bin/python"
|
||
else
|
||
PY="python3"
|
||
echo "[提示] .venv 不存在,使用系统 python3;视频/科学计算类算子可能缺依赖" >&2
|
||
echo "[提示] 建议执行: python3 -m venv .venv && .venv/bin/pip install -r requirements-dev.txt" >&2
|
||
fi
|
||
|
||
echo "====== 单算子本地调试 ======"
|
||
echo "算子: $slug / $op_name"
|
||
echo "handler: $handler"
|
||
echo "输入: ${input_file:-(空输入)}"
|
||
echo "产物目录: $run_dir"
|
||
echo "============================"
|
||
|
||
env \
|
||
RUN_ID="run-local-$slug" \
|
||
WORKFLOW_ID="wf-local" \
|
||
NODE_ID="node-local-$slug" \
|
||
NODE_LABEL="$op_name" \
|
||
NODE_DATA_JSON="$node_data_json" \
|
||
"${input_env[@]}" \
|
||
OUTPUT_DIR="$run_dir/outputs" \
|
||
RESULT_OUTPUT_PATH="$run_dir/result.json" \
|
||
STAGE_MANIFEST_PATH="$run_dir/stage-manifest.json" \
|
||
DROBOTICFLOW_OPERATOR_HANDLER="$handler" \
|
||
DROBOTICFLOW_OPERATOR_SOURCE="$op_dir/src/operator.py" \
|
||
"$PY" backend/operator-runner/main.py
|
||
|
||
echo
|
||
echo "====== 执行完成 ======"
|
||
echo "result.json: $run_dir/result.json"
|
||
echo "stage-manifest: $run_dir/stage-manifest.json"
|
||
echo "outputs/: $run_dir/outputs/"
|
||
if [[ -f "$run_dir/result.json" ]]; then
|
||
python3 - "$run_dir/result.json" <<'PY'
|
||
import json, sys
|
||
try:
|
||
result = json.load(open(sys.argv[1]))
|
||
except Exception as exc:
|
||
print(f"(result.json 解析失败: {exc})")
|
||
raise SystemExit(0)
|
||
artifacts = result.get("artifacts", [])
|
||
print(f"产物 artifact 数: {len(artifacts)}")
|
||
for artifact in artifacts:
|
||
print(f" - {artifact.get('name')} (type={artifact.get('type')}, path={artifact.get('path')})")
|
||
logs = result.get("logs", [])
|
||
errors = [entry for entry in logs if entry.get("level", "").lower() in ("error", "warn")]
|
||
if errors:
|
||
print(f"日志中的告警/错误 {len(errors)} 条(前 3 条):")
|
||
for entry in errors[:3]:
|
||
print(f" [{entry.get('level')}] {entry.get('message', '')[:160]}")
|
||
PY
|
||
fi
|