forked from longtao.wu/droboticflow
548 lines
18 KiB
Go
548 lines
18 KiB
Go
package lightweight
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
var pvcNamePattern = regexp.MustCompile(`^[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?$`)
|
|
|
|
const (
|
|
DefaultMountPath = "/opt/droboticflow/lightweight/operator"
|
|
ContainerArgsConfiguredKey = "containerArgsConfigured"
|
|
ContainerCommandConfiguredKey = "containerCommandConfigured"
|
|
maxCodeManifestFiles = 5000
|
|
maxCodeManifestTotalBytes = int64(128 << 20)
|
|
maxCodeManifestFileBytes = int64(32 << 20)
|
|
maxCodeManifestDuration = 15 * time.Second
|
|
codeManifestHashWorkers = 8
|
|
)
|
|
|
|
var manifestCacheDirectoryNames = map[string]struct{}{
|
|
".git": {}, ".mypy_cache": {}, ".pytest_cache": {}, ".venv": {},
|
|
"__pycache__": {}, "node_modules": {}, "venv": {},
|
|
}
|
|
|
|
var manifestNonRuntimeDirectoryNames = map[string]struct{}{
|
|
".github": {}, "docs": {}, "examples": {}, "tests": {},
|
|
}
|
|
|
|
var manifestTopLevelRuntimeDirectoryNames = map[string]struct{}{
|
|
"arrow": {}, "artifacts": {}, "checkpoints": {}, "datasets": {},
|
|
"failed-checkpoints": {}, "logs": {}, "model": {}, "outputs": {},
|
|
"results": {}, "resume-from": {}, "tensorboard": {},
|
|
}
|
|
|
|
type Config struct {
|
|
Enabled bool
|
|
CodeSubPath string
|
|
MountPath string
|
|
CodeReadOnly bool
|
|
ContainerArgsConfigured bool
|
|
ContainerCommandConfigured bool
|
|
Mounts []Mount
|
|
}
|
|
|
|
type Mount struct {
|
|
Name string `json:"name" bson:"name"`
|
|
PVCName string `json:"pvcName" bson:"pvcName"`
|
|
SubPath string `json:"subPath,omitempty" bson:"subPath,omitempty"`
|
|
MountPath string `json:"mountPath" bson:"mountPath"`
|
|
// ReadOnly is opt-in; omitted PVC mounts are writable by default.
|
|
ReadOnly bool `json:"readOnly" bson:"readOnly"`
|
|
IDEEnabled bool `json:"ideEnabled" bson:"ideEnabled"`
|
|
}
|
|
|
|
type RuntimeConfig struct {
|
|
NodeID string
|
|
NodeLabel string
|
|
Image string
|
|
CodeSubPath string
|
|
MountPath string
|
|
Command []string
|
|
Args []string
|
|
ArgsConfigured bool
|
|
WorkspaceRoot string
|
|
Mounts []Mount
|
|
}
|
|
|
|
func NormalizeContract(contract map[string]any, commandConfigured bool, argsConfigured bool) map[string]any {
|
|
next := map[string]any{}
|
|
for key, value := range contract {
|
|
next[key] = value
|
|
}
|
|
raw, _ := next["lightweightOperator"].(map[string]any)
|
|
lightweight := map[string]any{}
|
|
for key, value := range raw {
|
|
lightweight[key] = value
|
|
}
|
|
lightweight["enabled"] = true
|
|
if commandConfigured {
|
|
lightweight[ContainerCommandConfiguredKey] = true
|
|
}
|
|
if argsConfigured {
|
|
lightweight[ContainerArgsConfiguredKey] = true
|
|
}
|
|
if codeSubPath := NormalizeCodeSubPath(stringMapValue(lightweight, "codeSubPath")); codeSubPath != "" {
|
|
lightweight["codeSubPath"] = codeSubPath
|
|
}
|
|
if mountPath := strings.TrimSpace(stringMapValue(lightweight, "mountPath")); mountPath != "" {
|
|
lightweight["mountPath"] = mountPath
|
|
}
|
|
codeReadOnly, _ := lightweight["codeReadOnly"].(bool)
|
|
lightweight["codeReadOnly"] = codeReadOnly
|
|
lightweight["mounts"] = normalizeMounts(lightweight["mounts"])
|
|
next["lightweightOperator"] = lightweight
|
|
return next
|
|
}
|
|
|
|
func ConfigFromContract(contract map[string]any) Config {
|
|
raw, ok := contract["lightweightOperator"].(map[string]any)
|
|
if !ok {
|
|
return Config{}
|
|
}
|
|
enabled, _ := raw["enabled"].(bool)
|
|
codeSubPath := NormalizeCodeSubPath(stringMapValue(raw, "codeSubPath"))
|
|
mountPath := strings.TrimSpace(stringMapValue(raw, "mountPath"))
|
|
codeReadOnly, _ := raw["codeReadOnly"].(bool)
|
|
argsConfigured, _ := raw[ContainerArgsConfiguredKey].(bool)
|
|
commandConfigured, _ := raw[ContainerCommandConfiguredKey].(bool)
|
|
return Config{
|
|
Enabled: enabled || codeSubPath != "" || mountPath != "",
|
|
CodeSubPath: codeSubPath,
|
|
MountPath: mountPath,
|
|
CodeReadOnly: codeReadOnly,
|
|
ContainerArgsConfigured: argsConfigured,
|
|
ContainerCommandConfigured: commandConfigured,
|
|
Mounts: normalizeMounts(raw["mounts"]),
|
|
}
|
|
}
|
|
|
|
func NormalizeCodeSubPath(value string) string {
|
|
return strings.Trim(strings.TrimSpace(value), "/")
|
|
}
|
|
|
|
func ValidateRuntimeConfig(config RuntimeConfig) error {
|
|
if strings.TrimSpace(config.Image) == "" {
|
|
return lightweightConfigError(config, "base image is required")
|
|
}
|
|
if strings.TrimSpace(config.CodeSubPath) == "" {
|
|
return lightweightConfigError(config, "codeSubPath is required")
|
|
}
|
|
if err := validateVolumeSubPath(config.CodeSubPath); err != nil {
|
|
return lightweightConfigError(config, "codeSubPath is invalid: "+err.Error())
|
|
}
|
|
mountPath := strings.TrimSpace(config.MountPath)
|
|
if mountPath == "" {
|
|
return lightweightConfigError(config, "mountPath is required")
|
|
}
|
|
if !strings.HasPrefix(mountPath, "/") {
|
|
return lightweightConfigError(config, "mountPath must be absolute")
|
|
}
|
|
workspaceRoot := cleanAbsPath(config.WorkspaceRoot, "/workspace")
|
|
if pathHasPrefix(mountPath, filepath.ToSlash(filepath.Join(workspaceRoot, "system", "runs"))) {
|
|
return lightweightConfigError(config, "mountPath must not overlap platform run output paths")
|
|
}
|
|
if len(normalizeStringSlice(config.Command)) == 0 {
|
|
return lightweightConfigError(config, "containerCommand is required")
|
|
}
|
|
if len(config.Args) == 0 && !config.ArgsConfigured {
|
|
return lightweightConfigError(config, "containerArgs must be explicitly configured; use [] when no args are needed")
|
|
}
|
|
if err := ValidateMounts(config.Mounts, mountPath, workspaceRoot); err != nil {
|
|
return lightweightConfigError(config, err.Error())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ValidateMounts(mounts []Mount, codeMountPath string, workspaceRoot string) error {
|
|
names := map[string]struct{}{}
|
|
paths := []string{strings.TrimSpace(codeMountPath)}
|
|
for index, item := range mounts {
|
|
name := strings.TrimSpace(item.Name)
|
|
if name == "" {
|
|
return fmt.Errorf("mounts[%d].name is required", index)
|
|
}
|
|
if name == "code" {
|
|
return fmt.Errorf("mounts[%d].name is reserved: code", index)
|
|
}
|
|
if _, exists := names[name]; exists {
|
|
return fmt.Errorf("mounts[%d].name must be unique: %s", index, name)
|
|
}
|
|
names[name] = struct{}{}
|
|
pvcName := strings.TrimSpace(item.PVCName)
|
|
if pvcName == "" {
|
|
return fmt.Errorf("mounts[%d].pvcName is required", index)
|
|
}
|
|
if len(pvcName) > 253 || !pvcNamePattern.MatchString(pvcName) {
|
|
return fmt.Errorf("mounts[%d].pvcName must be a valid Kubernetes PVC name", index)
|
|
}
|
|
if item.SubPath != "" {
|
|
if err := validateVolumeSubPath(item.SubPath); err != nil {
|
|
return fmt.Errorf("mounts[%d].subPath is invalid: %w", index, err)
|
|
}
|
|
}
|
|
mountPath := strings.TrimSpace(item.MountPath)
|
|
if mountPath == "" || !strings.HasPrefix(mountPath, "/") {
|
|
return fmt.Errorf("mounts[%d].mountPath must be an absolute path", index)
|
|
}
|
|
if pathHasPrefix(mountPath, filepath.ToSlash(filepath.Join(workspaceRoot, "system", "runs"))) {
|
|
return fmt.Errorf("mounts[%d].mountPath must not overlap platform run output paths", index)
|
|
}
|
|
for _, existing := range paths {
|
|
if existing != "" && (pathHasPrefix(mountPath, existing) || pathHasPrefix(existing, mountPath)) {
|
|
return fmt.Errorf("mounts[%d].mountPath overlaps another mount: %s", index, mountPath)
|
|
}
|
|
}
|
|
paths = append(paths, mountPath)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func normalizeMounts(value any) []Mount {
|
|
if value == nil {
|
|
return []Mount{}
|
|
}
|
|
raw, err := json.Marshal(value)
|
|
if err != nil {
|
|
return []Mount{}
|
|
}
|
|
var decoded []Mount
|
|
if err := json.Unmarshal(raw, &decoded); err != nil {
|
|
return []Mount{}
|
|
}
|
|
result := make([]Mount, 0, len(decoded))
|
|
for _, mount := range decoded {
|
|
mount.Name = strings.TrimSpace(mount.Name)
|
|
mount.PVCName = strings.TrimSpace(mount.PVCName)
|
|
mount.SubPath = NormalizeCodeSubPath(mount.SubPath)
|
|
mount.MountPath = strings.TrimSpace(mount.MountPath)
|
|
result = append(result, mount)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func GenerateWorkflowCodeManifests(workspaceRoot string, workflow domain.WorkflowRecord) (map[string]domain.LightweightOperatorCodeManifest, error) {
|
|
workspaceRoot = cleanAbsPath(workspaceRoot, "/workspace")
|
|
items := map[string]domain.LightweightOperatorCodeManifest{}
|
|
for _, node := range workflow.Nodes {
|
|
config := ConfigFromContract(node.Data.RuntimeContract)
|
|
if !config.Enabled {
|
|
continue
|
|
}
|
|
if err := ValidateRuntimeConfig(RuntimeConfig{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
Image: node.Data.Image,
|
|
CodeSubPath: config.CodeSubPath,
|
|
MountPath: config.MountPath,
|
|
Command: node.Data.ContainerCommand,
|
|
Args: node.Data.ContainerArgs,
|
|
ArgsConfigured: config.ContainerArgsConfigured,
|
|
WorkspaceRoot: workspaceRoot,
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
manifest, err := GenerateCodeManifest(workspaceRoot, node, config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items[node.ID] = manifest
|
|
}
|
|
if len(items) == 0 {
|
|
return nil, nil
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func GenerateCodeManifest(workspaceRoot string, node domain.WorkflowNode, config Config) (domain.LightweightOperatorCodeManifest, error) {
|
|
codeSubPath := NormalizeCodeSubPath(config.CodeSubPath)
|
|
if err := validateVolumeSubPath(codeSubPath); err != nil {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s codeSubPath is invalid: %w", nodeLabel(node), err)
|
|
}
|
|
workspaceRoot = cleanAbsPath(workspaceRoot, "/workspace")
|
|
rootReal, err := filepath.EvalSymlinks(workspaceRoot)
|
|
if err != nil {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s workspace root is not readable: %w", nodeLabel(node), err)
|
|
}
|
|
codePath := filepath.Join(workspaceRoot, filepath.FromSlash(codeSubPath))
|
|
codeInfo, err := os.Lstat(codePath)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory does not exist: %s", nodeLabel(node), codePath)
|
|
}
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory is not readable: %w", nodeLabel(node), err)
|
|
}
|
|
if codeInfo.Mode()&os.ModeSymlink != 0 {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory must not be a symlink: %s", nodeLabel(node), codePath)
|
|
}
|
|
if !codeInfo.IsDir() {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s codeSubPath must be a directory: %s", nodeLabel(node), codePath)
|
|
}
|
|
codeReal, err := filepath.EvalSymlinks(codePath)
|
|
if err != nil {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory is not readable: %w", nodeLabel(node), err)
|
|
}
|
|
if !pathInside(codeReal, rootReal) {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory escapes workspace root: %s", nodeLabel(node), codePath)
|
|
}
|
|
|
|
files := make([]domain.LightweightOperatorCodeFileManifest, 0)
|
|
excludedPaths := make([]string, 0)
|
|
totalBytes := int64(0)
|
|
startedAt := time.Now()
|
|
err = filepath.WalkDir(codePath, func(itemPath string, entry os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if itemPath == codePath {
|
|
return nil
|
|
}
|
|
rel, err := filepath.Rel(codePath, itemPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel = filepath.ToSlash(rel)
|
|
if entry.IsDir() && excludeCodeManifestDirectory(rel, entry.Name()) {
|
|
excludedPaths = append(excludedPaths, rel)
|
|
return filepath.SkipDir
|
|
}
|
|
if time.Since(startedAt) > maxCodeManifestDuration {
|
|
return fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
|
|
}
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Mode()&os.ModeSymlink != 0 {
|
|
target, err := filepath.EvalSymlinks(itemPath)
|
|
if err != nil {
|
|
return fmt.Errorf("symlink is not readable: %s: %w", itemPath, err)
|
|
}
|
|
if !pathInside(target, rootReal) {
|
|
return fmt.Errorf("symlink escapes workspace root: %s", itemPath)
|
|
}
|
|
return nil
|
|
}
|
|
if !info.Mode().IsRegular() {
|
|
return nil
|
|
}
|
|
if len(files) >= maxCodeManifestFiles {
|
|
return fmt.Errorf("code manifest exceeds %d source files; move generated data outside the code directory", maxCodeManifestFiles)
|
|
}
|
|
if info.Size() > maxCodeManifestFileBytes {
|
|
return fmt.Errorf("source file %s exceeds %d MiB; store models and generated data outside the code directory", rel, maxCodeManifestFileBytes>>20)
|
|
}
|
|
if totalBytes+info.Size() > maxCodeManifestTotalBytes {
|
|
return fmt.Errorf("code manifest exceeds %d MiB; move models and generated data outside the code directory", maxCodeManifestTotalBytes>>20)
|
|
}
|
|
totalBytes += info.Size()
|
|
files = append(files, domain.LightweightOperatorCodeFileManifest{
|
|
Path: rel,
|
|
SizeBytes: info.Size(),
|
|
Mode: info.Mode().String(),
|
|
ModTime: info.ModTime().UTC().Format(time.RFC3339Nano),
|
|
})
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err)
|
|
}
|
|
if len(files) == 0 {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory has no regular files: %s", nodeLabel(node), codePath)
|
|
}
|
|
if err := hashCodeManifestFiles(codePath, files, startedAt); err != nil {
|
|
return domain.LightweightOperatorCodeManifest{}, fmt.Errorf("lightweight operator %s code directory cannot be scanned: %w", nodeLabel(node), err)
|
|
}
|
|
sort.Slice(files, func(left int, right int) bool {
|
|
return files[left].Path < files[right].Path
|
|
})
|
|
aggregate := sha256.New()
|
|
for _, file := range files {
|
|
_, _ = fmt.Fprintf(aggregate, "%s\x00%d\x00%s\x00", file.Path, file.SizeBytes, file.SHA256)
|
|
}
|
|
return domain.LightweightOperatorCodeManifest{
|
|
NodeID: node.ID,
|
|
NodeLabel: node.Data.Label,
|
|
CodeSubPath: codeSubPath,
|
|
MountPath: config.MountPath,
|
|
WorkspacePath: filepath.ToSlash(codePath),
|
|
// Code lives on the shared workspace PVC and is writable by default.
|
|
// Read-only behavior is opt-in through codeReadOnly.
|
|
ReadOnly: config.CodeReadOnly,
|
|
FileCount: len(files),
|
|
TotalBytes: totalBytes,
|
|
SHA256: hex.EncodeToString(aggregate.Sum(nil)),
|
|
GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
|
Files: files,
|
|
ExcludedPaths: excludedPaths,
|
|
}, nil
|
|
}
|
|
|
|
func hashCodeManifestFiles(codePath string, files []domain.LightweightOperatorCodeFileManifest, startedAt time.Time) error {
|
|
workerCount := min(codeManifestHashWorkers, len(files))
|
|
jobs := make(chan int)
|
|
var workers sync.WaitGroup
|
|
var errorOnce sync.Once
|
|
var firstError error
|
|
for range workerCount {
|
|
workers.Add(1)
|
|
go func() {
|
|
defer workers.Done()
|
|
for index := range jobs {
|
|
if time.Since(startedAt) > maxCodeManifestDuration {
|
|
errorOnce.Do(func() {
|
|
firstError = fmt.Errorf("code manifest exceeded %s; move generated data outside the code directory", maxCodeManifestDuration)
|
|
})
|
|
continue
|
|
}
|
|
sum, err := fileSHA256(filepath.Join(codePath, filepath.FromSlash(files[index].Path)))
|
|
if err != nil {
|
|
errorOnce.Do(func() { firstError = err })
|
|
continue
|
|
}
|
|
files[index].SHA256 = sum
|
|
}
|
|
}()
|
|
}
|
|
for index := range files {
|
|
jobs <- index
|
|
}
|
|
close(jobs)
|
|
workers.Wait()
|
|
return firstError
|
|
}
|
|
|
|
func excludeCodeManifestDirectory(relativePath string, name string) bool {
|
|
if _, excluded := manifestCacheDirectoryNames[name]; excluded {
|
|
return true
|
|
}
|
|
if _, excluded := manifestNonRuntimeDirectoryNames[name]; excluded {
|
|
return true
|
|
}
|
|
if strings.Contains(relativePath, "/") {
|
|
return false
|
|
}
|
|
_, excluded := manifestTopLevelRuntimeDirectoryNames[name]
|
|
return excluded
|
|
}
|
|
|
|
func MergeRunMetadata(run *domain.RunRecord, manifests map[string]domain.LightweightOperatorCodeManifest) {
|
|
if run == nil || len(manifests) == 0 {
|
|
return
|
|
}
|
|
if run.Metadata == nil {
|
|
run.Metadata = &domain.RunMetadata{}
|
|
}
|
|
run.Metadata.LightweightOperatorCodeManifests = manifests
|
|
}
|
|
|
|
func stringMapValue(items map[string]any, key string) string {
|
|
value, ok := items[key]
|
|
if !ok || value == nil {
|
|
return ""
|
|
}
|
|
return fmt.Sprint(value)
|
|
}
|
|
|
|
func normalizeStringSlice(items []string) []string {
|
|
normalized := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
value := strings.TrimSpace(item)
|
|
if value != "" {
|
|
normalized = append(normalized, value)
|
|
}
|
|
}
|
|
return normalized
|
|
}
|
|
|
|
func lightweightConfigError(config RuntimeConfig, message string) error {
|
|
label := strings.TrimSpace(config.NodeLabel)
|
|
if label == "" {
|
|
label = config.NodeID
|
|
}
|
|
if label == "" {
|
|
label = "unknown"
|
|
}
|
|
return fmt.Errorf("lightweight operator %s is not runnable: %s", label, message)
|
|
}
|
|
|
|
func validateVolumeSubPath(value string) error {
|
|
trimmed := NormalizeCodeSubPath(value)
|
|
if trimmed == "" {
|
|
return fmt.Errorf("value is required")
|
|
}
|
|
cleaned := filepath.ToSlash(filepath.Clean(trimmed))
|
|
if cleaned == "." || strings.HasPrefix(cleaned, "../") || cleaned == ".." || strings.Contains(cleaned, "/../") {
|
|
return fmt.Errorf("must stay within the workspace PVC")
|
|
}
|
|
if strings.HasPrefix(cleaned, "/") {
|
|
return fmt.Errorf("must be relative")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func cleanAbsPath(value string, fallback string) string {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
trimmed = fallback
|
|
}
|
|
if !strings.HasPrefix(trimmed, "/") {
|
|
trimmed = fallback
|
|
}
|
|
return strings.TrimRight(filepath.ToSlash(filepath.Clean(trimmed)), "/")
|
|
}
|
|
|
|
func pathHasPrefix(value string, prefix string) bool {
|
|
value = cleanAbsPath(value, value)
|
|
prefix = cleanAbsPath(prefix, prefix)
|
|
return value == prefix || strings.HasPrefix(value, prefix+"/")
|
|
}
|
|
|
|
func pathInside(value string, root string) bool {
|
|
value = filepath.Clean(value)
|
|
root = filepath.Clean(root)
|
|
rel, err := filepath.Rel(root, value)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..")
|
|
}
|
|
|
|
func fileSHA256(path string) (string, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer file.Close()
|
|
hash := sha256.New()
|
|
if _, err := io.Copy(hash, file); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
|
}
|
|
|
|
func nodeLabel(node domain.WorkflowNode) string {
|
|
label := strings.TrimSpace(node.Data.Label)
|
|
if label != "" {
|
|
return label
|
|
}
|
|
if strings.TrimSpace(node.ID) != "" {
|
|
return node.ID
|
|
}
|
|
return "unknown"
|
|
}
|