forked from longtao.wu/droboticflow
144 lines
4.1 KiB
Go
144 lines
4.1 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
type contextFileReader struct {
|
|
ctx context.Context
|
|
io.Reader
|
|
}
|
|
|
|
func (r contextFileReader) Read(p []byte) (int, error) {
|
|
if err := r.ctx.Err(); err != nil {
|
|
return 0, err
|
|
}
|
|
return r.Reader.Read(p)
|
|
}
|
|
|
|
func copyAnnotationFile(ctx context.Context, from, to, expectedSHA string) error {
|
|
source, err := os.Open(from)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer source.Close()
|
|
if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil {
|
|
return err
|
|
}
|
|
target, err := os.Create(to)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
h := sha256.New()
|
|
_, err = io.Copy(io.MultiWriter(target, h), contextFileReader{ctx, source})
|
|
closeErr := target.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
if expectedSHA != "" && hex.EncodeToString(h.Sum(nil)) != expectedSHA {
|
|
return fmt.Errorf("cached media checksum mismatch: %s", filepath.Base(from))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func copyAnnotationTree(ctx context.Context, from, to string) error {
|
|
return filepath.WalkDir(from, func(path string, entry os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(from, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entry.Type()&os.ModeSymlink != 0 {
|
|
return fmt.Errorf("symlink in materialization output")
|
|
}
|
|
if entry.IsDir() {
|
|
return os.MkdirAll(filepath.Join(to, rel), 0o755)
|
|
}
|
|
return copyAnnotationFile(ctx, path, filepath.Join(to, rel), "")
|
|
})
|
|
}
|
|
|
|
type reusableSliceMedia struct {
|
|
directory string
|
|
files map[string]any
|
|
}
|
|
|
|
// Reuse verified bytes, not references into a generation that retention may
|
|
// delete. The destination always owns a complete independent set of files.
|
|
func reusableAnnotationMedia(run domain.RunRecord, task domain.AnnotationTaskRecord, item domain.HumanItemRecord, workspace, fingerprint string, revision int) map[string]reusableSliceMedia {
|
|
result := make(map[string]reusableSliceMedia)
|
|
prefix := fmt.Sprintf("%s-%s-%s-materialized-revision-", task.RunID, task.NodeID, task.ID)
|
|
for _, artifact := range run.Artifacts {
|
|
old, err := strconv.Atoi(strings.TrimPrefix(artifact.ID, prefix))
|
|
if !strings.HasPrefix(artifact.ID, prefix) || err != nil || old >= revision || old < 1 || stringValue(artifact.Data["sourceFingerprint"]) != fingerprint {
|
|
continue
|
|
}
|
|
expected := filepath.Join(workspace, "system", "runs", task.RunID, "human-materializations", item.ID, task.ID, fmt.Sprintf("revision-%06d", old))
|
|
if artifact.Path != expected {
|
|
continue
|
|
}
|
|
resolved, err := filepath.EvalSymlinks(expected)
|
|
workspaceReal, rootErr := filepath.EvalSymlinks(workspace)
|
|
rel, relErr := filepath.Rel(workspace, expected)
|
|
if err != nil || rootErr != nil || relErr != nil || resolved != filepath.Join(workspaceReal, rel) {
|
|
continue
|
|
}
|
|
bytes, err := os.ReadFile(filepath.Join(expected, "revision-manifest.json"))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
digest := sha256.Sum256(bytes)
|
|
if artifact.Metadata == nil || artifact.Metadata.SHA256 != hex.EncodeToString(digest[:]) {
|
|
continue
|
|
}
|
|
var manifest map[string]any
|
|
if json.Unmarshal(bytes, &manifest) != nil {
|
|
continue
|
|
}
|
|
items, _ := manifest["items"].([]any)
|
|
for _, value := range items {
|
|
entry := mapValue(value)
|
|
key := stringValue(entry["mediaKey"])
|
|
dir := stringValue(entry["directory"])
|
|
if key == "" || filepath.IsAbs(dir) || strings.Contains(dir, "..") {
|
|
continue
|
|
}
|
|
result[key] = reusableSliceMedia{directory: filepath.Join(resolved, filepath.FromSlash(dir)), files: mapValue(entry["files"])}
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func copyReusableVideo(ctx context.Context, media reusableSliceMedia, side, target string) bool {
|
|
file := mapValue(media.files[side+"Video"])
|
|
sha := stringValue(file["sha256"])
|
|
if sha == "" {
|
|
return false
|
|
}
|
|
path := filepath.Join(media.directory, side+".mp4")
|
|
resolved, err := filepath.EvalSymlinks(path)
|
|
if err != nil || resolved != path {
|
|
return false
|
|
}
|
|
return copyAnnotationFile(ctx, path, target, sha) == nil
|
|
}
|