forked from longtao.wu/droboticflow
294 lines
12 KiB
Python
294 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.metadata
|
|
import math
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from typing import Any
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import torch
|
|
from fastapi import FastAPI, HTTPException, Request
|
|
from fastapi.responses import FileResponse
|
|
from pydantic import BaseModel, Field
|
|
from sixdrepnet import SixDRepNet
|
|
|
|
|
|
MODEL_PATH = pathlib.Path(os.environ.get("HEADPOSE_MODEL_PATH", "/opt/models/6DRepNet_300W_LP_AFLW2000.pth"))
|
|
MODEL_SHA256 = os.environ.get("HEADPOSE_MODEL_SHA256", "sha256:9bdcd0e072df0d21e92288b40ea1d1b3cf7173c44728ab9c45c4d673f361e018").lower()
|
|
LICENSE_ID = os.environ.get("HEADPOSE_LICENSE_ID", "MIT")
|
|
OUTPUT_ROOT = pathlib.Path(os.environ.get("HEADPOSE_OUTPUT_ROOT", "/workspace/system/headpose-service"))
|
|
MAX_DOWNLOAD_BYTES = int(os.environ.get("HEADPOSE_MAX_DOWNLOAD_BYTES", str(4 * 1024 * 1024 * 1024)))
|
|
GPU_ID = int(os.environ.get("HEADPOSE_GPU_ID", "0" if torch.cuda.is_available() else "-1"))
|
|
STARTED_AT = time.time()
|
|
|
|
|
|
def sha256_file(path: pathlib.Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return "sha256:" + digest.hexdigest()
|
|
|
|
|
|
if not MODEL_PATH.is_file():
|
|
raise RuntimeError(f"6DRepNet weights are missing: {MODEL_PATH}")
|
|
if sha256_file(MODEL_PATH) != MODEL_SHA256:
|
|
raise RuntimeError("6DRepNet model digest does not match HEADPOSE_MODEL_SHA256")
|
|
if LICENSE_ID != "MIT":
|
|
raise RuntimeError("6DRepNet service requires the reviewed MIT license identifier")
|
|
|
|
OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
|
|
MODEL = SixDRepNet(gpu_id=GPU_ID, dict_path=str(MODEL_PATH))
|
|
MODEL_LOCK = threading.Lock()
|
|
FACE_DETECTOR = cv2.CascadeClassifier(str(pathlib.Path(cv2.data.haarcascades) / "haarcascade_frontalface_default.xml"))
|
|
if FACE_DETECTOR.empty():
|
|
raise RuntimeError("OpenCV frontal-face detector could not be loaded")
|
|
|
|
app = FastAPI(title="Droboticflow 6DRepNet Headpose Service", version="1.0.0")
|
|
|
|
|
|
class InferenceRequest(BaseModel):
|
|
source: str = Field(min_length=1)
|
|
frameStride: int = Field(default=3, ge=1, le=300)
|
|
maxFrames: int = Field(default=120, ge=1, le=10000)
|
|
model: str = "6DRepNet"
|
|
modelSha256: str
|
|
|
|
|
|
def safe_output(request_id: str, name: str) -> pathlib.Path:
|
|
if not request_id.replace("-", "").isalnum() or pathlib.PurePath(name).name != name:
|
|
raise HTTPException(status_code=404, detail="artifact not found")
|
|
target = (OUTPUT_ROOT / request_id / name).resolve()
|
|
if OUTPUT_ROOT.resolve() not in target.parents:
|
|
raise HTTPException(status_code=404, detail="artifact not found")
|
|
return target
|
|
|
|
|
|
def materialize_source(source: str, directory: pathlib.Path) -> pathlib.Path:
|
|
parsed = urllib.parse.urlparse(source)
|
|
if parsed.scheme in {"http", "https"}:
|
|
suffix = pathlib.PurePosixPath(parsed.path).suffix or ".bin"
|
|
destination = directory / f"input{suffix}"
|
|
request = urllib.request.Request(source, headers={"User-Agent": "droboticflow-headpose/1.0"})
|
|
with urllib.request.urlopen(request, timeout=300) as response, destination.open("wb") as output:
|
|
total = 0
|
|
while True:
|
|
chunk = response.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
total += len(chunk)
|
|
if total > MAX_DOWNLOAD_BYTES:
|
|
raise RuntimeError("input media exceeds HEADPOSE_MAX_DOWNLOAD_BYTES")
|
|
output.write(chunk)
|
|
return destination
|
|
if parsed.scheme:
|
|
raise RuntimeError(f"unsupported source scheme: {parsed.scheme}")
|
|
local = pathlib.Path(source).resolve()
|
|
workspace = pathlib.Path("/workspace").resolve()
|
|
if not local.is_file() or workspace not in local.parents:
|
|
raise RuntimeError("local source must be a readable file below /workspace")
|
|
return local
|
|
|
|
|
|
def detect_face(frame: np.ndarray) -> tuple[np.ndarray, tuple[int, int, int, int], float]:
|
|
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
boxes, _, weights = FACE_DETECTOR.detectMultiScale3(
|
|
gray,
|
|
scaleFactor=1.1,
|
|
minNeighbors=5,
|
|
minSize=(40, 40),
|
|
outputRejectLevels=True,
|
|
)
|
|
if len(boxes) == 0:
|
|
raise RuntimeError("no face detected in sampled frame")
|
|
best = max(range(len(boxes)), key=lambda index: float(weights[index]))
|
|
x, y, width, height = [int(value) for value in boxes[best]]
|
|
padding = int(max(width, height) * 0.2)
|
|
x0, y0 = max(0, x - padding), max(0, y - padding)
|
|
x1, y1 = min(frame.shape[1], x + width + padding), min(frame.shape[0], y + height + padding)
|
|
crop = frame[y0:y1, x0:x1]
|
|
confidence = 1.0 / (1.0 + math.exp(-float(weights[best])))
|
|
return crop, (x, y, width, height), confidence
|
|
|
|
|
|
def predict_frame(frame: np.ndarray) -> tuple[float, float, float, float, tuple[int, int, int, int]]:
|
|
crop, box, confidence = detect_face(frame)
|
|
with MODEL_LOCK, torch.inference_mode():
|
|
pitch, yaw, roll = MODEL.predict(crop)
|
|
return float(pitch[0]), float(yaw[0]), float(roll[0]), confidence, box
|
|
|
|
|
|
def encode_browser_video(source: pathlib.Path, destination: pathlib.Path) -> None:
|
|
command = [
|
|
"ffmpeg",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-y",
|
|
"-i",
|
|
str(source),
|
|
"-an",
|
|
"-c:v",
|
|
"libx264",
|
|
"-preset",
|
|
"veryfast",
|
|
"-crf",
|
|
"23",
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-movflags",
|
|
"+faststart",
|
|
str(destination),
|
|
]
|
|
completed = subprocess.run(command, capture_output=True, text=True, check=False)
|
|
if completed.returncode != 0 or not destination.is_file() or destination.stat().st_size <= 0:
|
|
detail = completed.stderr.strip() or completed.stdout.strip()
|
|
raise RuntimeError(f"failed to encode browser-playable HeadPose render: {detail[:800]}")
|
|
|
|
|
|
def process_video(source: pathlib.Path, destination: pathlib.Path, stride: int, max_frames: int) -> list[dict[str, Any]]:
|
|
capture = cv2.VideoCapture(str(source))
|
|
if not capture.isOpened():
|
|
raise RuntimeError("input video could not be opened")
|
|
fps = float(capture.get(cv2.CAP_PROP_FPS) or 0)
|
|
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
|
|
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
|
if fps <= 0 or width <= 0 or height <= 0:
|
|
capture.release()
|
|
raise RuntimeError("input video metadata is invalid")
|
|
working_video = destination.with_name(destination.stem + "-working.mp4")
|
|
writer = cv2.VideoWriter(str(working_video), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
|
|
if not writer.isOpened():
|
|
capture.release()
|
|
raise RuntimeError("annotated video writer could not be opened")
|
|
results: list[dict[str, Any]] = []
|
|
frame_index = 0
|
|
try:
|
|
while len(results) < max_frames:
|
|
ok, frame = capture.read()
|
|
if not ok:
|
|
break
|
|
if frame_index % stride == 0:
|
|
try:
|
|
pitch, yaw, roll, confidence, (x, y, box_width, box_height) = predict_frame(frame)
|
|
except RuntimeError as error:
|
|
if str(error) != "no face detected in sampled frame":
|
|
raise
|
|
else:
|
|
MODEL.draw_axis(frame, yaw, pitch, roll, x + box_width / 2, y + box_height / 2, max(40, box_width // 2))
|
|
cv2.rectangle(frame, (x, y), (x + box_width, y + box_height), (255, 255, 0), 2)
|
|
results.append(
|
|
{
|
|
"frameIndex": frame_index,
|
|
"timestampSeconds": frame_index / fps,
|
|
"yawDeg": yaw,
|
|
"pitchDeg": pitch,
|
|
"rollDeg": roll,
|
|
"confidence": confidence,
|
|
}
|
|
)
|
|
writer.write(frame)
|
|
frame_index += 1
|
|
finally:
|
|
capture.release()
|
|
writer.release()
|
|
if not results:
|
|
working_video.unlink(missing_ok=True)
|
|
raise RuntimeError("no sampled video frame produced a headpose result")
|
|
try:
|
|
encode_browser_video(working_video, destination)
|
|
finally:
|
|
working_video.unlink(missing_ok=True)
|
|
return results
|
|
|
|
|
|
def process_image(source: pathlib.Path, destination: pathlib.Path) -> list[dict[str, Any]]:
|
|
frame = cv2.imread(str(source))
|
|
if frame is None:
|
|
raise RuntimeError("input image could not be opened")
|
|
pitch, yaw, roll, confidence, (x, y, width, height) = predict_frame(frame)
|
|
MODEL.draw_axis(frame, yaw, pitch, roll, x + width / 2, y + height / 2, max(40, width // 2))
|
|
cv2.rectangle(frame, (x, y), (x + width, y + height), (255, 255, 0), 2)
|
|
if not cv2.imwrite(str(destination), frame):
|
|
raise RuntimeError("annotated image could not be written")
|
|
return [{"frameIndex": 0, "timestampSeconds": 0.0, "yawDeg": yaw, "pitchDeg": pitch, "rollDeg": roll, "confidence": confidence}]
|
|
|
|
|
|
@app.get("/health")
|
|
def health() -> dict[str, Any]:
|
|
return {
|
|
"ready": True,
|
|
"model": "6DRepNet",
|
|
"modelSha256": MODEL_SHA256,
|
|
"licenseId": LICENSE_ID,
|
|
"cuda": torch.cuda.is_available(),
|
|
"gpu": torch.cuda.get_device_name(GPU_ID) if GPU_ID >= 0 and torch.cuda.is_available() else "cpu",
|
|
}
|
|
|
|
|
|
@app.post("/v1/headpose/infer")
|
|
def infer(payload: InferenceRequest, request: Request) -> dict[str, Any]:
|
|
if payload.model != "6DRepNet" or payload.modelSha256.lower() != MODEL_SHA256:
|
|
raise HTTPException(status_code=422, detail="model identity does not match the deployed 6DRepNet weights")
|
|
request_id = uuid.uuid4().hex
|
|
output_dir = OUTPUT_ROOT / request_id
|
|
output_dir.mkdir(parents=True, exist_ok=False)
|
|
started = time.time()
|
|
try:
|
|
with tempfile.TemporaryDirectory(prefix="headpose-input-") as temporary:
|
|
source = materialize_source(payload.source, pathlib.Path(temporary))
|
|
is_image = source.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
|
|
output_name = "headpose-annotated.jpg" if is_image else "headpose-annotated.mp4"
|
|
output_path = output_dir / output_name
|
|
frames = process_image(source, output_path) if is_image else process_video(source, output_path, payload.frameStride, payload.maxFrames)
|
|
except Exception as exc:
|
|
shutil.rmtree(output_dir, ignore_errors=True)
|
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
|
elapsed_ms = int((time.time() - started) * 1000)
|
|
artifact_url = str(request.base_url).rstrip("/") + f"/artifacts/{request_id}/{output_name}"
|
|
return {
|
|
"frames": frames,
|
|
"annotatedMediaUrl": artifact_url,
|
|
"rendering": {
|
|
"kind": "headpose_axis_overlay",
|
|
"videoCodec": "h264",
|
|
"pixelFormat": "yuv420p",
|
|
"browserPlayable": True,
|
|
},
|
|
"runtimeEvidence": {
|
|
"service": "droboticflow-headpose",
|
|
"serviceVersion": "1.0.0",
|
|
"sixdrepnetVersion": importlib.metadata.version("sixdrepnet"),
|
|
"torchVersion": torch.__version__,
|
|
"cudaAvailable": torch.cuda.is_available(),
|
|
"gpu": torch.cuda.get_device_name(GPU_ID) if GPU_ID >= 0 and torch.cuda.is_available() else "cpu",
|
|
"modelSha256": MODEL_SHA256,
|
|
"licenseId": LICENSE_ID,
|
|
"sampledFrameCount": len(frames),
|
|
"latencyMs": elapsed_ms,
|
|
"serviceUptimeSeconds": int(time.time() - STARTED_AT),
|
|
"renderCodec": "h264" if not is_image else "jpeg",
|
|
"browserPlayable": True,
|
|
},
|
|
}
|
|
|
|
|
|
@app.get("/artifacts/{request_id}/{name}")
|
|
def artifact(request_id: str, name: str) -> FileResponse:
|
|
target = safe_output(request_id, name)
|
|
if not target.is_file():
|
|
raise HTTPException(status_code=404, detail="artifact not found")
|
|
media_type = "video/mp4" if target.suffix.lower() == ".mp4" else "image/jpeg"
|
|
return FileResponse(target, media_type=media_type, filename=target.name)
|