gs_playground/scripts/render_real2sim_intro.py

187 lines
9.0 KiB
Python

#!/usr/bin/env python3
"""Render an attributed screenshot-based intro with local TTS and FFmpeg.
Pillow draws only original typography and layout layers. Source screenshots are
scaled/padded by FFmpeg without retouching their model content. Requires macOS
say, FFmpeg, ffprobe and Pillow. No API key, stock music or generated 3D footage.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import subprocess
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
ROOT = Path(__file__).resolve().parents[1]
ASSETS = ROOT / "docs/assets/real2sim-debug-optimization"
BG = "#111615"
WHITE = "#f4f7f6"
MUTED = "#a9bab4"
ACCENT = "#80e3b1"
def run(args: list[str]) -> None:
subprocess.run(args, check=True)
def probe(path: Path) -> dict:
return json.loads(subprocess.check_output([
"ffprobe", "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)
]))
def wrapped(text: str, draw: ImageDraw.ImageDraw, font: ImageFont.FreeTypeFont,
width: int) -> list[str]:
# Wrap Chinese by character and English by word without splitting a word.
tokens = list(text) if any("\u4e00" <= c <= "\u9fff" for c in text) else text.split(" ")
joiner = "" if any("\u4e00" <= c <= "\u9fff" for c in text) else " "
lines, line = [], ""
for token in tokens:
candidate = line + (joiner if line else "") + token
if line and draw.textlength(candidate, font=font) > width:
lines.append(line)
line = token
else:
line = candidate
return lines + ([line] if line else [])
def draw_text(draw, xy, text, font_path, size, width, color=WHITE, max_lines=3):
font = ImageFont.truetype(str(font_path), size)
lines = wrapped(text, draw, font, width)
if len(lines) > max_lines or any(draw.textlength(s, font=font) > width for s in lines):
raise ValueError(f"Text does not fit: {text}")
for i, line in enumerate(lines):
draw.text((xy[0], xy[1] + i * (size + 14)), line, font=font, fill=color)
def layout(scene: dict, index: int, count: int, portrait: bool, font: Path,
dest: Path, provenance: str) -> list[tuple[int, int, int, int]]:
w, h = (1080, 1920) if portrait else (1920, 1080)
pad = 72 if portrait else 96
im = Image.new("RGB", (w, h), BG)
d = ImageDraw.Draw(im)
d.line((pad, 114, w-pad, 114), fill="#34443c", width=2)
draw_text(d, (pad, 58), "REAL2SIM ASSET LAB", font, 28, w-2*pad, ACCENT)
draw_text(d, (w-pad-115, 58), f"{index+1:02d} / {count:02d}", font, 24, 120, MUTED)
draw_text(d, (pad, 154), scene["title"], font, 62 if portrait else 68, w-2*pad, max_lines=2)
draw_text(d, (pad, 260), scene["subtitle"], font, 26 if portrait else 30, w-2*pad, MUTED, 2)
boxes = []
if scene.get("images"):
n = len(scene["images"])
if portrait:
bw, bh = w-2*pad, 790 if n == 1 else 350
for i in range(n):
y = 388 + i * 445
draw_text(d, (pad, y-45), scene["labels"][i], font, 25, bw, MUTED, 1)
boxes.append((pad, y, bw, bh))
else:
gap = 36
bw, bh = ((w-2*pad-gap*(n-1))//n, 440)
for i in range(n):
x = pad + i*(bw+gap)
draw_text(d, (x, 329), scene["labels"][i], font, 26, bw, MUTED, 1)
boxes.append((x, 380, bw, bh))
else:
for i, (num, title, detail) in enumerate(scene["steps"]):
x = pad if portrait else pad + i*588
y = 408 + i*283 if portrait else 440
width = w-2*pad if portrait else 540
draw_text(d, (x, y), num, font, 30, width, ACCENT)
draw_text(d, (x, y+65), title, font, 52 if portrait else 46, width, max_lines=1)
draw_text(d, (x, y+145), detail, font, 30, width, MUTED, 2)
d.line((x, y+223, x+width-12, y+223), fill="#34443c", width=2)
draw_text(d, (pad, 1310 if portrait else 844), scene["narration"], font,
32 if portrait else 27, w-2*pad, WHITE, 4 if portrait else 2)
# Permanent English summary stays visible even without optional SRT subtitles.
ey = 1540 if portrait else 935
draw_text(d, (pad, ey), scene["english"], font, 28 if portrait else 25,
w-2*pad, WHITE, 4 if portrait else 2)
fy = 1760 if portrait else 1009
draw_text(d, (pad, fy), provenance, font, 23 if portrait else 20, w-2*pad, MUTED, 2)
d.rectangle((pad, h-37, pad+int((w-2*pad)*(index+1)/count), h-32), fill=ACCENT)
im.save(dest)
return boxes
def timestamp(seconds: float) -> str:
ms = round(seconds*1000)
return f"{ms//3600000:02d}:{ms//60000%60:02d}:{ms//1000%60:02d},{ms%1000:03d}"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, default=ROOT / "artifacts/open-source-launch")
parser.add_argument("--font", type=Path, default=Path("/System/Library/Fonts/Hiragino Sans GB.ttc"))
parser.add_argument("--voice", default="Tingting")
parser.add_argument("--language", choices=("zh", "en"), default="zh")
args = parser.parse_args()
data = json.loads((ROOT / "docs/open-source/storyboard.json").read_text())
out = args.output.resolve()
out.mkdir(parents=True, exist_ok=True)
durations = []
for i, scene in enumerate(data["scenes"]):
narration = out / f"narration-{i:02d}.txt"
narration.write_text(scene["english"] if args.language == "en" else scene["narration"], encoding="utf-8")
voice = out / f"voice-{i:02d}.aiff"
run(["say", "-v", args.voice, "-r", "205", "-f", str(narration), "-o", str(voice)])
durations.append(math.ceil((float(probe(voice)["format"]["duration"])+0.7)*30)/30)
for portrait, name in [(False, "landscape"), (True, "portrait")]:
folder = out / name
folder.mkdir(exist_ok=True)
parts = []
for i, scene in enumerate(data["scenes"]):
bg = folder / f"layout-{i:02d}.png"
boxes = layout(scene, i, len(durations), portrait, args.font, bg, data["provenance"])
command = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-loop", "1", "-framerate", "30", "-i", str(bg)]
for filename in scene.get("images", []):
command += ["-loop", "1", "-framerate", "30", "-i", str(ASSETS / filename)]
command += ["-i", str(out / f"voice-{i:02d}.aiff")]
filters = []
prev = "0:v"
for j, (x, y, bw, bh) in enumerate(boxes):
filters.append(f"[{j+1}:v]scale={bw}:{bh}:force_original_aspect_ratio=decrease,setsar=1,pad={bw}:{bh}:(ow-iw)/2:(oh-ih)/2:color={BG.replace('#','0x')}[im{j}]")
filters.append(f"[{prev}][im{j}]overlay={x}:{y}[v{j}]")
prev = f"v{j}"
filters.append(f"[{prev}]fade=t=in:d=0.22,format=yuv420p[v]")
filters.append(f"[{len(boxes)+1}:a]apad,alimiter=limit=0.90[a]")
part = folder / f"scene-{i:02d}.mp4"
command += ["-filter_complex", ";".join(filters), "-map", "[v]", "-map", "[a]", "-t", str(durations[i]),
"-c:v", "libx264", "-preset", "fast", "-crf", "19", "-threads", "4", "-r", "30",
"-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-movflags", "+faststart", str(part)]
run(command)
parts.append(part)
concat = folder / "concat.txt"
concat.write_text("".join(f"file '{p.name}'\n" for p in parts))
video = out / f"real2sim-asset-lab-{name}.mp4"
run(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-f", "concat", "-safe", "0", "-i", str(concat),
"-c", "copy", "-movflags", "+faststart", str(video)])
for scene_index in [0, 3, 5, 7]:
run(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-ss", str(sum(durations[:scene_index])+1),
"-i", str(video), "-frames:v", "1", str(out / f"{name}-check-{scene_index:02d}.png")])
for lang, key in [("zh", "narration"), ("en", "english")]:
time = 0.0
rows = []
for i, (scene, duration) in enumerate(zip(data["scenes"], durations)):
rows.append(f"{i+1}\n{timestamp(time)} --> {timestamp(time+duration)}\n{scene[key]}\n")
time += duration
(out / f"subtitles.{lang}.srt").write_text("\n".join(rows), encoding="utf-8")
manifest = {"format": "historical screenshot-based intro", "voice": args.voice, "language": args.language,
"durations": durations, "sources": {}, "videos": {}}
for source in sorted({s for scene in data["scenes"] for s in scene.get("images", [])}):
manifest["sources"][source] = hashlib.sha256((ASSETS/source).read_bytes()).hexdigest()
for video in out.glob("real2sim-asset-lab-*.mp4"):
manifest["videos"][video.name] = {"sha256": hashlib.sha256(video.read_bytes()).hexdigest(), "probe": probe(video)}
(out / "media-manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"Rendered {sum(durations):.2f}s in both formats: {out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())