data-gen-bpu/scripts/test_salty_client.py
bench 45b1262f01 feat: Salty 远程编译(05-compile PTQ),默认关闭
把本地工作区里编译上云的改动单独拿出来,放到当前生产代码(c2c52ed)上:
新增 salty_client / salty_stage / salty_stage_worker 及两组测试,并在
stagerun、pipeline、stages/base、s06_hbm 里接线。只有配置里写了
execution.salty.enabled: true 且 stages 含 05-compile 时才生效,
现有配置不受影响。

合到生产代码上时补的衔接:
- 编译次数上限:已记录过提交的 Salty 编译只做找回、绝不重提,不受上限
  阻拦;只准备好、从未提交的照常拒绝(新增 salty_stage.dispatch_recorded)。
- selected() 与 cleanup_state 读取在上下文缺字段时按"未启用 / VERIFIED"处理。

未包含:进程优雅退出(owned_process / namespace_owner 等)和其余板卡侧改动,
仍留在本地工作区。

测试(Linux, python3.10):test_salty_client、test_salty_stage 全过;
stagerun/pipeline/阶段相关的 13 个已有测试与 c2c52ed 结果一致。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 14:19:48 +08:00

667 lines
34 KiB
Python

"""Salty transport contracts against a stateful fixture CLI, never the cluster.
Run: python3 -B -m unittest scripts.test_salty_client
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
import errno
import hashlib
import json
from pathlib import Path
import sys
import tempfile
import threading
import unittest
from unittest.mock import patch
from harness.salty_client import (AmbiguousOperation, CliRunner, CommandResult, FormatError,
SaltyClient, TransportError, validate_image)
IMAGE = "registry.example.test/private/runtime@sha256:" + "a" * 64
JOB = "salty-0ac12d34-1111-4222-8333-0123456789ab"
class FakeCli:
"""Mutable remote state captures whether a single submission took effect."""
def __init__(self):
self.calls = []
self.directories = set()
self.uploads = {}
self.jobs = {}
self.run_mode = "success"
self.describe_drift = False
self.cleanup_timeout = False
self.version_failure = False
self.upload_failure = False
self.upload_timeout = False
self.mkdir_failure_after_create = False
self.mkdir_timeout_after_create = False
self.download_barrier = None
self.download_bytes = b"verified-by-caller-result"
self.check_before_run = None
def __call__(self, argv, stdout, stderr, timeout):
self.calls.append(list(argv))
args = list(argv[1:])
output, rc = "", 0
if args == ["version"]:
output = "salty 0.2.0\n"
if self.version_failure:
rc = 1
elif args[:2] == ["storage", "mkdir"]:
if args[2] in self.directories:
rc = 1
else:
self.directories.add(args[2])
if self.mkdir_failure_after_create:
rc = 1
if self.mkdir_timeout_after_create:
stdout.write_text("")
stderr.write_text("")
return CommandResult(None, pid=5123, timed_out=True, process_left_running=True)
elif args[:2] == ["storage", "cp"]:
if args[2].startswith("personal:/"):
Path(args[3]).write_bytes(self.download_bytes)
if self.download_barrier:
self.download_barrier.wait(timeout=5)
else:
self.uploads[args[3]] = Path(args[2]).read_bytes()
if self.upload_failure:
rc = 1
if self.upload_timeout:
stdout.write_text("")
stderr.write_text("")
return CommandResult(None, pid=6123, timed_out=True, process_left_running=True)
elif args[0] == "run":
if self.check_before_run:
self.check_before_run()
self.jobs[JOB] = "pending"
output = "已提交作业: " + JOB + "\n"
if self.run_mode == "timeout":
stdout.write_text(output)
stderr.write_text("")
return CommandResult(None, pid=9123, timed_out=True, process_left_running=True)
if self.run_mode == "failed":
rc = 1 # Server may still have accepted it.
if self.run_mode == "format":
output = "submission accepted but no parseable ID\n"
if self.run_mode == "exception":
raise RuntimeError("fixture transport lost after acceptance")
elif args[0] == "describe":
state = self.jobs[args[1]]
output = "ID: %s\n状态: %s\n" % (args[1], state)
if self.describe_drift:
output = "ID: %s\nStatus: %s\n" % (args[1], state)
elif args[0] == "rm":
if self.jobs[args[1]] not in {"succeeded", "failed"}:
raise AssertionError("must never remove an active fixture job")
del self.jobs[args[1]]
elif args[:3] == ["storage", "rm", "-r"]:
self.directories.remove(args[3])
if self.cleanup_timeout:
stdout.write_text("")
stderr.write_text("")
return CommandResult(None, pid=8123, timed_out=True, process_left_running=True)
else:
raise AssertionError("unexpected fixture command: " + repr(args))
stdout.write_text(output)
stderr.write_text("")
return CommandResult(rc, pid=7123)
def count(self, command):
return sum(call[1] == command for call in self.calls)
class TransportTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix="salty-transport-contract-")
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name)
self.bundle = self.root / "input.tar"
self.worker = self.root / "input.py"
self.bundle.write_bytes(b"credential-free fixture bundle")
self.worker.write_text("print('fixture worker')\n")
self.fake = FakeCli()
self.directory = self.root / "receipt"
self.client = self.prepare()
def prepare(self, **overrides):
options = {"image": IMAGE, "bundle": self.bundle, "worker": self.worker,
"command": ["python3", "-B", "{container_root}/worker.py",
"--bundle", "{container_root}/bundle.tar"],
"env": {"OMP_NUM_THREADS": "1"}, "runner": self.fake}
options.update(overrides)
return SaltyClient.prepare(self.directory, **options)
def restart(self):
return SaltyClient(self.directory, runner=self.fake)
def saved(self):
return json.loads((self.directory / "receipt.json").read_text())
def test_prepare_has_no_cli_effect_and_exclusive_snapshots(self):
self.assertEqual(self.fake.calls, [])
self.assertEqual(self.saved()["phase"], "PREPARED")
self.bundle.write_bytes(b"caller later edits original")
self.assertEqual((self.directory / "payload/bundle.tar").read_bytes(), b"credential-free fixture bundle")
with self.assertRaises(FileExistsError):
self.prepare()
self.assertEqual(self.fake.calls, [])
def test_submission_intent_is_durable_before_run_and_restart_never_resubmits(self):
def check():
receipt = self.saved()
self.assertEqual(receipt["phase"], "SUBMITTING")
self.assertEqual(receipt["submission_state"], "UNKNOWN")
self.assertEqual(receipt["commands"][-1]["label"], "submit")
self.assertEqual(receipt["commands"][-1]["state"], "STARTED")
self.fake.check_before_run = check
receipt = self.client.submit()
self.assertEqual(receipt["job_id"], JOB)
self.assertEqual(self.fake.count("run"), 1)
restarted = self.restart()
self.assertEqual(restarted.submit()["job_id"], JOB)
self.assertEqual(restarted.poll()["job_state"], "pending")
self.fake.jobs[JOB] = "succeeded"
self.assertEqual(self.restart().poll()["job_state"], "succeeded")
self.assertEqual(self.fake.count("run"), 1)
self.assertTrue(all(call[2] == JOB for call in self.fake.calls if call[1] == "describe"))
def test_resources_use_platform_defaults_unless_explicit(self):
self.client.submit()
run = next(call for call in self.fake.calls if call[1] == "run")
self.assertNotIn("--cpu", run)
self.assertNotIn("--memory", run)
explicit_fake = FakeCli()
explicit = SaltyClient.prepare(self.root / "explicit", image=IMAGE, bundle=self.bundle,
worker=self.worker, command=["python3", "worker.py"],
cpu=8, memory="16Gi", runner=explicit_fake)
explicit.submit()
run = next(call for call in explicit_fake.calls if call[1] == "run")
self.assertEqual(run[run.index("--cpu") + 1], "8")
self.assertEqual(run[run.index("--memory") + 1], "16Gi")
def test_concurrent_call_cannot_cross_receipt_lock(self):
def check():
with self.assertRaisesRegex(TransportError, "another caller"):
self.restart().submit()
self.fake.check_before_run = check
self.client.submit()
self.assertEqual(self.fake.count("run"), 1)
def test_changed_snapshot_is_detected_on_restart_before_remote_operations(self):
(self.directory / "payload/bundle.tar").write_bytes(b"replaced bytes")
with self.assertRaisesRegex(ValueError, "payload/source hash"):
self.restart()
self.assertEqual(self.fake.calls, [])
def test_unknown_submissions_never_retry_or_cleanup(self):
for mode in ("timeout", "failed", "format", "exception"):
with self.subTest(mode=mode):
root = self.root / mode
fake = FakeCli()
fake.run_mode = mode
client = SaltyClient.prepare(root, image=IMAGE, bundle=self.bundle,
worker=self.worker, command=["python3", "worker.py"], runner=fake)
with self.assertRaises(TransportError):
client.submit()
restart = SaltyClient(root, runner=fake)
self.assertEqual(restart.receipt["phase"], "SUBMISSION_UNKNOWN")
with self.assertRaises(AmbiguousOperation):
restart.submit()
with self.assertRaises(AmbiguousOperation):
restart.cleanup(authorized=True)
self.assertEqual(fake.count("run"), 1)
self.assertIn(JOB, fake.jobs)
if mode == "timeout":
entry = restart.receipt["commands"][-1]
self.assertEqual(entry["local_pid"], 9123)
self.assertTrue(entry["process_left_running"])
def test_crash_after_completed_submission_recovers_recorded_exact_id(self):
self.client.submit()
receipt = self.saved()
receipt.update(job_id=None, phase="SUBMITTING", submission_state="UNKNOWN")
(self.directory / "receipt.json").write_text(json.dumps(receipt))
restarted = self.restart()
self.assertEqual(restarted.recover_recorded_submission()["job_id"], JOB)
self.assertEqual(restarted.poll()["job_state"], "pending")
self.assertEqual(self.fake.count("run"), 1)
def test_unknown_completed_process_cannot_be_inferred_from_stdout(self):
self.fake.run_mode = "timeout"
with self.assertRaises(AmbiguousOperation):
self.client.submit()
with self.assertRaises(AmbiguousOperation):
self.restart().recover_recorded_submission()
self.assertEqual(self.fake.count("run"), 1)
def test_cli_deadline_leaves_process_to_finish_naturally(self):
marker = self.root / "natural-completion.txt"
stdout, stderr = self.root / "natural.stdout", self.root / "natural.stderr"
runner = CliRunner()
code = "import pathlib,time; time.sleep(0.3); pathlib.Path(%r).write_text('naturally finished')" % str(marker)
starts = []
result = runner([sys.executable, "-B", "-c", code], stdout, stderr, 0.02,
on_started=lambda pid, identity: starts.append((pid, identity)))
self.assertTrue(result.timed_out)
self.assertTrue(result.process_left_running)
self.assertEqual(starts, [(result.pid, result.birth_identity)])
if sys.platform == 'linux':
self.assertEqual(result.birth_identity['status'], 'KNOWN')
self.assertGreater(result.birth_identity['start_ticks'], 0)
process = runner._unresolved[result.pid]
self.assertIsNone(process.poll())
self.assertEqual(process.wait(timeout=5), 0)
self.assertEqual(marker.read_text(), "naturally finished")
del runner._unresolved[result.pid]
def test_cli_interrupt_retains_started_birth_identity_in_receipt(self):
from types import SimpleNamespace
process = SimpleNamespace(pid=4567, poll=lambda: (_ for _ in ()).throw(KeyboardInterrupt()))
identity = {'pid': 4567, 'status': 'KNOWN', 'boot_id': 'fixture', 'start_ticks': 12}
runner = CliRunner()
self.client.runner = runner
with patch('harness.salty_client.subprocess.Popen', return_value=process), \
patch('harness.salty_client._cli_birth_identity', return_value=identity):
with self.assertRaises(AmbiguousOperation):
self.client.submit()
entry = self.saved()['commands'][0]
self.assertEqual(entry['state'], 'UNKNOWN')
self.assertEqual(entry['local_birth_identity'], identity)
self.assertEqual(entry['local_pid'], 4567)
self.assertTrue(entry['process_left_running'])
self.assertIs(runner._unresolved[4567], process)
def test_pending_and_running_are_never_removed(self):
self.client.submit()
for state in ("pending", "running"):
self.fake.jobs[JOB] = state
with self.assertRaisesRegex(TransportError, "cannot be stopped or deleted"):
self.client.cleanup(authorized=True)
self.assertEqual(self.fake.count("rm"), 0)
self.assertEqual(len(self.fake.directories), 1)
self.assertEqual(self.saved()["cleanup"], {})
def test_prepared_without_cli_can_discard_only_local_temporary_files(self):
result = self.client.cleanup(authorized=True)
self.assertEqual(result["phase"], "CLEANED")
self.assertTrue(result["cleanup"]["never_submitted"])
self.assertEqual(self.fake.calls, [])
self.assertEqual({p.name for p in self.directory.iterdir()}, {"receipt.json", "receipt.lock"})
def test_version_failure_never_submitted_cleanup_has_no_remote_mutation(self):
self.fake.version_failure = True
with self.assertRaises(TransportError):
self.client.submit()
result = self.restart().cleanup(authorized=True)
self.assertEqual(result["phase"], "CLEANED")
self.assertEqual(self.fake.calls, [["salty", "version"]])
self.assertEqual(self.fake.directories, set())
self.assertFalse((self.directory / "payload").exists())
self.assertTrue(result["cleanup"]["storage_not_created_by_attempt"])
self.assertFalse(result["cleanup"]["remote_cleanup_unknown"])
def test_upload_nonzero_cleanup_removes_only_confirmed_owned_partial_directory(self):
self.fake.upload_failure = True
with self.assertRaises(TransportError):
self.client.submit()
self.assertEqual(len(self.fake.directories), 1)
foreign = "personal:/another-owned-directory"
self.fake.directories.add(foreign)
self.assertEqual(self.fake.count("run"), 0)
result = self.restart().cleanup(authorized=True)
self.assertEqual(result["phase"], "CLEANED")
self.assertEqual(self.fake.directories, {foreign})
self.assertEqual(self.fake.count("describe"), 0)
self.assertEqual(self.fake.count("rm"), 0)
self.assertEqual(self.fake.calls[-1], ["salty", "storage", "rm", "-r", result["remote_directory"]])
self.assertFalse((self.directory / "payload").exists())
def test_upload_timeout_does_not_delete_anything_or_allow_submit_retry(self):
self.fake.upload_timeout = True
with self.assertRaises(AmbiguousOperation):
self.client.submit()
before = list(self.fake.calls)
with self.assertRaises(AmbiguousOperation):
self.restart().cleanup(authorized=True)
with self.assertRaises(AmbiguousOperation):
self.restart().submit()
self.assertEqual(self.fake.calls, before)
self.assertEqual(len(self.fake.directories), 1)
self.assertTrue((self.directory / "payload").is_dir())
def test_failed_exclusive_mkdir_cannot_delete_foreign_uuid_directory(self):
remote = self.client.receipt["remote_directory"]
self.fake.directories.add(remote) # Target already existed: not ours.
with self.assertRaises(TransportError):
self.client.submit()
before = list(self.fake.calls)
with self.assertRaises(AmbiguousOperation):
self.restart().cleanup(authorized=True)
result = self.saved()
self.assertEqual(result["phase"], "UNRECONCILED")
self.assertTrue(result["cleanup"]["remote_cleanup_unknown"])
self.assertFalse(result["cleanup"]["storage_removed"])
self.assertNotIn("storage_not_created_by_attempt", result["cleanup"])
self.assertEqual(self.fake.calls, before)
self.assertEqual(self.fake.directories, {remote})
self.assertFalse((self.directory / "payload").exists())
def test_mkdir_nonzero_or_timeout_after_server_create_remains_unreconciled(self):
for mode in ("nonzero", "timeout"):
with self.subTest(mode=mode):
fake = FakeCli()
fake.mkdir_failure_after_create = mode == "nonzero"
fake.mkdir_timeout_after_create = mode == "timeout"
directory = self.root / ("mkdir-" + mode)
client = SaltyClient.prepare(directory, image=IMAGE, bundle=self.bundle,
worker=self.worker, command=["python3", "worker.py"], runner=fake)
with self.assertRaises(TransportError):
client.submit()
remote = client.receipt["remote_directory"]
self.assertEqual(fake.directories, {remote})
before = list(fake.calls)
with self.assertRaisesRegex(AmbiguousOperation, "unreconciled"):
client.cleanup(authorized=True)
receipt = client.receipt
self.assertEqual(receipt["phase"], "UNRECONCILED")
self.assertTrue(receipt["cleanup"]["remote_cleanup_unknown"])
self.assertFalse(receipt["cleanup"]["remote_owned"])
self.assertFalse(receipt["cleanup"]["storage_removed"])
self.assertTrue(receipt["cleanup"]["local_payload_removed"])
mkdir = next(entry for entry in receipt["commands"] if entry["label"] == "mkdir-exclusive")
self.assertEqual({path.name for path in directory.iterdir()},
{"receipt.json", "receipt.lock", mkdir["stdout"], mkdir["stderr"]})
restarted = SaltyClient(directory, runner=fake)
with self.assertRaises(AmbiguousOperation):
restarted.cleanup(authorized=True)
with self.assertRaisesRegex(TransportError, "cleanup already started"):
restarted.submit()
self.assertEqual(fake.calls, before)
self.assertEqual(fake.directories, {remote})
self.assertEqual(fake.count("run"), 0)
def test_unreconciled_mkdir_can_resume_partly_discarded_local_payload(self):
self.fake.mkdir_failure_after_create = True
with self.assertRaises(TransportError):
self.client.submit()
remote = self.client.receipt["remote_directory"]
original_unlink = Path.unlink
def fail_second_snapshot(path, *args, **kwargs):
if path.parent.name == "payload" and path.name == "worker.py":
raise OSError(errno.EIO, "fixture interrupted snapshot cleanup")
return original_unlink(path, *args, **kwargs)
with patch.object(Path, "unlink", fail_second_snapshot):
with self.assertRaises(OSError):
self.client.cleanup(authorized=True)
self.assertFalse((self.directory / "payload/bundle.tar").exists())
self.assertTrue((self.directory / "payload/worker.py").exists())
restarted = self.restart()
self.assertTrue(restarted.receipt["cleanup"]["local_payload_removal_started"])
before = list(self.fake.calls)
with self.assertRaises(AmbiguousOperation):
restarted.cleanup(authorized=True)
self.assertFalse((self.directory / "payload").exists())
self.assertEqual(restarted.receipt["phase"], "UNRECONCILED")
self.assertEqual(self.fake.calls, before)
self.assertEqual(self.fake.directories, {remote})
def test_terminal_cleanup_is_exact_authorized_and_leaves_compact_receipt(self):
submitted = self.client.submit()
remote = submitted["remote_directory"]
self.fake.jobs[JOB] = "succeeded"
result = self.root / "result.tar"
fetched = self.client.fetch("result.tar", result)
self.assertEqual(fetched["state"], "COMPLETE")
self.assertEqual(result.read_bytes(), b"verified-by-caller-result")
with self.assertRaises(TransportError):
self.client.cleanup()
cleaned = self.client.cleanup(authorized=True)
self.assertEqual(cleaned["phase"], "CLEANED")
self.assertEqual(self.fake.jobs, {})
self.assertEqual(self.fake.directories, set())
self.assertIn(["salty", "rm", JOB], self.fake.calls)
self.assertIn(["salty", "storage", "rm", "-r", remote], self.fake.calls)
self.assertEqual({p.name for p in self.directory.iterdir()}, {"receipt.json", "receipt.lock"})
self.assertTrue(result.is_file())
calls_before = len(self.fake.calls)
self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED")
self.assertEqual(len(self.fake.calls), calls_before)
def test_failed_job_cleanup_also_requires_exact_terminal_identity(self):
self.client.submit()
self.fake.jobs[JOB] = "failed"
self.assertEqual(self.client.cleanup(authorized=True)["cleanup"]["terminal_state"], "failed")
def test_unknown_cleanup_does_not_retry_delete(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
self.fake.cleanup_timeout = True
with self.assertRaises(AmbiguousOperation):
self.client.cleanup(authorized=True)
commands_before = list(self.fake.calls)
with self.assertRaises(AmbiguousOperation):
self.restart().cleanup(authorized=True)
self.assertEqual(self.fake.calls, commands_before)
self.assertTrue((self.directory / "payload").is_dir())
def test_terminal_job_does_not_authorize_deletion_with_prior_started_cli(self):
self.client.submit()
self.fake.jobs[JOB] = 'succeeded'
receipt = self.saved()
stdout, stderr = 'crash-window.stdout.log', 'crash-window.stderr.log'
(self.directory / stdout).write_text('still writable')
(self.directory / stderr).write_text('')
receipt['commands'].append({'label': 'describe', 'state': 'STARTED',
'argv': ['salty', 'describe', JOB], 'stdout': stdout, 'stderr': stderr})
(self.directory / 'receipt.json').write_text(json.dumps(receipt))
before = list(self.fake.calls)
with self.assertRaisesRegex(AmbiguousOperation, 'unresolved CLI'):
self.restart().cleanup(authorized=True)
self.assertEqual(self.fake.calls, before)
self.assertIn(JOB, self.fake.jobs)
self.assertIn(receipt['remote_directory'], self.fake.directories)
self.assertEqual((self.directory / stdout).read_text(), 'still writable')
self.assertTrue((self.directory / 'payload').is_dir())
def test_prior_unknown_read_keeps_terminal_job_and_storage_untouched(self):
self.client.submit()
self.fake.describe_drift = True
with self.assertRaises(FormatError):
self.client.poll()
receipt = self.saved()
receipt['commands'][-1]['state'] = 'UNKNOWN'
(self.directory / 'receipt.json').write_text(json.dumps(receipt))
self.fake.jobs[JOB] = 'succeeded'
before = list(self.fake.calls)
with self.assertRaises(AmbiguousOperation):
self.restart().cleanup(authorized=True)
self.assertEqual(self.fake.calls, before)
self.assertIn(JOB, self.fake.jobs)
def test_partial_cleanup_resume_skips_already_deleted_job(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
original_logs = {path.name: path.read_bytes() for path in self.directory.glob("*.log")}
# A normal completion supplies the persisted hashes. Roll the local
# receipt back to that exact post-command/pre-flag window, leaving the
# fixture's job removed. Storage was not yet removed in that window.
self.client.cleanup(authorized=True)
receipt = self.saved()
removed = next(entry for entry in receipt["commands"] if entry["label"] == "remove-terminal-job")
# Restore that command's fixture logs from its known empty payload.
for key in ("stdout", "stderr"):
(self.directory / removed[key]).write_text("")
removed.pop("logs_removed", None)
for name, content in original_logs.items():
(self.directory / name).write_bytes(content)
receipt["commands"] = receipt["commands"][:receipt["commands"].index(removed) + 1]
receipt["cleanup"] = {"operation_state": "UNKNOWN", "operation": "remove-terminal-job",
"authorized": True, "terminal_state": "succeeded"}
receipt["phase"] = "TERMINAL"
# Snapshots are present in the simulated crash window.
(self.directory / "payload").mkdir()
(self.directory / "payload/bundle.tar").write_bytes(self.bundle.read_bytes())
(self.directory / "payload/worker.py").write_bytes(self.worker.read_bytes())
self.fake.directories.add(receipt["remote_directory"])
(self.directory / "receipt.json").write_text(json.dumps(receipt))
before = len(self.fake.calls)
result = self.restart().cleanup(authorized=True)
self.assertEqual(result["phase"], "CLEANED")
calls = self.fake.calls[before:]
self.assertFalse(any(call[1] in {"describe", "rm"} for call in calls))
self.assertEqual(calls, [["salty", "storage", "rm", "-r", receipt["remote_directory"]]])
self.assertTrue(result["cleanup"]["job_removed"])
self.assertTrue(result["cleanup"]["storage_removed"])
self.assertTrue(result["cleanup"]["local_payload_removed"])
def test_format_drift_does_not_guess_terminal_or_delete(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
self.fake.describe_drift = True
with self.assertRaises(FormatError):
self.client.poll()
with self.assertRaises(FormatError):
self.client.cleanup(authorized=True)
self.assertIn(JOB, self.fake.jobs)
self.assertEqual(self.fake.count("rm"), 0)
def test_download_guards_and_no_overwrite(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
for unsafe in ("../another", "/data/shared/result", "./result.tar", "personal:/x", ".", "worker.py"):
with self.subTest(unsafe=unsafe), self.assertRaises(ValueError):
self.client.fetch(unsafe, self.root / "result.tar")
destination = self.root / "result.tar"
self.client.fetch("result.tar", destination)
with self.assertRaises(ValueError):
self.client.fetch("result.tar", destination)
with self.assertRaises(AmbiguousOperation):
self.client.fetch("result.tar", self.root / "second.tar")
def test_different_receipts_atomically_publish_one_destination_without_overwrite(self):
other_fake = FakeCli()
other_fake.download_bytes = b"independent-other-result"
other_directory = self.root / "other-receipt"
other = SaltyClient.prepare(other_directory, image=IMAGE, bundle=self.bundle,
worker=self.worker, command=["python3", "worker.py"], runner=other_fake)
for client, fake in ((self.client, self.fake), (other, other_fake)):
client.submit()
fake.jobs[JOB] = "succeeded"
barrier = threading.Barrier(2)
self.fake.download_barrier = other_fake.download_barrier = barrier
destination = self.root / "shared-result.tar"
def fetch(client):
try:
return client.fetch("result.tar", destination)
except TransportError as exc:
return exc
with ThreadPoolExecutor(max_workers=2) as pool:
futures = [pool.submit(fetch, client) for client in (self.client, other)]
results = [future.result(timeout=10) for future in futures]
successes = [result for result in results if isinstance(result, dict)]
failures = [result for result in results if isinstance(result, TransportError)]
self.assertEqual(len(successes), 1)
self.assertEqual(len(failures), 1)
self.assertIn("destination already exists", str(failures[0]))
self.assertEqual(hashlib.sha256(destination.read_bytes()).hexdigest(), successes[0]["sha256"])
receipts = [client.receipt for client in (self.client, other)]
self.assertEqual({receipt["downloads"][0]["state"] for receipt in receipts},
{"COMPLETE", "PUBLISH_FAILED"})
loser = next(receipt["downloads"][0] for receipt in receipts
if receipt["downloads"][0]["state"] == "PUBLISH_FAILED")
self.assertEqual(loser["publication_errno"], errno.EEXIST)
for client, directory in ((self.client, self.directory), (other, other_directory)):
self.assertEqual(list(directory.glob("*.part")), [])
downloaded = next(call for call in client.runner.calls
if call[1:3] == ["storage", "cp"] and call[3].startswith("personal:/"))
self.assertEqual(Path(downloaded[-1]).parent, directory.resolve())
self.assertNotEqual(downloaded[-1], str(destination))
self.assertEqual(client.cleanup(authorized=True)["phase"], "CLEANED")
self.assertEqual({path.name for path in directory.iterdir()}, {"receipt.json", "receipt.lock"})
self.assertEqual(hashlib.sha256(destination.read_bytes()).hexdigest(), successes[0]["sha256"])
def test_cross_filesystem_publication_has_no_copy_fallback_and_is_cleanable(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
destination = self.root / "cross-filesystem-result.tar"
with patch("harness.salty_client.os.link", side_effect=OSError(errno.EXDEV, "fixture cross device")):
with self.assertRaisesRegex(TransportError, "crosses filesystems"):
self.client.fetch("result.tar", destination)
download = self.client.receipt["downloads"][0]
self.assertEqual(download["state"], "PUBLISH_FAILED")
self.assertEqual(download["publication_errno"], errno.EXDEV)
self.assertTrue(download["temporary_removed"])
self.assertFalse(destination.exists())
self.assertEqual(list(self.directory.glob("*.part")), [])
self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED")
def test_completed_publication_cleanup_removes_only_its_leftover_temporary_link(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
destination = self.root / "published-result.tar"
original_unlink = Path.unlink
def fail_temporary_unlink(path, *args, **kwargs):
if path.suffix == ".part":
raise OSError(errno.EIO, "fixture interrupted local unlink")
return original_unlink(path, *args, **kwargs)
with patch.object(Path, "unlink", fail_temporary_unlink):
with self.assertRaises(OSError):
self.client.fetch("result.tar", destination)
download = self.client.receipt["downloads"][0]
self.assertEqual(download["state"], "COMPLETE")
self.assertNotIn("temporary_removed", download)
self.assertEqual(len(list(self.directory.glob("*.part"))), 1)
self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED")
self.assertEqual(list(self.directory.glob("*.part")), [])
self.assertEqual(destination.read_bytes(), self.fake.download_bytes)
def test_mutated_remote_path_cannot_delete_shared_or_root(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
receipt = self.saved()
for unsafe in ("personal:/", "shared:/", "personal:/someone-else"):
receipt["remote_directory"] = unsafe
(self.directory / "receipt.json").write_text(json.dumps(receipt))
with self.subTest(unsafe=unsafe), self.assertRaises(ValueError):
self.client.cleanup(authorized=True)
self.assertEqual(self.fake.count("rm"), 0)
def test_mutated_job_id_cannot_delete_another_complete_uuid(self):
self.client.submit()
self.fake.jobs[JOB] = "succeeded"
receipt = self.saved()
other = "salty-0ac12d34-1111-4222-8333-0123456789ac"
self.fake.jobs[other] = "succeeded"
receipt["job_id"] = other
(self.directory / "receipt.json").write_text(json.dumps(receipt))
with self.assertRaisesRegex(ValueError, "job ID differs"):
self.client.cleanup(authorized=True)
self.assertEqual(self.fake.count("rm"), 0)
self.assertIn(other, self.fake.jobs)
def test_immutable_image_and_non_sensitive_environment_gate(self):
for image in ("runtime:latest", "https://registry.example/x", "registry.example/x:latest",
"registry.example/x@sha256:abc", "registry.example/user:password/x@sha256:" + "a" * 64):
with self.subTest(image=image), self.assertRaises(ValueError):
validate_image(image)
self.assertEqual(validate_image(IMAGE), IMAGE)
for env in ({"TOKEN": "secret"}, {"OMP_NUM_THREADS": "$(cat file)"},
{"PYTHONUNBUFFERED": "123"}, {"OPENBLAS_NUM_THREADS": "0"}):
with self.subTest(env=env), self.assertRaises(ValueError):
self.prepare(env=env)
if __name__ == "__main__":
unittest.main()