把本地工作区里编译上云的改动单独拿出来,放到当前生产代码(c2c52ed)上: 新增 salty_client / salty_stage / salty_stage_worker 及两组测试,并在 stagerun、pipeline、stages/base、s06_hbm 里接线。只有配置里写了 execution.salty.enabled: true 且 stages 含 05-compile 时才生效, 现有配置不受影响。 合到生产代码上时补的衔接: - 编译次数上限:已记录过提交的 Salty 编译只做找回、绝不重提,不受上限 阻拦;只准备好、从未提交的照常拒绝(新增 salty_stage.dispatch_recorded)。 - selected() 与 cleanup_state 读取在上下文缺字段时按"未启用 / VERIFIED"处理。 未包含:进程优雅退出(owned_process / namespace_owner 等)和其余板卡侧改动, 仍留在本地工作区。 测试(Linux, python3.10):test_salty_client、test_salty_stage 全过; stagerun/pipeline/阶段相关的 13 个已有测试与 c2c52ed 结果一致。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
667 lines
34 KiB
Python
667 lines
34 KiB
Python
"""Salty transport contracts against a stateful fixture CLI, never the cluster.
|
|
|
|
Run: python3 -B -m unittest scripts.test_salty_client
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import errno
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness.salty_client import (AmbiguousOperation, CliRunner, CommandResult, FormatError,
|
|
SaltyClient, TransportError, validate_image)
|
|
|
|
|
|
IMAGE = "registry.example.test/private/runtime@sha256:" + "a" * 64
|
|
JOB = "salty-0ac12d34-1111-4222-8333-0123456789ab"
|
|
|
|
|
|
class FakeCli:
|
|
"""Mutable remote state captures whether a single submission took effect."""
|
|
|
|
def __init__(self):
|
|
self.calls = []
|
|
self.directories = set()
|
|
self.uploads = {}
|
|
self.jobs = {}
|
|
self.run_mode = "success"
|
|
self.describe_drift = False
|
|
self.cleanup_timeout = False
|
|
self.version_failure = False
|
|
self.upload_failure = False
|
|
self.upload_timeout = False
|
|
self.mkdir_failure_after_create = False
|
|
self.mkdir_timeout_after_create = False
|
|
self.download_barrier = None
|
|
self.download_bytes = b"verified-by-caller-result"
|
|
self.check_before_run = None
|
|
|
|
def __call__(self, argv, stdout, stderr, timeout):
|
|
self.calls.append(list(argv))
|
|
args = list(argv[1:])
|
|
output, rc = "", 0
|
|
if args == ["version"]:
|
|
output = "salty 0.2.0\n"
|
|
if self.version_failure:
|
|
rc = 1
|
|
elif args[:2] == ["storage", "mkdir"]:
|
|
if args[2] in self.directories:
|
|
rc = 1
|
|
else:
|
|
self.directories.add(args[2])
|
|
if self.mkdir_failure_after_create:
|
|
rc = 1
|
|
if self.mkdir_timeout_after_create:
|
|
stdout.write_text("")
|
|
stderr.write_text("")
|
|
return CommandResult(None, pid=5123, timed_out=True, process_left_running=True)
|
|
elif args[:2] == ["storage", "cp"]:
|
|
if args[2].startswith("personal:/"):
|
|
Path(args[3]).write_bytes(self.download_bytes)
|
|
if self.download_barrier:
|
|
self.download_barrier.wait(timeout=5)
|
|
else:
|
|
self.uploads[args[3]] = Path(args[2]).read_bytes()
|
|
if self.upload_failure:
|
|
rc = 1
|
|
if self.upload_timeout:
|
|
stdout.write_text("")
|
|
stderr.write_text("")
|
|
return CommandResult(None, pid=6123, timed_out=True, process_left_running=True)
|
|
elif args[0] == "run":
|
|
if self.check_before_run:
|
|
self.check_before_run()
|
|
self.jobs[JOB] = "pending"
|
|
output = "已提交作业: " + JOB + "\n"
|
|
if self.run_mode == "timeout":
|
|
stdout.write_text(output)
|
|
stderr.write_text("")
|
|
return CommandResult(None, pid=9123, timed_out=True, process_left_running=True)
|
|
if self.run_mode == "failed":
|
|
rc = 1 # Server may still have accepted it.
|
|
if self.run_mode == "format":
|
|
output = "submission accepted but no parseable ID\n"
|
|
if self.run_mode == "exception":
|
|
raise RuntimeError("fixture transport lost after acceptance")
|
|
elif args[0] == "describe":
|
|
state = self.jobs[args[1]]
|
|
output = "ID: %s\n状态: %s\n" % (args[1], state)
|
|
if self.describe_drift:
|
|
output = "ID: %s\nStatus: %s\n" % (args[1], state)
|
|
elif args[0] == "rm":
|
|
if self.jobs[args[1]] not in {"succeeded", "failed"}:
|
|
raise AssertionError("must never remove an active fixture job")
|
|
del self.jobs[args[1]]
|
|
elif args[:3] == ["storage", "rm", "-r"]:
|
|
self.directories.remove(args[3])
|
|
if self.cleanup_timeout:
|
|
stdout.write_text("")
|
|
stderr.write_text("")
|
|
return CommandResult(None, pid=8123, timed_out=True, process_left_running=True)
|
|
else:
|
|
raise AssertionError("unexpected fixture command: " + repr(args))
|
|
stdout.write_text(output)
|
|
stderr.write_text("")
|
|
return CommandResult(rc, pid=7123)
|
|
|
|
def count(self, command):
|
|
return sum(call[1] == command for call in self.calls)
|
|
|
|
|
|
class TransportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory(prefix="salty-transport-contract-")
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name)
|
|
self.bundle = self.root / "input.tar"
|
|
self.worker = self.root / "input.py"
|
|
self.bundle.write_bytes(b"credential-free fixture bundle")
|
|
self.worker.write_text("print('fixture worker')\n")
|
|
self.fake = FakeCli()
|
|
self.directory = self.root / "receipt"
|
|
self.client = self.prepare()
|
|
|
|
def prepare(self, **overrides):
|
|
options = {"image": IMAGE, "bundle": self.bundle, "worker": self.worker,
|
|
"command": ["python3", "-B", "{container_root}/worker.py",
|
|
"--bundle", "{container_root}/bundle.tar"],
|
|
"env": {"OMP_NUM_THREADS": "1"}, "runner": self.fake}
|
|
options.update(overrides)
|
|
return SaltyClient.prepare(self.directory, **options)
|
|
|
|
def restart(self):
|
|
return SaltyClient(self.directory, runner=self.fake)
|
|
|
|
def saved(self):
|
|
return json.loads((self.directory / "receipt.json").read_text())
|
|
|
|
def test_prepare_has_no_cli_effect_and_exclusive_snapshots(self):
|
|
self.assertEqual(self.fake.calls, [])
|
|
self.assertEqual(self.saved()["phase"], "PREPARED")
|
|
self.bundle.write_bytes(b"caller later edits original")
|
|
self.assertEqual((self.directory / "payload/bundle.tar").read_bytes(), b"credential-free fixture bundle")
|
|
with self.assertRaises(FileExistsError):
|
|
self.prepare()
|
|
self.assertEqual(self.fake.calls, [])
|
|
|
|
def test_submission_intent_is_durable_before_run_and_restart_never_resubmits(self):
|
|
def check():
|
|
receipt = self.saved()
|
|
self.assertEqual(receipt["phase"], "SUBMITTING")
|
|
self.assertEqual(receipt["submission_state"], "UNKNOWN")
|
|
self.assertEqual(receipt["commands"][-1]["label"], "submit")
|
|
self.assertEqual(receipt["commands"][-1]["state"], "STARTED")
|
|
self.fake.check_before_run = check
|
|
receipt = self.client.submit()
|
|
self.assertEqual(receipt["job_id"], JOB)
|
|
self.assertEqual(self.fake.count("run"), 1)
|
|
restarted = self.restart()
|
|
self.assertEqual(restarted.submit()["job_id"], JOB)
|
|
self.assertEqual(restarted.poll()["job_state"], "pending")
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
self.assertEqual(self.restart().poll()["job_state"], "succeeded")
|
|
self.assertEqual(self.fake.count("run"), 1)
|
|
self.assertTrue(all(call[2] == JOB for call in self.fake.calls if call[1] == "describe"))
|
|
|
|
def test_resources_use_platform_defaults_unless_explicit(self):
|
|
self.client.submit()
|
|
run = next(call for call in self.fake.calls if call[1] == "run")
|
|
self.assertNotIn("--cpu", run)
|
|
self.assertNotIn("--memory", run)
|
|
explicit_fake = FakeCli()
|
|
explicit = SaltyClient.prepare(self.root / "explicit", image=IMAGE, bundle=self.bundle,
|
|
worker=self.worker, command=["python3", "worker.py"],
|
|
cpu=8, memory="16Gi", runner=explicit_fake)
|
|
explicit.submit()
|
|
run = next(call for call in explicit_fake.calls if call[1] == "run")
|
|
self.assertEqual(run[run.index("--cpu") + 1], "8")
|
|
self.assertEqual(run[run.index("--memory") + 1], "16Gi")
|
|
|
|
def test_concurrent_call_cannot_cross_receipt_lock(self):
|
|
def check():
|
|
with self.assertRaisesRegex(TransportError, "another caller"):
|
|
self.restart().submit()
|
|
self.fake.check_before_run = check
|
|
self.client.submit()
|
|
self.assertEqual(self.fake.count("run"), 1)
|
|
|
|
def test_changed_snapshot_is_detected_on_restart_before_remote_operations(self):
|
|
(self.directory / "payload/bundle.tar").write_bytes(b"replaced bytes")
|
|
with self.assertRaisesRegex(ValueError, "payload/source hash"):
|
|
self.restart()
|
|
self.assertEqual(self.fake.calls, [])
|
|
|
|
def test_unknown_submissions_never_retry_or_cleanup(self):
|
|
for mode in ("timeout", "failed", "format", "exception"):
|
|
with self.subTest(mode=mode):
|
|
root = self.root / mode
|
|
fake = FakeCli()
|
|
fake.run_mode = mode
|
|
client = SaltyClient.prepare(root, image=IMAGE, bundle=self.bundle,
|
|
worker=self.worker, command=["python3", "worker.py"], runner=fake)
|
|
with self.assertRaises(TransportError):
|
|
client.submit()
|
|
restart = SaltyClient(root, runner=fake)
|
|
self.assertEqual(restart.receipt["phase"], "SUBMISSION_UNKNOWN")
|
|
with self.assertRaises(AmbiguousOperation):
|
|
restart.submit()
|
|
with self.assertRaises(AmbiguousOperation):
|
|
restart.cleanup(authorized=True)
|
|
self.assertEqual(fake.count("run"), 1)
|
|
self.assertIn(JOB, fake.jobs)
|
|
if mode == "timeout":
|
|
entry = restart.receipt["commands"][-1]
|
|
self.assertEqual(entry["local_pid"], 9123)
|
|
self.assertTrue(entry["process_left_running"])
|
|
|
|
def test_crash_after_completed_submission_recovers_recorded_exact_id(self):
|
|
self.client.submit()
|
|
receipt = self.saved()
|
|
receipt.update(job_id=None, phase="SUBMITTING", submission_state="UNKNOWN")
|
|
(self.directory / "receipt.json").write_text(json.dumps(receipt))
|
|
restarted = self.restart()
|
|
self.assertEqual(restarted.recover_recorded_submission()["job_id"], JOB)
|
|
self.assertEqual(restarted.poll()["job_state"], "pending")
|
|
self.assertEqual(self.fake.count("run"), 1)
|
|
|
|
def test_unknown_completed_process_cannot_be_inferred_from_stdout(self):
|
|
self.fake.run_mode = "timeout"
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.client.submit()
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.restart().recover_recorded_submission()
|
|
self.assertEqual(self.fake.count("run"), 1)
|
|
|
|
def test_cli_deadline_leaves_process_to_finish_naturally(self):
|
|
marker = self.root / "natural-completion.txt"
|
|
stdout, stderr = self.root / "natural.stdout", self.root / "natural.stderr"
|
|
runner = CliRunner()
|
|
code = "import pathlib,time; time.sleep(0.3); pathlib.Path(%r).write_text('naturally finished')" % str(marker)
|
|
starts = []
|
|
result = runner([sys.executable, "-B", "-c", code], stdout, stderr, 0.02,
|
|
on_started=lambda pid, identity: starts.append((pid, identity)))
|
|
self.assertTrue(result.timed_out)
|
|
self.assertTrue(result.process_left_running)
|
|
self.assertEqual(starts, [(result.pid, result.birth_identity)])
|
|
if sys.platform == 'linux':
|
|
self.assertEqual(result.birth_identity['status'], 'KNOWN')
|
|
self.assertGreater(result.birth_identity['start_ticks'], 0)
|
|
process = runner._unresolved[result.pid]
|
|
self.assertIsNone(process.poll())
|
|
self.assertEqual(process.wait(timeout=5), 0)
|
|
self.assertEqual(marker.read_text(), "naturally finished")
|
|
del runner._unresolved[result.pid]
|
|
|
|
def test_cli_interrupt_retains_started_birth_identity_in_receipt(self):
|
|
from types import SimpleNamespace
|
|
process = SimpleNamespace(pid=4567, poll=lambda: (_ for _ in ()).throw(KeyboardInterrupt()))
|
|
identity = {'pid': 4567, 'status': 'KNOWN', 'boot_id': 'fixture', 'start_ticks': 12}
|
|
runner = CliRunner()
|
|
self.client.runner = runner
|
|
with patch('harness.salty_client.subprocess.Popen', return_value=process), \
|
|
patch('harness.salty_client._cli_birth_identity', return_value=identity):
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.client.submit()
|
|
entry = self.saved()['commands'][0]
|
|
self.assertEqual(entry['state'], 'UNKNOWN')
|
|
self.assertEqual(entry['local_birth_identity'], identity)
|
|
self.assertEqual(entry['local_pid'], 4567)
|
|
self.assertTrue(entry['process_left_running'])
|
|
self.assertIs(runner._unresolved[4567], process)
|
|
|
|
def test_pending_and_running_are_never_removed(self):
|
|
self.client.submit()
|
|
for state in ("pending", "running"):
|
|
self.fake.jobs[JOB] = state
|
|
with self.assertRaisesRegex(TransportError, "cannot be stopped or deleted"):
|
|
self.client.cleanup(authorized=True)
|
|
self.assertEqual(self.fake.count("rm"), 0)
|
|
self.assertEqual(len(self.fake.directories), 1)
|
|
self.assertEqual(self.saved()["cleanup"], {})
|
|
|
|
def test_prepared_without_cli_can_discard_only_local_temporary_files(self):
|
|
result = self.client.cleanup(authorized=True)
|
|
self.assertEqual(result["phase"], "CLEANED")
|
|
self.assertTrue(result["cleanup"]["never_submitted"])
|
|
self.assertEqual(self.fake.calls, [])
|
|
self.assertEqual({p.name for p in self.directory.iterdir()}, {"receipt.json", "receipt.lock"})
|
|
|
|
def test_version_failure_never_submitted_cleanup_has_no_remote_mutation(self):
|
|
self.fake.version_failure = True
|
|
with self.assertRaises(TransportError):
|
|
self.client.submit()
|
|
result = self.restart().cleanup(authorized=True)
|
|
self.assertEqual(result["phase"], "CLEANED")
|
|
self.assertEqual(self.fake.calls, [["salty", "version"]])
|
|
self.assertEqual(self.fake.directories, set())
|
|
self.assertFalse((self.directory / "payload").exists())
|
|
self.assertTrue(result["cleanup"]["storage_not_created_by_attempt"])
|
|
self.assertFalse(result["cleanup"]["remote_cleanup_unknown"])
|
|
|
|
def test_upload_nonzero_cleanup_removes_only_confirmed_owned_partial_directory(self):
|
|
self.fake.upload_failure = True
|
|
with self.assertRaises(TransportError):
|
|
self.client.submit()
|
|
self.assertEqual(len(self.fake.directories), 1)
|
|
foreign = "personal:/another-owned-directory"
|
|
self.fake.directories.add(foreign)
|
|
self.assertEqual(self.fake.count("run"), 0)
|
|
result = self.restart().cleanup(authorized=True)
|
|
self.assertEqual(result["phase"], "CLEANED")
|
|
self.assertEqual(self.fake.directories, {foreign})
|
|
self.assertEqual(self.fake.count("describe"), 0)
|
|
self.assertEqual(self.fake.count("rm"), 0)
|
|
self.assertEqual(self.fake.calls[-1], ["salty", "storage", "rm", "-r", result["remote_directory"]])
|
|
self.assertFalse((self.directory / "payload").exists())
|
|
|
|
def test_upload_timeout_does_not_delete_anything_or_allow_submit_retry(self):
|
|
self.fake.upload_timeout = True
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.client.submit()
|
|
before = list(self.fake.calls)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.restart().cleanup(authorized=True)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.restart().submit()
|
|
self.assertEqual(self.fake.calls, before)
|
|
self.assertEqual(len(self.fake.directories), 1)
|
|
self.assertTrue((self.directory / "payload").is_dir())
|
|
|
|
def test_failed_exclusive_mkdir_cannot_delete_foreign_uuid_directory(self):
|
|
remote = self.client.receipt["remote_directory"]
|
|
self.fake.directories.add(remote) # Target already existed: not ours.
|
|
with self.assertRaises(TransportError):
|
|
self.client.submit()
|
|
before = list(self.fake.calls)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.restart().cleanup(authorized=True)
|
|
result = self.saved()
|
|
self.assertEqual(result["phase"], "UNRECONCILED")
|
|
self.assertTrue(result["cleanup"]["remote_cleanup_unknown"])
|
|
self.assertFalse(result["cleanup"]["storage_removed"])
|
|
self.assertNotIn("storage_not_created_by_attempt", result["cleanup"])
|
|
self.assertEqual(self.fake.calls, before)
|
|
self.assertEqual(self.fake.directories, {remote})
|
|
self.assertFalse((self.directory / "payload").exists())
|
|
|
|
def test_mkdir_nonzero_or_timeout_after_server_create_remains_unreconciled(self):
|
|
for mode in ("nonzero", "timeout"):
|
|
with self.subTest(mode=mode):
|
|
fake = FakeCli()
|
|
fake.mkdir_failure_after_create = mode == "nonzero"
|
|
fake.mkdir_timeout_after_create = mode == "timeout"
|
|
directory = self.root / ("mkdir-" + mode)
|
|
client = SaltyClient.prepare(directory, image=IMAGE, bundle=self.bundle,
|
|
worker=self.worker, command=["python3", "worker.py"], runner=fake)
|
|
with self.assertRaises(TransportError):
|
|
client.submit()
|
|
remote = client.receipt["remote_directory"]
|
|
self.assertEqual(fake.directories, {remote})
|
|
before = list(fake.calls)
|
|
with self.assertRaisesRegex(AmbiguousOperation, "unreconciled"):
|
|
client.cleanup(authorized=True)
|
|
receipt = client.receipt
|
|
self.assertEqual(receipt["phase"], "UNRECONCILED")
|
|
self.assertTrue(receipt["cleanup"]["remote_cleanup_unknown"])
|
|
self.assertFalse(receipt["cleanup"]["remote_owned"])
|
|
self.assertFalse(receipt["cleanup"]["storage_removed"])
|
|
self.assertTrue(receipt["cleanup"]["local_payload_removed"])
|
|
mkdir = next(entry for entry in receipt["commands"] if entry["label"] == "mkdir-exclusive")
|
|
self.assertEqual({path.name for path in directory.iterdir()},
|
|
{"receipt.json", "receipt.lock", mkdir["stdout"], mkdir["stderr"]})
|
|
restarted = SaltyClient(directory, runner=fake)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
restarted.cleanup(authorized=True)
|
|
with self.assertRaisesRegex(TransportError, "cleanup already started"):
|
|
restarted.submit()
|
|
self.assertEqual(fake.calls, before)
|
|
self.assertEqual(fake.directories, {remote})
|
|
self.assertEqual(fake.count("run"), 0)
|
|
|
|
def test_unreconciled_mkdir_can_resume_partly_discarded_local_payload(self):
|
|
self.fake.mkdir_failure_after_create = True
|
|
with self.assertRaises(TransportError):
|
|
self.client.submit()
|
|
remote = self.client.receipt["remote_directory"]
|
|
original_unlink = Path.unlink
|
|
|
|
def fail_second_snapshot(path, *args, **kwargs):
|
|
if path.parent.name == "payload" and path.name == "worker.py":
|
|
raise OSError(errno.EIO, "fixture interrupted snapshot cleanup")
|
|
return original_unlink(path, *args, **kwargs)
|
|
|
|
with patch.object(Path, "unlink", fail_second_snapshot):
|
|
with self.assertRaises(OSError):
|
|
self.client.cleanup(authorized=True)
|
|
self.assertFalse((self.directory / "payload/bundle.tar").exists())
|
|
self.assertTrue((self.directory / "payload/worker.py").exists())
|
|
restarted = self.restart()
|
|
self.assertTrue(restarted.receipt["cleanup"]["local_payload_removal_started"])
|
|
before = list(self.fake.calls)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
restarted.cleanup(authorized=True)
|
|
self.assertFalse((self.directory / "payload").exists())
|
|
self.assertEqual(restarted.receipt["phase"], "UNRECONCILED")
|
|
self.assertEqual(self.fake.calls, before)
|
|
self.assertEqual(self.fake.directories, {remote})
|
|
|
|
def test_terminal_cleanup_is_exact_authorized_and_leaves_compact_receipt(self):
|
|
submitted = self.client.submit()
|
|
remote = submitted["remote_directory"]
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
result = self.root / "result.tar"
|
|
fetched = self.client.fetch("result.tar", result)
|
|
self.assertEqual(fetched["state"], "COMPLETE")
|
|
self.assertEqual(result.read_bytes(), b"verified-by-caller-result")
|
|
with self.assertRaises(TransportError):
|
|
self.client.cleanup()
|
|
cleaned = self.client.cleanup(authorized=True)
|
|
self.assertEqual(cleaned["phase"], "CLEANED")
|
|
self.assertEqual(self.fake.jobs, {})
|
|
self.assertEqual(self.fake.directories, set())
|
|
self.assertIn(["salty", "rm", JOB], self.fake.calls)
|
|
self.assertIn(["salty", "storage", "rm", "-r", remote], self.fake.calls)
|
|
self.assertEqual({p.name for p in self.directory.iterdir()}, {"receipt.json", "receipt.lock"})
|
|
self.assertTrue(result.is_file())
|
|
calls_before = len(self.fake.calls)
|
|
self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED")
|
|
self.assertEqual(len(self.fake.calls), calls_before)
|
|
|
|
def test_failed_job_cleanup_also_requires_exact_terminal_identity(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "failed"
|
|
self.assertEqual(self.client.cleanup(authorized=True)["cleanup"]["terminal_state"], "failed")
|
|
|
|
def test_unknown_cleanup_does_not_retry_delete(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
self.fake.cleanup_timeout = True
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.client.cleanup(authorized=True)
|
|
commands_before = list(self.fake.calls)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.restart().cleanup(authorized=True)
|
|
self.assertEqual(self.fake.calls, commands_before)
|
|
self.assertTrue((self.directory / "payload").is_dir())
|
|
|
|
def test_terminal_job_does_not_authorize_deletion_with_prior_started_cli(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = 'succeeded'
|
|
receipt = self.saved()
|
|
stdout, stderr = 'crash-window.stdout.log', 'crash-window.stderr.log'
|
|
(self.directory / stdout).write_text('still writable')
|
|
(self.directory / stderr).write_text('')
|
|
receipt['commands'].append({'label': 'describe', 'state': 'STARTED',
|
|
'argv': ['salty', 'describe', JOB], 'stdout': stdout, 'stderr': stderr})
|
|
(self.directory / 'receipt.json').write_text(json.dumps(receipt))
|
|
before = list(self.fake.calls)
|
|
with self.assertRaisesRegex(AmbiguousOperation, 'unresolved CLI'):
|
|
self.restart().cleanup(authorized=True)
|
|
self.assertEqual(self.fake.calls, before)
|
|
self.assertIn(JOB, self.fake.jobs)
|
|
self.assertIn(receipt['remote_directory'], self.fake.directories)
|
|
self.assertEqual((self.directory / stdout).read_text(), 'still writable')
|
|
self.assertTrue((self.directory / 'payload').is_dir())
|
|
|
|
def test_prior_unknown_read_keeps_terminal_job_and_storage_untouched(self):
|
|
self.client.submit()
|
|
self.fake.describe_drift = True
|
|
with self.assertRaises(FormatError):
|
|
self.client.poll()
|
|
receipt = self.saved()
|
|
receipt['commands'][-1]['state'] = 'UNKNOWN'
|
|
(self.directory / 'receipt.json').write_text(json.dumps(receipt))
|
|
self.fake.jobs[JOB] = 'succeeded'
|
|
before = list(self.fake.calls)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.restart().cleanup(authorized=True)
|
|
self.assertEqual(self.fake.calls, before)
|
|
self.assertIn(JOB, self.fake.jobs)
|
|
|
|
def test_partial_cleanup_resume_skips_already_deleted_job(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
original_logs = {path.name: path.read_bytes() for path in self.directory.glob("*.log")}
|
|
# A normal completion supplies the persisted hashes. Roll the local
|
|
# receipt back to that exact post-command/pre-flag window, leaving the
|
|
# fixture's job removed. Storage was not yet removed in that window.
|
|
self.client.cleanup(authorized=True)
|
|
receipt = self.saved()
|
|
removed = next(entry for entry in receipt["commands"] if entry["label"] == "remove-terminal-job")
|
|
# Restore that command's fixture logs from its known empty payload.
|
|
for key in ("stdout", "stderr"):
|
|
(self.directory / removed[key]).write_text("")
|
|
removed.pop("logs_removed", None)
|
|
for name, content in original_logs.items():
|
|
(self.directory / name).write_bytes(content)
|
|
receipt["commands"] = receipt["commands"][:receipt["commands"].index(removed) + 1]
|
|
receipt["cleanup"] = {"operation_state": "UNKNOWN", "operation": "remove-terminal-job",
|
|
"authorized": True, "terminal_state": "succeeded"}
|
|
receipt["phase"] = "TERMINAL"
|
|
# Snapshots are present in the simulated crash window.
|
|
(self.directory / "payload").mkdir()
|
|
(self.directory / "payload/bundle.tar").write_bytes(self.bundle.read_bytes())
|
|
(self.directory / "payload/worker.py").write_bytes(self.worker.read_bytes())
|
|
self.fake.directories.add(receipt["remote_directory"])
|
|
(self.directory / "receipt.json").write_text(json.dumps(receipt))
|
|
before = len(self.fake.calls)
|
|
result = self.restart().cleanup(authorized=True)
|
|
self.assertEqual(result["phase"], "CLEANED")
|
|
calls = self.fake.calls[before:]
|
|
self.assertFalse(any(call[1] in {"describe", "rm"} for call in calls))
|
|
self.assertEqual(calls, [["salty", "storage", "rm", "-r", receipt["remote_directory"]]])
|
|
self.assertTrue(result["cleanup"]["job_removed"])
|
|
self.assertTrue(result["cleanup"]["storage_removed"])
|
|
self.assertTrue(result["cleanup"]["local_payload_removed"])
|
|
|
|
def test_format_drift_does_not_guess_terminal_or_delete(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
self.fake.describe_drift = True
|
|
with self.assertRaises(FormatError):
|
|
self.client.poll()
|
|
with self.assertRaises(FormatError):
|
|
self.client.cleanup(authorized=True)
|
|
self.assertIn(JOB, self.fake.jobs)
|
|
self.assertEqual(self.fake.count("rm"), 0)
|
|
|
|
def test_download_guards_and_no_overwrite(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
for unsafe in ("../another", "/data/shared/result", "./result.tar", "personal:/x", ".", "worker.py"):
|
|
with self.subTest(unsafe=unsafe), self.assertRaises(ValueError):
|
|
self.client.fetch(unsafe, self.root / "result.tar")
|
|
destination = self.root / "result.tar"
|
|
self.client.fetch("result.tar", destination)
|
|
with self.assertRaises(ValueError):
|
|
self.client.fetch("result.tar", destination)
|
|
with self.assertRaises(AmbiguousOperation):
|
|
self.client.fetch("result.tar", self.root / "second.tar")
|
|
|
|
def test_different_receipts_atomically_publish_one_destination_without_overwrite(self):
|
|
other_fake = FakeCli()
|
|
other_fake.download_bytes = b"independent-other-result"
|
|
other_directory = self.root / "other-receipt"
|
|
other = SaltyClient.prepare(other_directory, image=IMAGE, bundle=self.bundle,
|
|
worker=self.worker, command=["python3", "worker.py"], runner=other_fake)
|
|
for client, fake in ((self.client, self.fake), (other, other_fake)):
|
|
client.submit()
|
|
fake.jobs[JOB] = "succeeded"
|
|
barrier = threading.Barrier(2)
|
|
self.fake.download_barrier = other_fake.download_barrier = barrier
|
|
destination = self.root / "shared-result.tar"
|
|
|
|
def fetch(client):
|
|
try:
|
|
return client.fetch("result.tar", destination)
|
|
except TransportError as exc:
|
|
return exc
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
futures = [pool.submit(fetch, client) for client in (self.client, other)]
|
|
results = [future.result(timeout=10) for future in futures]
|
|
successes = [result for result in results if isinstance(result, dict)]
|
|
failures = [result for result in results if isinstance(result, TransportError)]
|
|
self.assertEqual(len(successes), 1)
|
|
self.assertEqual(len(failures), 1)
|
|
self.assertIn("destination already exists", str(failures[0]))
|
|
self.assertEqual(hashlib.sha256(destination.read_bytes()).hexdigest(), successes[0]["sha256"])
|
|
receipts = [client.receipt for client in (self.client, other)]
|
|
self.assertEqual({receipt["downloads"][0]["state"] for receipt in receipts},
|
|
{"COMPLETE", "PUBLISH_FAILED"})
|
|
loser = next(receipt["downloads"][0] for receipt in receipts
|
|
if receipt["downloads"][0]["state"] == "PUBLISH_FAILED")
|
|
self.assertEqual(loser["publication_errno"], errno.EEXIST)
|
|
for client, directory in ((self.client, self.directory), (other, other_directory)):
|
|
self.assertEqual(list(directory.glob("*.part")), [])
|
|
downloaded = next(call for call in client.runner.calls
|
|
if call[1:3] == ["storage", "cp"] and call[3].startswith("personal:/"))
|
|
self.assertEqual(Path(downloaded[-1]).parent, directory.resolve())
|
|
self.assertNotEqual(downloaded[-1], str(destination))
|
|
self.assertEqual(client.cleanup(authorized=True)["phase"], "CLEANED")
|
|
self.assertEqual({path.name for path in directory.iterdir()}, {"receipt.json", "receipt.lock"})
|
|
self.assertEqual(hashlib.sha256(destination.read_bytes()).hexdigest(), successes[0]["sha256"])
|
|
|
|
def test_cross_filesystem_publication_has_no_copy_fallback_and_is_cleanable(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
destination = self.root / "cross-filesystem-result.tar"
|
|
with patch("harness.salty_client.os.link", side_effect=OSError(errno.EXDEV, "fixture cross device")):
|
|
with self.assertRaisesRegex(TransportError, "crosses filesystems"):
|
|
self.client.fetch("result.tar", destination)
|
|
download = self.client.receipt["downloads"][0]
|
|
self.assertEqual(download["state"], "PUBLISH_FAILED")
|
|
self.assertEqual(download["publication_errno"], errno.EXDEV)
|
|
self.assertTrue(download["temporary_removed"])
|
|
self.assertFalse(destination.exists())
|
|
self.assertEqual(list(self.directory.glob("*.part")), [])
|
|
self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED")
|
|
|
|
def test_completed_publication_cleanup_removes_only_its_leftover_temporary_link(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
destination = self.root / "published-result.tar"
|
|
original_unlink = Path.unlink
|
|
|
|
def fail_temporary_unlink(path, *args, **kwargs):
|
|
if path.suffix == ".part":
|
|
raise OSError(errno.EIO, "fixture interrupted local unlink")
|
|
return original_unlink(path, *args, **kwargs)
|
|
|
|
with patch.object(Path, "unlink", fail_temporary_unlink):
|
|
with self.assertRaises(OSError):
|
|
self.client.fetch("result.tar", destination)
|
|
download = self.client.receipt["downloads"][0]
|
|
self.assertEqual(download["state"], "COMPLETE")
|
|
self.assertNotIn("temporary_removed", download)
|
|
self.assertEqual(len(list(self.directory.glob("*.part"))), 1)
|
|
self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED")
|
|
self.assertEqual(list(self.directory.glob("*.part")), [])
|
|
self.assertEqual(destination.read_bytes(), self.fake.download_bytes)
|
|
|
|
def test_mutated_remote_path_cannot_delete_shared_or_root(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
receipt = self.saved()
|
|
for unsafe in ("personal:/", "shared:/", "personal:/someone-else"):
|
|
receipt["remote_directory"] = unsafe
|
|
(self.directory / "receipt.json").write_text(json.dumps(receipt))
|
|
with self.subTest(unsafe=unsafe), self.assertRaises(ValueError):
|
|
self.client.cleanup(authorized=True)
|
|
self.assertEqual(self.fake.count("rm"), 0)
|
|
|
|
def test_mutated_job_id_cannot_delete_another_complete_uuid(self):
|
|
self.client.submit()
|
|
self.fake.jobs[JOB] = "succeeded"
|
|
receipt = self.saved()
|
|
other = "salty-0ac12d34-1111-4222-8333-0123456789ac"
|
|
self.fake.jobs[other] = "succeeded"
|
|
receipt["job_id"] = other
|
|
(self.directory / "receipt.json").write_text(json.dumps(receipt))
|
|
with self.assertRaisesRegex(ValueError, "job ID differs"):
|
|
self.client.cleanup(authorized=True)
|
|
self.assertEqual(self.fake.count("rm"), 0)
|
|
self.assertIn(other, self.fake.jobs)
|
|
|
|
def test_immutable_image_and_non_sensitive_environment_gate(self):
|
|
for image in ("runtime:latest", "https://registry.example/x", "registry.example/x:latest",
|
|
"registry.example/x@sha256:abc", "registry.example/user:password/x@sha256:" + "a" * 64):
|
|
with self.subTest(image=image), self.assertRaises(ValueError):
|
|
validate_image(image)
|
|
self.assertEqual(validate_image(IMAGE), IMAGE)
|
|
for env in ({"TOKEN": "secret"}, {"OMP_NUM_THREADS": "$(cat file)"},
|
|
{"PYTHONUNBUFFERED": "123"}, {"OPENBLAS_NUM_THREADS": "0"}):
|
|
with self.subTest(env=env), self.assertRaises(ValueError):
|
|
self.prepare(env=env)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|