"""Salty transport contracts against a stateful fixture CLI, never the cluster. Run: python3 -B -m unittest scripts.test_salty_client """ from __future__ import annotations from concurrent.futures import ThreadPoolExecutor import errno import hashlib import json from pathlib import Path import sys import tempfile import threading import unittest from unittest.mock import patch from harness.salty_client import (AmbiguousOperation, CliRunner, CommandResult, FormatError, SaltyClient, TransportError, validate_image) IMAGE = "registry.example.test/private/runtime@sha256:" + "a" * 64 JOB = "salty-0ac12d34-1111-4222-8333-0123456789ab" class FakeCli: """Mutable remote state captures whether a single submission took effect.""" def __init__(self): self.calls = [] self.directories = set() self.uploads = {} self.jobs = {} self.run_mode = "success" self.describe_drift = False self.cleanup_timeout = False self.version_failure = False self.upload_failure = False self.upload_timeout = False self.mkdir_failure_after_create = False self.mkdir_timeout_after_create = False self.download_barrier = None self.download_bytes = b"verified-by-caller-result" self.check_before_run = None def __call__(self, argv, stdout, stderr, timeout): self.calls.append(list(argv)) args = list(argv[1:]) output, rc = "", 0 if args == ["version"]: output = "salty 0.2.0\n" if self.version_failure: rc = 1 elif args[:2] == ["storage", "mkdir"]: if args[2] in self.directories: rc = 1 else: self.directories.add(args[2]) if self.mkdir_failure_after_create: rc = 1 if self.mkdir_timeout_after_create: stdout.write_text("") stderr.write_text("") return CommandResult(None, pid=5123, timed_out=True, process_left_running=True) elif args[:2] == ["storage", "cp"]: if args[2].startswith("personal:/"): Path(args[3]).write_bytes(self.download_bytes) if self.download_barrier: self.download_barrier.wait(timeout=5) else: self.uploads[args[3]] = Path(args[2]).read_bytes() if self.upload_failure: rc = 1 if self.upload_timeout: stdout.write_text("") stderr.write_text("") return CommandResult(None, pid=6123, timed_out=True, process_left_running=True) elif args[0] == "run": if self.check_before_run: self.check_before_run() self.jobs[JOB] = "pending" output = "已提交作业: " + JOB + "\n" if self.run_mode == "timeout": stdout.write_text(output) stderr.write_text("") return CommandResult(None, pid=9123, timed_out=True, process_left_running=True) if self.run_mode == "failed": rc = 1 # Server may still have accepted it. if self.run_mode == "format": output = "submission accepted but no parseable ID\n" if self.run_mode == "exception": raise RuntimeError("fixture transport lost after acceptance") elif args[0] == "describe": state = self.jobs[args[1]] output = "ID: %s\n状态: %s\n" % (args[1], state) if self.describe_drift: output = "ID: %s\nStatus: %s\n" % (args[1], state) elif args[0] == "rm": if self.jobs[args[1]] not in {"succeeded", "failed"}: raise AssertionError("must never remove an active fixture job") del self.jobs[args[1]] elif args[:3] == ["storage", "rm", "-r"]: self.directories.remove(args[3]) if self.cleanup_timeout: stdout.write_text("") stderr.write_text("") return CommandResult(None, pid=8123, timed_out=True, process_left_running=True) else: raise AssertionError("unexpected fixture command: " + repr(args)) stdout.write_text(output) stderr.write_text("") return CommandResult(rc, pid=7123) def count(self, command): return sum(call[1] == command for call in self.calls) class TransportTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory(prefix="salty-transport-contract-") self.addCleanup(self.temporary.cleanup) self.root = Path(self.temporary.name) self.bundle = self.root / "input.tar" self.worker = self.root / "input.py" self.bundle.write_bytes(b"credential-free fixture bundle") self.worker.write_text("print('fixture worker')\n") self.fake = FakeCli() self.directory = self.root / "receipt" self.client = self.prepare() def prepare(self, **overrides): options = {"image": IMAGE, "bundle": self.bundle, "worker": self.worker, "command": ["python3", "-B", "{container_root}/worker.py", "--bundle", "{container_root}/bundle.tar"], "env": {"OMP_NUM_THREADS": "1"}, "runner": self.fake} options.update(overrides) return SaltyClient.prepare(self.directory, **options) def restart(self): return SaltyClient(self.directory, runner=self.fake) def saved(self): return json.loads((self.directory / "receipt.json").read_text()) def test_prepare_has_no_cli_effect_and_exclusive_snapshots(self): self.assertEqual(self.fake.calls, []) self.assertEqual(self.saved()["phase"], "PREPARED") self.bundle.write_bytes(b"caller later edits original") self.assertEqual((self.directory / "payload/bundle.tar").read_bytes(), b"credential-free fixture bundle") with self.assertRaises(FileExistsError): self.prepare() self.assertEqual(self.fake.calls, []) def test_submission_intent_is_durable_before_run_and_restart_never_resubmits(self): def check(): receipt = self.saved() self.assertEqual(receipt["phase"], "SUBMITTING") self.assertEqual(receipt["submission_state"], "UNKNOWN") self.assertEqual(receipt["commands"][-1]["label"], "submit") self.assertEqual(receipt["commands"][-1]["state"], "STARTED") self.fake.check_before_run = check receipt = self.client.submit() self.assertEqual(receipt["job_id"], JOB) self.assertEqual(self.fake.count("run"), 1) restarted = self.restart() self.assertEqual(restarted.submit()["job_id"], JOB) self.assertEqual(restarted.poll()["job_state"], "pending") self.fake.jobs[JOB] = "succeeded" self.assertEqual(self.restart().poll()["job_state"], "succeeded") self.assertEqual(self.fake.count("run"), 1) self.assertTrue(all(call[2] == JOB for call in self.fake.calls if call[1] == "describe")) def test_resources_use_platform_defaults_unless_explicit(self): self.client.submit() run = next(call for call in self.fake.calls if call[1] == "run") self.assertNotIn("--cpu", run) self.assertNotIn("--memory", run) explicit_fake = FakeCli() explicit = SaltyClient.prepare(self.root / "explicit", image=IMAGE, bundle=self.bundle, worker=self.worker, command=["python3", "worker.py"], cpu=8, memory="16Gi", runner=explicit_fake) explicit.submit() run = next(call for call in explicit_fake.calls if call[1] == "run") self.assertEqual(run[run.index("--cpu") + 1], "8") self.assertEqual(run[run.index("--memory") + 1], "16Gi") def test_concurrent_call_cannot_cross_receipt_lock(self): def check(): with self.assertRaisesRegex(TransportError, "another caller"): self.restart().submit() self.fake.check_before_run = check self.client.submit() self.assertEqual(self.fake.count("run"), 1) def test_changed_snapshot_is_detected_on_restart_before_remote_operations(self): (self.directory / "payload/bundle.tar").write_bytes(b"replaced bytes") with self.assertRaisesRegex(ValueError, "payload/source hash"): self.restart() self.assertEqual(self.fake.calls, []) def test_unknown_submissions_never_retry_or_cleanup(self): for mode in ("timeout", "failed", "format", "exception"): with self.subTest(mode=mode): root = self.root / mode fake = FakeCli() fake.run_mode = mode client = SaltyClient.prepare(root, image=IMAGE, bundle=self.bundle, worker=self.worker, command=["python3", "worker.py"], runner=fake) with self.assertRaises(TransportError): client.submit() restart = SaltyClient(root, runner=fake) self.assertEqual(restart.receipt["phase"], "SUBMISSION_UNKNOWN") with self.assertRaises(AmbiguousOperation): restart.submit() with self.assertRaises(AmbiguousOperation): restart.cleanup(authorized=True) self.assertEqual(fake.count("run"), 1) self.assertIn(JOB, fake.jobs) if mode == "timeout": entry = restart.receipt["commands"][-1] self.assertEqual(entry["local_pid"], 9123) self.assertTrue(entry["process_left_running"]) def test_crash_after_completed_submission_recovers_recorded_exact_id(self): self.client.submit() receipt = self.saved() receipt.update(job_id=None, phase="SUBMITTING", submission_state="UNKNOWN") (self.directory / "receipt.json").write_text(json.dumps(receipt)) restarted = self.restart() self.assertEqual(restarted.recover_recorded_submission()["job_id"], JOB) self.assertEqual(restarted.poll()["job_state"], "pending") self.assertEqual(self.fake.count("run"), 1) def test_unknown_completed_process_cannot_be_inferred_from_stdout(self): self.fake.run_mode = "timeout" with self.assertRaises(AmbiguousOperation): self.client.submit() with self.assertRaises(AmbiguousOperation): self.restart().recover_recorded_submission() self.assertEqual(self.fake.count("run"), 1) def test_cli_deadline_leaves_process_to_finish_naturally(self): marker = self.root / "natural-completion.txt" stdout, stderr = self.root / "natural.stdout", self.root / "natural.stderr" runner = CliRunner() code = "import pathlib,time; time.sleep(0.3); pathlib.Path(%r).write_text('naturally finished')" % str(marker) starts = [] result = runner([sys.executable, "-B", "-c", code], stdout, stderr, 0.02, on_started=lambda pid, identity: starts.append((pid, identity))) self.assertTrue(result.timed_out) self.assertTrue(result.process_left_running) self.assertEqual(starts, [(result.pid, result.birth_identity)]) if sys.platform == 'linux': self.assertEqual(result.birth_identity['status'], 'KNOWN') self.assertGreater(result.birth_identity['start_ticks'], 0) process = runner._unresolved[result.pid] self.assertIsNone(process.poll()) self.assertEqual(process.wait(timeout=5), 0) self.assertEqual(marker.read_text(), "naturally finished") del runner._unresolved[result.pid] def test_cli_interrupt_retains_started_birth_identity_in_receipt(self): from types import SimpleNamespace process = SimpleNamespace(pid=4567, poll=lambda: (_ for _ in ()).throw(KeyboardInterrupt())) identity = {'pid': 4567, 'status': 'KNOWN', 'boot_id': 'fixture', 'start_ticks': 12} runner = CliRunner() self.client.runner = runner with patch('harness.salty_client.subprocess.Popen', return_value=process), \ patch('harness.salty_client._cli_birth_identity', return_value=identity): with self.assertRaises(AmbiguousOperation): self.client.submit() entry = self.saved()['commands'][0] self.assertEqual(entry['state'], 'UNKNOWN') self.assertEqual(entry['local_birth_identity'], identity) self.assertEqual(entry['local_pid'], 4567) self.assertTrue(entry['process_left_running']) self.assertIs(runner._unresolved[4567], process) def test_pending_and_running_are_never_removed(self): self.client.submit() for state in ("pending", "running"): self.fake.jobs[JOB] = state with self.assertRaisesRegex(TransportError, "cannot be stopped or deleted"): self.client.cleanup(authorized=True) self.assertEqual(self.fake.count("rm"), 0) self.assertEqual(len(self.fake.directories), 1) self.assertEqual(self.saved()["cleanup"], {}) def test_prepared_without_cli_can_discard_only_local_temporary_files(self): result = self.client.cleanup(authorized=True) self.assertEqual(result["phase"], "CLEANED") self.assertTrue(result["cleanup"]["never_submitted"]) self.assertEqual(self.fake.calls, []) self.assertEqual({p.name for p in self.directory.iterdir()}, {"receipt.json", "receipt.lock"}) def test_version_failure_never_submitted_cleanup_has_no_remote_mutation(self): self.fake.version_failure = True with self.assertRaises(TransportError): self.client.submit() result = self.restart().cleanup(authorized=True) self.assertEqual(result["phase"], "CLEANED") self.assertEqual(self.fake.calls, [["salty", "version"]]) self.assertEqual(self.fake.directories, set()) self.assertFalse((self.directory / "payload").exists()) self.assertTrue(result["cleanup"]["storage_not_created_by_attempt"]) self.assertFalse(result["cleanup"]["remote_cleanup_unknown"]) def test_upload_nonzero_cleanup_removes_only_confirmed_owned_partial_directory(self): self.fake.upload_failure = True with self.assertRaises(TransportError): self.client.submit() self.assertEqual(len(self.fake.directories), 1) foreign = "personal:/another-owned-directory" self.fake.directories.add(foreign) self.assertEqual(self.fake.count("run"), 0) result = self.restart().cleanup(authorized=True) self.assertEqual(result["phase"], "CLEANED") self.assertEqual(self.fake.directories, {foreign}) self.assertEqual(self.fake.count("describe"), 0) self.assertEqual(self.fake.count("rm"), 0) self.assertEqual(self.fake.calls[-1], ["salty", "storage", "rm", "-r", result["remote_directory"]]) self.assertFalse((self.directory / "payload").exists()) def test_upload_timeout_does_not_delete_anything_or_allow_submit_retry(self): self.fake.upload_timeout = True with self.assertRaises(AmbiguousOperation): self.client.submit() before = list(self.fake.calls) with self.assertRaises(AmbiguousOperation): self.restart().cleanup(authorized=True) with self.assertRaises(AmbiguousOperation): self.restart().submit() self.assertEqual(self.fake.calls, before) self.assertEqual(len(self.fake.directories), 1) self.assertTrue((self.directory / "payload").is_dir()) def test_failed_exclusive_mkdir_cannot_delete_foreign_uuid_directory(self): remote = self.client.receipt["remote_directory"] self.fake.directories.add(remote) # Target already existed: not ours. with self.assertRaises(TransportError): self.client.submit() before = list(self.fake.calls) with self.assertRaises(AmbiguousOperation): self.restart().cleanup(authorized=True) result = self.saved() self.assertEqual(result["phase"], "UNRECONCILED") self.assertTrue(result["cleanup"]["remote_cleanup_unknown"]) self.assertFalse(result["cleanup"]["storage_removed"]) self.assertNotIn("storage_not_created_by_attempt", result["cleanup"]) self.assertEqual(self.fake.calls, before) self.assertEqual(self.fake.directories, {remote}) self.assertFalse((self.directory / "payload").exists()) def test_mkdir_nonzero_or_timeout_after_server_create_remains_unreconciled(self): for mode in ("nonzero", "timeout"): with self.subTest(mode=mode): fake = FakeCli() fake.mkdir_failure_after_create = mode == "nonzero" fake.mkdir_timeout_after_create = mode == "timeout" directory = self.root / ("mkdir-" + mode) client = SaltyClient.prepare(directory, image=IMAGE, bundle=self.bundle, worker=self.worker, command=["python3", "worker.py"], runner=fake) with self.assertRaises(TransportError): client.submit() remote = client.receipt["remote_directory"] self.assertEqual(fake.directories, {remote}) before = list(fake.calls) with self.assertRaisesRegex(AmbiguousOperation, "unreconciled"): client.cleanup(authorized=True) receipt = client.receipt self.assertEqual(receipt["phase"], "UNRECONCILED") self.assertTrue(receipt["cleanup"]["remote_cleanup_unknown"]) self.assertFalse(receipt["cleanup"]["remote_owned"]) self.assertFalse(receipt["cleanup"]["storage_removed"]) self.assertTrue(receipt["cleanup"]["local_payload_removed"]) mkdir = next(entry for entry in receipt["commands"] if entry["label"] == "mkdir-exclusive") self.assertEqual({path.name for path in directory.iterdir()}, {"receipt.json", "receipt.lock", mkdir["stdout"], mkdir["stderr"]}) restarted = SaltyClient(directory, runner=fake) with self.assertRaises(AmbiguousOperation): restarted.cleanup(authorized=True) with self.assertRaisesRegex(TransportError, "cleanup already started"): restarted.submit() self.assertEqual(fake.calls, before) self.assertEqual(fake.directories, {remote}) self.assertEqual(fake.count("run"), 0) def test_unreconciled_mkdir_can_resume_partly_discarded_local_payload(self): self.fake.mkdir_failure_after_create = True with self.assertRaises(TransportError): self.client.submit() remote = self.client.receipt["remote_directory"] original_unlink = Path.unlink def fail_second_snapshot(path, *args, **kwargs): if path.parent.name == "payload" and path.name == "worker.py": raise OSError(errno.EIO, "fixture interrupted snapshot cleanup") return original_unlink(path, *args, **kwargs) with patch.object(Path, "unlink", fail_second_snapshot): with self.assertRaises(OSError): self.client.cleanup(authorized=True) self.assertFalse((self.directory / "payload/bundle.tar").exists()) self.assertTrue((self.directory / "payload/worker.py").exists()) restarted = self.restart() self.assertTrue(restarted.receipt["cleanup"]["local_payload_removal_started"]) before = list(self.fake.calls) with self.assertRaises(AmbiguousOperation): restarted.cleanup(authorized=True) self.assertFalse((self.directory / "payload").exists()) self.assertEqual(restarted.receipt["phase"], "UNRECONCILED") self.assertEqual(self.fake.calls, before) self.assertEqual(self.fake.directories, {remote}) def test_terminal_cleanup_is_exact_authorized_and_leaves_compact_receipt(self): submitted = self.client.submit() remote = submitted["remote_directory"] self.fake.jobs[JOB] = "succeeded" result = self.root / "result.tar" fetched = self.client.fetch("result.tar", result) self.assertEqual(fetched["state"], "COMPLETE") self.assertEqual(result.read_bytes(), b"verified-by-caller-result") with self.assertRaises(TransportError): self.client.cleanup() cleaned = self.client.cleanup(authorized=True) self.assertEqual(cleaned["phase"], "CLEANED") self.assertEqual(self.fake.jobs, {}) self.assertEqual(self.fake.directories, set()) self.assertIn(["salty", "rm", JOB], self.fake.calls) self.assertIn(["salty", "storage", "rm", "-r", remote], self.fake.calls) self.assertEqual({p.name for p in self.directory.iterdir()}, {"receipt.json", "receipt.lock"}) self.assertTrue(result.is_file()) calls_before = len(self.fake.calls) self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED") self.assertEqual(len(self.fake.calls), calls_before) def test_failed_job_cleanup_also_requires_exact_terminal_identity(self): self.client.submit() self.fake.jobs[JOB] = "failed" self.assertEqual(self.client.cleanup(authorized=True)["cleanup"]["terminal_state"], "failed") def test_unknown_cleanup_does_not_retry_delete(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" self.fake.cleanup_timeout = True with self.assertRaises(AmbiguousOperation): self.client.cleanup(authorized=True) commands_before = list(self.fake.calls) with self.assertRaises(AmbiguousOperation): self.restart().cleanup(authorized=True) self.assertEqual(self.fake.calls, commands_before) self.assertTrue((self.directory / "payload").is_dir()) def test_terminal_job_does_not_authorize_deletion_with_prior_started_cli(self): self.client.submit() self.fake.jobs[JOB] = 'succeeded' receipt = self.saved() stdout, stderr = 'crash-window.stdout.log', 'crash-window.stderr.log' (self.directory / stdout).write_text('still writable') (self.directory / stderr).write_text('') receipt['commands'].append({'label': 'describe', 'state': 'STARTED', 'argv': ['salty', 'describe', JOB], 'stdout': stdout, 'stderr': stderr}) (self.directory / 'receipt.json').write_text(json.dumps(receipt)) before = list(self.fake.calls) with self.assertRaisesRegex(AmbiguousOperation, 'unresolved CLI'): self.restart().cleanup(authorized=True) self.assertEqual(self.fake.calls, before) self.assertIn(JOB, self.fake.jobs) self.assertIn(receipt['remote_directory'], self.fake.directories) self.assertEqual((self.directory / stdout).read_text(), 'still writable') self.assertTrue((self.directory / 'payload').is_dir()) def test_prior_unknown_read_keeps_terminal_job_and_storage_untouched(self): self.client.submit() self.fake.describe_drift = True with self.assertRaises(FormatError): self.client.poll() receipt = self.saved() receipt['commands'][-1]['state'] = 'UNKNOWN' (self.directory / 'receipt.json').write_text(json.dumps(receipt)) self.fake.jobs[JOB] = 'succeeded' before = list(self.fake.calls) with self.assertRaises(AmbiguousOperation): self.restart().cleanup(authorized=True) self.assertEqual(self.fake.calls, before) self.assertIn(JOB, self.fake.jobs) def test_partial_cleanup_resume_skips_already_deleted_job(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" original_logs = {path.name: path.read_bytes() for path in self.directory.glob("*.log")} # A normal completion supplies the persisted hashes. Roll the local # receipt back to that exact post-command/pre-flag window, leaving the # fixture's job removed. Storage was not yet removed in that window. self.client.cleanup(authorized=True) receipt = self.saved() removed = next(entry for entry in receipt["commands"] if entry["label"] == "remove-terminal-job") # Restore that command's fixture logs from its known empty payload. for key in ("stdout", "stderr"): (self.directory / removed[key]).write_text("") removed.pop("logs_removed", None) for name, content in original_logs.items(): (self.directory / name).write_bytes(content) receipt["commands"] = receipt["commands"][:receipt["commands"].index(removed) + 1] receipt["cleanup"] = {"operation_state": "UNKNOWN", "operation": "remove-terminal-job", "authorized": True, "terminal_state": "succeeded"} receipt["phase"] = "TERMINAL" # Snapshots are present in the simulated crash window. (self.directory / "payload").mkdir() (self.directory / "payload/bundle.tar").write_bytes(self.bundle.read_bytes()) (self.directory / "payload/worker.py").write_bytes(self.worker.read_bytes()) self.fake.directories.add(receipt["remote_directory"]) (self.directory / "receipt.json").write_text(json.dumps(receipt)) before = len(self.fake.calls) result = self.restart().cleanup(authorized=True) self.assertEqual(result["phase"], "CLEANED") calls = self.fake.calls[before:] self.assertFalse(any(call[1] in {"describe", "rm"} for call in calls)) self.assertEqual(calls, [["salty", "storage", "rm", "-r", receipt["remote_directory"]]]) self.assertTrue(result["cleanup"]["job_removed"]) self.assertTrue(result["cleanup"]["storage_removed"]) self.assertTrue(result["cleanup"]["local_payload_removed"]) def test_format_drift_does_not_guess_terminal_or_delete(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" self.fake.describe_drift = True with self.assertRaises(FormatError): self.client.poll() with self.assertRaises(FormatError): self.client.cleanup(authorized=True) self.assertIn(JOB, self.fake.jobs) self.assertEqual(self.fake.count("rm"), 0) def test_download_guards_and_no_overwrite(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" for unsafe in ("../another", "/data/shared/result", "./result.tar", "personal:/x", ".", "worker.py"): with self.subTest(unsafe=unsafe), self.assertRaises(ValueError): self.client.fetch(unsafe, self.root / "result.tar") destination = self.root / "result.tar" self.client.fetch("result.tar", destination) with self.assertRaises(ValueError): self.client.fetch("result.tar", destination) with self.assertRaises(AmbiguousOperation): self.client.fetch("result.tar", self.root / "second.tar") def test_different_receipts_atomically_publish_one_destination_without_overwrite(self): other_fake = FakeCli() other_fake.download_bytes = b"independent-other-result" other_directory = self.root / "other-receipt" other = SaltyClient.prepare(other_directory, image=IMAGE, bundle=self.bundle, worker=self.worker, command=["python3", "worker.py"], runner=other_fake) for client, fake in ((self.client, self.fake), (other, other_fake)): client.submit() fake.jobs[JOB] = "succeeded" barrier = threading.Barrier(2) self.fake.download_barrier = other_fake.download_barrier = barrier destination = self.root / "shared-result.tar" def fetch(client): try: return client.fetch("result.tar", destination) except TransportError as exc: return exc with ThreadPoolExecutor(max_workers=2) as pool: futures = [pool.submit(fetch, client) for client in (self.client, other)] results = [future.result(timeout=10) for future in futures] successes = [result for result in results if isinstance(result, dict)] failures = [result for result in results if isinstance(result, TransportError)] self.assertEqual(len(successes), 1) self.assertEqual(len(failures), 1) self.assertIn("destination already exists", str(failures[0])) self.assertEqual(hashlib.sha256(destination.read_bytes()).hexdigest(), successes[0]["sha256"]) receipts = [client.receipt for client in (self.client, other)] self.assertEqual({receipt["downloads"][0]["state"] for receipt in receipts}, {"COMPLETE", "PUBLISH_FAILED"}) loser = next(receipt["downloads"][0] for receipt in receipts if receipt["downloads"][0]["state"] == "PUBLISH_FAILED") self.assertEqual(loser["publication_errno"], errno.EEXIST) for client, directory in ((self.client, self.directory), (other, other_directory)): self.assertEqual(list(directory.glob("*.part")), []) downloaded = next(call for call in client.runner.calls if call[1:3] == ["storage", "cp"] and call[3].startswith("personal:/")) self.assertEqual(Path(downloaded[-1]).parent, directory.resolve()) self.assertNotEqual(downloaded[-1], str(destination)) self.assertEqual(client.cleanup(authorized=True)["phase"], "CLEANED") self.assertEqual({path.name for path in directory.iterdir()}, {"receipt.json", "receipt.lock"}) self.assertEqual(hashlib.sha256(destination.read_bytes()).hexdigest(), successes[0]["sha256"]) def test_cross_filesystem_publication_has_no_copy_fallback_and_is_cleanable(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" destination = self.root / "cross-filesystem-result.tar" with patch("harness.salty_client.os.link", side_effect=OSError(errno.EXDEV, "fixture cross device")): with self.assertRaisesRegex(TransportError, "crosses filesystems"): self.client.fetch("result.tar", destination) download = self.client.receipt["downloads"][0] self.assertEqual(download["state"], "PUBLISH_FAILED") self.assertEqual(download["publication_errno"], errno.EXDEV) self.assertTrue(download["temporary_removed"]) self.assertFalse(destination.exists()) self.assertEqual(list(self.directory.glob("*.part")), []) self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED") def test_completed_publication_cleanup_removes_only_its_leftover_temporary_link(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" destination = self.root / "published-result.tar" original_unlink = Path.unlink def fail_temporary_unlink(path, *args, **kwargs): if path.suffix == ".part": raise OSError(errno.EIO, "fixture interrupted local unlink") return original_unlink(path, *args, **kwargs) with patch.object(Path, "unlink", fail_temporary_unlink): with self.assertRaises(OSError): self.client.fetch("result.tar", destination) download = self.client.receipt["downloads"][0] self.assertEqual(download["state"], "COMPLETE") self.assertNotIn("temporary_removed", download) self.assertEqual(len(list(self.directory.glob("*.part"))), 1) self.assertEqual(self.restart().cleanup(authorized=True)["phase"], "CLEANED") self.assertEqual(list(self.directory.glob("*.part")), []) self.assertEqual(destination.read_bytes(), self.fake.download_bytes) def test_mutated_remote_path_cannot_delete_shared_or_root(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" receipt = self.saved() for unsafe in ("personal:/", "shared:/", "personal:/someone-else"): receipt["remote_directory"] = unsafe (self.directory / "receipt.json").write_text(json.dumps(receipt)) with self.subTest(unsafe=unsafe), self.assertRaises(ValueError): self.client.cleanup(authorized=True) self.assertEqual(self.fake.count("rm"), 0) def test_mutated_job_id_cannot_delete_another_complete_uuid(self): self.client.submit() self.fake.jobs[JOB] = "succeeded" receipt = self.saved() other = "salty-0ac12d34-1111-4222-8333-0123456789ac" self.fake.jobs[other] = "succeeded" receipt["job_id"] = other (self.directory / "receipt.json").write_text(json.dumps(receipt)) with self.assertRaisesRegex(ValueError, "job ID differs"): self.client.cleanup(authorized=True) self.assertEqual(self.fake.count("rm"), 0) self.assertIn(other, self.fake.jobs) def test_immutable_image_and_non_sensitive_environment_gate(self): for image in ("runtime:latest", "https://registry.example/x", "registry.example/x:latest", "registry.example/x@sha256:abc", "registry.example/user:password/x@sha256:" + "a" * 64): with self.subTest(image=image), self.assertRaises(ValueError): validate_image(image) self.assertEqual(validate_image(IMAGE), IMAGE) for env in ({"TOKEN": "secret"}, {"OMP_NUM_THREADS": "$(cat file)"}, {"PYTHONUNBUFFERED": "123"}, {"OPENBLAS_NUM_THREADS": "0"}): with self.subTest(env=env), self.assertRaises(ValueError): self.prepare(env=env) if __name__ == "__main__": unittest.main()