forked from longtao.wu/droboticflow
142 lines
7.7 KiB
Python
142 lines
7.7 KiB
Python
import importlib.util
|
|
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parent
|
|
spec = importlib.util.spec_from_file_location("node4_pilot", ROOT / "pilot.py")
|
|
pilot = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(pilot)
|
|
|
|
|
|
class SafetyTests(unittest.TestCase):
|
|
def test_query25_rejects_failed_small_recovery(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
evidence = pathlib.Path(directory) / "failed.json"
|
|
evidence.write_text(json.dumps({"result": "failed"}))
|
|
argv = ["pilot.py", "--kubeconfig", "not-used", "--output", str(pathlib.Path(directory) / "out"), "--query25", "--acceptance-file", str(evidence)]
|
|
with patch("sys.argv", argv), patch.object(pilot.subprocess, "run") as call:
|
|
with self.assertRaises(AssertionError):
|
|
pilot.main()
|
|
call.assert_not_called()
|
|
|
|
def test_query25_render_is_offline_and_uses_distinct_bounded_storage(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
evidence = pathlib.Path(directory) / "passed.json"
|
|
evidence.write_text(json.dumps({"result": "small-smoke-passed", "firstPodUID": "first", "secondPodUID": "second"}))
|
|
output = pathlib.Path(directory) / "out"
|
|
argv = ["pilot.py", "--kubeconfig", "not-used", "--output", str(output), "--query25", "--acceptance-file", str(evidence)]
|
|
with patch("sys.argv", argv), patch.object(pilot.subprocess, "run") as call, contextlib.redirect_stdout(io.StringIO()):
|
|
pilot.main()
|
|
call.assert_not_called()
|
|
plan = json.loads((output / "plan.json").read_text())
|
|
self.assertEqual(plan["bytes"], 25 * 1024 ** 3)
|
|
self.assertEqual(plan["acceptanceSha256"], hashlib.sha256(evidence.read_bytes()).hexdigest())
|
|
self.assertNotEqual(plan["hostPath"], pilot.PATH)
|
|
objects = json.loads((output / "objects.json").read_text())
|
|
self.assertFalse(any(obj["kind"] in ("Service", "Secret", "PersistentVolumeClaim") for obj in objects))
|
|
config = objects[-2]["data"]
|
|
self.assertIn("truncate -s 26843545600", config["init.sh"])
|
|
self.assertIn("umount /lab/data", config["init.sh"])
|
|
self.assertNotIn("umount -", config["init.sh"])
|
|
self.assertIn("--bind_ip 127.0.0.1", config["query25.sh"])
|
|
|
|
def test_boot_does_not_mark_ready_after_guard_failure(self):
|
|
command = pilot.pod()["spec"]["containers"][0]["command"][2]
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
guard = pathlib.Path(directory) / "guard.sh"
|
|
guard.write_text("exit 77\n")
|
|
marker = pathlib.Path(directory) / "ready"
|
|
command = command.replace("/scripts/guard.sh", str(guard)).replace("/tmp/ready", str(marker))
|
|
p = subprocess.run(["bash", "-c", command], capture_output=True, timeout=3)
|
|
self.assertEqual(p.returncode, 77)
|
|
self.assertFalse(marker.exists())
|
|
|
|
def test_invalid_node_or_pod_identity_fails_before_host_access(self):
|
|
for node, uid in [("10.189.16.5", "00000000-0000-0000-0000-000000000000"),
|
|
("10.189.16.4", "../../kubepods.slice")]:
|
|
with self.subTest(node=node, uid=uid):
|
|
p = subprocess.run(["bash", str(ROOT / "init.sh")],
|
|
env={**os.environ, "NODE_NAME": node, "POD_UID": uid},
|
|
capture_output=True, timeout=3)
|
|
self.assertNotEqual(p.returncode, 0)
|
|
|
|
def test_rebuild_identity_uses_new_pod_and_guaranteed_qos(self):
|
|
obj = pilot.pod()
|
|
self.assertEqual(obj["spec"]["nodeName"], "10.189.16.4")
|
|
self.assertFalse(obj["spec"]["automountServiceAccountToken"])
|
|
for container in obj["spec"]["containers"] + obj["spec"]["initContainers"]:
|
|
self.assertEqual(container["resources"]["requests"], container["resources"]["limits"])
|
|
identity = next(e for e in container["env"] if e["name"] == "POD_UID")
|
|
self.assertEqual(identity["valueFrom"]["fieldRef"]["fieldPath"], "metadata.uid")
|
|
db = obj["spec"]["containers"][0]
|
|
self.assertTrue(db["securityContext"]["runAsNonRoot"])
|
|
self.assertFalse(db["securityContext"]["allowPrivilegeEscalation"])
|
|
self.assertEqual(db["securityContext"]["capabilities"]["drop"], ["ALL"])
|
|
self.assertTrue(next(v for v in db["volumeMounts"] if v["name"] == "cg")["readOnly"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
class ProfileTests(unittest.TestCase):
|
|
def test_profiles_preserve_disk_security_and_both_io_layers(self):
|
|
profiles = pilot.query_lab_profiles
|
|
for name in profiles.PROFILES:
|
|
objects = pilot.manifests(True, name)
|
|
config, pod = objects[-2:]
|
|
p = profiles.profile(name)
|
|
profiles.verify_resources(pod, name)
|
|
profiles.verify_scripts(config['data'], name)
|
|
expected = ' '.join(f'{k}={p[k]}' for k in ('rbps', 'wbps', 'riops', 'wiops'))
|
|
for filename in ('init.sh', 'guard.sh'):
|
|
self.assertIn('252:0 ' + expected, config['data'][filename])
|
|
self.assertIn('$loop_device ' + expected, config['data'][filename])
|
|
self.assertIn('--wiredTigerCacheSizeGB ' + p['cacheGi'], config['data']['query25.sh'])
|
|
self.assertIn('WT cache drift', config['data']['verify25.sh'])
|
|
self.assertIn('truncate -s 26843545600', config['data']['init.sh'])
|
|
self.assertIn('80530636800', config['data']['query25.sh'])
|
|
self.assertIn('memory.max', config['data']['guard.sh'])
|
|
pod['spec']['containers'][0]['resources']['limits']['memory'] = '16Gi'
|
|
with self.assertRaises(AssertionError):
|
|
profiles.verify_resources(pod, name)
|
|
profiles.verify_scripts(config['data'], name)
|
|
|
|
def test_unknown_profile_fails_closed(self):
|
|
with self.assertRaises(KeyError):
|
|
pilot.manifests(True, 'unlimited')
|
|
|
|
def test_one_device_io_drift_is_rejected(self):
|
|
for filename in ('init.sh', 'guard.sh'):
|
|
data = pilot.manifests(True, 'balanced')[-2]['data']
|
|
data[filename] = data[filename].replace('252:0 rbps=4194304', '252:0 rbps=2097152')
|
|
with self.assertRaises(AssertionError):
|
|
pilot.query_lab_profiles.verify_scripts(data, 'balanced')
|
|
|
|
class RollbackTests(unittest.TestCase):
|
|
def test_rollback_rejects_storage_or_script_drift(self):
|
|
import copy
|
|
spec = importlib.util.spec_from_file_location('resource_stage', ROOT.parent / 'query-lab-resource-stage.py')
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
for name in pilot.query_lab_profiles.PROFILES:
|
|
objects = pilot.manifests(True, name)
|
|
saved = {'pod': objects[-1], 'config': objects[-2], 'quota': objects[1]}
|
|
module.validate_saved(saved, pilot)
|
|
for drift in ('node', 'path', 'script', 'resource'):
|
|
bad = copy.deepcopy(saved)
|
|
if drift == 'node': bad['pod']['spec']['nodeName'] = 'other-node'
|
|
elif drift == 'path': bad['pod']['spec']['volumes'][0]['hostPath']['path'] = '/var/lib/production'
|
|
elif drift == 'script': bad['config']['data']['init.sh'] += '\necho unreviewed\n'
|
|
else: bad['pod']['spec']['containers'][0]['resources']['limits']['cpu'] = '8'
|
|
with self.assertRaises(AssertionError, msg=f'{name}/{drift}'):
|
|
module.validate_saved(bad, pilot)
|