787 lines
37 KiB
Python
787 lines
37 KiB
Python
#!/usr/bin/env python3
|
|
"""Boot/install/reboot acceptance on a fresh, isolated UEFI/KVM VM only.
|
|
|
|
No existing disk option, host sudo, SSH, host TTY, or GUI is used. Artifacts and
|
|
logs remain in a new private scratch directory even after failure.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import select
|
|
import shlex
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from hydra.qmp import QMPClient, QMPError
|
|
|
|
|
|
class SmokeError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def lifecycle_socket(path, *, connect_socket=True):
|
|
path = Path(path)
|
|
info = path.lstat()
|
|
if not stat.S_ISSOCK(info.st_mode) or info.st_uid != os.getuid():
|
|
raise SmokeError('Lifecycle socket is not owned by the installed user')
|
|
identity = [info.st_dev, info.st_ino, info.st_uid, info.st_mode]
|
|
if connect_socket:
|
|
with socket.socket(socket.AF_UNIX) as connection:
|
|
connection.settimeout(2)
|
|
connection.connect(str(path))
|
|
after = path.lstat()
|
|
if identity != [after.st_dev, after.st_ino, after.st_uid, after.st_mode]:
|
|
raise SmokeError('Lifecycle socket changed while connecting')
|
|
return identity
|
|
|
|
|
|
def lifecycle_snapshot(context):
|
|
"""Prove live identity and owned runtime endpoints, never trust PID alone."""
|
|
try:
|
|
if context['uid'] != os.getuid():
|
|
raise SmokeError('Lifecycle probe is not the installed normal user')
|
|
runtime = Path(context['runtime'])
|
|
run = runtime / context['name']
|
|
directories = {}
|
|
for directory in (runtime, run):
|
|
info = directory.lstat()
|
|
if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid()
|
|
or stat.S_IMODE(info.st_mode) != 0o700):
|
|
raise SmokeError('Lifecycle runtime must remain private and owned')
|
|
directories[str(directory)] = [info.st_dev, info.st_ino, info.st_mode]
|
|
metadata = json.loads((run / 'identity.json').read_text())
|
|
if metadata != {'name': context['name'], 'uuid': context['uuid']}:
|
|
raise SmokeError('Lifecycle metadata is stale or foreign')
|
|
pid = int((run / 'qemu.pid').read_text())
|
|
process = Path('/proc') / str(pid)
|
|
if process.stat().st_uid != os.getuid():
|
|
raise SmokeError('Lifecycle process belongs to another user')
|
|
fields = (process / 'stat').read_text().rsplit(')', 1)[1].split()
|
|
if fields[0] == 'Z':
|
|
raise SmokeError('Lifecycle process has exited')
|
|
argv = (process / 'cmdline').read_bytes().split(b'\0')
|
|
for option, expected in ((b'-name', context['name']), (b'-uuid', context['uuid'])):
|
|
if argv[argv.index(option) + 1] != expected.encode():
|
|
raise SmokeError('Lifecycle PID is not the owned VM identity')
|
|
endpoints = {'qmp': lifecycle_socket(run / 'qmp.sock', connect_socket=False)}
|
|
with QMPClient(run / 'qmp.sock', timeout=2) as client:
|
|
if (client.command('query-name') != {'name': context['name']}
|
|
or client.command('query-uuid') != {'UUID': context['uuid']}):
|
|
raise SmokeError('Lifecycle QMP identity is foreign')
|
|
if client.command('query-status').get('status') != 'running':
|
|
raise SmokeError('Lifecycle VM is not running')
|
|
if not client.command('query-kvm').get('enabled'):
|
|
raise SmokeError('Lifecycle VM did not initialise nested KVM')
|
|
endpoints['spice'] = lifecycle_socket(run / 'spice.sock')
|
|
audio = Path(context['user_runtime'])
|
|
endpoints['pipewire'] = lifecycle_socket(audio / 'pipewire-0')
|
|
endpoints['pulse'] = lifecycle_socket(audio / 'pulse/native')
|
|
services = {}
|
|
for service in context.get('audio_services', []):
|
|
subprocess.run(['systemctl', '--user', 'is-active', service], check=True,
|
|
capture_output=True, timeout=5)
|
|
services[service] = subprocess.run(['systemctl', '--user', 'show', service,
|
|
'-p', 'InvocationID', '-p', 'ExecMainPID'], check=True,
|
|
capture_output=True, text=True, timeout=5).stdout
|
|
return {'name': context['name'], 'uuid': context['uuid'], 'pid': pid,
|
|
'process_start': fields[19], 'directories': directories,
|
|
'sockets': endpoints, 'audio_services': services}
|
|
except (OSError, ValueError, KeyError, IndexError, QMPError, subprocess.SubprocessError) as exc:
|
|
raise SmokeError('Lifecycle runtime or process was lost: ' + str(exc)) from exc
|
|
|
|
|
|
def verify_lifecycle_survival(before, after):
|
|
if before != after:
|
|
raise SmokeError('VM identity, private runtime, or audio endpoints changed across logout')
|
|
|
|
|
|
LIFECYCLE_BODY = r'''
|
|
def run(*args):
|
|
return subprocess.run(args, check=True, capture_output=True, text=True, timeout=40).stdout
|
|
|
|
def cli(context, *args):
|
|
return json.loads(run('hydra', '--config', context['config'], '--state', context['state'],
|
|
'--runtime', context['runtime'], *args))
|
|
|
|
def no_user_sessions(uid):
|
|
rows = run('loginctl', 'list-sessions', '--no-legend', '--no-pager').splitlines()
|
|
return not any(len(row.split()) > 1 and row.split()[1] == str(uid) for row in rows)
|
|
|
|
def write_report(path, data):
|
|
temporary = path.with_suffix('.tmp')
|
|
temporary.write_text(json.dumps(data, indent=2) + '\n')
|
|
temporary.chmod(0o600)
|
|
temporary.replace(path)
|
|
|
|
mode, base_arg = sys.argv[1:]
|
|
base = Path(base_arg)
|
|
context_path = base / 'context.json'
|
|
observation_path = base / 'logout-observation.json'
|
|
if mode == 'start':
|
|
if not Path('/dev/kvm').exists():
|
|
print('SKIP: installed guest nested KVM unavailable')
|
|
raise SystemExit(0)
|
|
if os.getuid() == 0 or not os.access('/dev/kvm', os.R_OK | os.W_OK):
|
|
raise SmokeError('Lifecycle test needs installed-user nested KVM access')
|
|
user_runtime = Path(os.environ['XDG_RUNTIME_DIR'])
|
|
if user_runtime != Path('/run/user') / str(os.getuid()):
|
|
raise SmokeError('Lifecycle test needs the real login runtime directory')
|
|
if run('loginctl', 'show-user', str(os.getuid()), '-p', 'Linger', '--value').strip() != 'no':
|
|
raise SmokeError('Lingering would mask the installed logout policy')
|
|
rows = run('loginctl', 'list-sessions', '--no-legend', '--no-pager').splitlines()
|
|
if sum(len(row.split()) > 1 and row.split()[1] == str(os.getuid()) for row in rows) != 1:
|
|
raise SmokeError('Lifecycle test requires exactly the serial user login')
|
|
name = 'logout-' + base.name.rsplit('-', 1)[1]
|
|
state = base / 'state'
|
|
runtime = user_runtime / base.name
|
|
context = {'uid': os.getuid(), 'name': name, 'config': str(base / 'config'),
|
|
'state': str(state), 'runtime': str(runtime), 'user_runtime': str(user_runtime),
|
|
'uuid': str(uuid.uuid5(uuid.NAMESPACE_URL, str(state / name / 'disk.qcow2'))),
|
|
'audio_services': ['pipewire.service', 'pipewire-pulse.service', 'wireplumber.service']}
|
|
run('systemctl', '--user', 'start', *context['audio_services'])
|
|
cli(context, 'add', name, '--memory', '256', '--cpus', '1', '--disk-size', '64M',
|
|
'--graphics', 'software', '--network', 'bridge', '--bridge', 'hydrabr0')
|
|
if cli(context, 'start', name).get('status') != 'running':
|
|
raise SmokeError('Real installed hydra CLI failed to start lifecycle VM')
|
|
try:
|
|
disk = Path(context['state']) / name / 'disk.qcow2'
|
|
info = disk.lstat()
|
|
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.getuid():
|
|
raise SmokeError('Lifecycle VM disk is not a new owned regular qcow2')
|
|
context['before'] = lifecycle_snapshot(context)
|
|
except Exception:
|
|
# CLI control is scoped to the newly created private identity.
|
|
cli(context, 'stop', name, '--force')
|
|
raise
|
|
write_report(context_path, context)
|
|
with open(base / 'monitor.log', 'ab', buffering=0) as log:
|
|
subprocess.Popen([sys.executable, __file__, 'monitor', str(base)],
|
|
stdin=subprocess.DEVNULL, stdout=log, stderr=log,
|
|
start_new_session=True)
|
|
print('READY: normal-user Hydra VM and audio sockets recorded before real logout')
|
|
elif mode == 'monitor':
|
|
context = json.loads(context_path.read_text())
|
|
try:
|
|
deadline = time.monotonic() + 35
|
|
while not no_user_sessions(context['uid']):
|
|
if time.monotonic() >= deadline:
|
|
raise SmokeError('Serial session did not actually log out')
|
|
time.sleep(.2)
|
|
# Exceed systemd's usual 10-second user-manager stop delay. No helper
|
|
# login or lingering is allowed to keep the tested user session alive.
|
|
time.sleep(15)
|
|
if not no_user_sessions(context['uid']):
|
|
raise SmokeError('Another user login masked the logout interval')
|
|
during = lifecycle_snapshot(context)
|
|
verify_lifecycle_survival(context['before'], during)
|
|
write_report(observation_path, {'status': 'passed', 'no_user_sessions': True,
|
|
'survival_seconds': 15, 'during_logout': during})
|
|
except Exception as exc:
|
|
write_report(observation_path, {'status': 'failed', 'error': str(exc)})
|
|
raise
|
|
elif mode == 'finish':
|
|
context = json.loads(context_path.read_text())
|
|
try:
|
|
if os.environ.get('XDG_RUNTIME_DIR') != context['user_runtime']:
|
|
raise SmokeError('Relogin replaced the user runtime path')
|
|
deadline = time.monotonic() + 10
|
|
while not observation_path.is_file():
|
|
if time.monotonic() >= deadline:
|
|
raise SmokeError('Logout monitor died or did not record survival')
|
|
time.sleep(.2)
|
|
observation = json.loads(observation_path.read_text())
|
|
if observation.get('status') != 'passed' or observation.get('no_user_sessions') is not True:
|
|
raise SmokeError('No-session observation failed: ' + json.dumps(observation))
|
|
after = lifecycle_snapshot(context)
|
|
verify_lifecycle_survival(context['before'], after)
|
|
if cli(context, 'status', context['name']).get('status') != 'running':
|
|
raise SmokeError('Real installed hydra CLI could not reconnect after login')
|
|
print(json.dumps({'status': 'passed', 'real_logout': True, 'no_user_sessions': True,
|
|
'survival_seconds': observation['survival_seconds'], 'same_vm': after,
|
|
'audio_scope': 'Socket and service continuity only; signal/playback untested'}))
|
|
finally:
|
|
# Real product CLI verifies metadata and live QMP name/UUID before quit.
|
|
# Never signal a saved PID or control a VM outside these private paths.
|
|
cleanup_status = cli(context, 'status', context['name']).get('status')
|
|
if cleanup_status == 'running':
|
|
run_path = Path(context['runtime']) / context['name']
|
|
if json.loads((run_path / 'identity.json').read_text()) != {'name': context['name'], 'uuid': context['uuid']}:
|
|
raise SmokeError('Refusing lifecycle cleanup for a foreign identity')
|
|
with QMPClient(run_path / 'qmp.sock', timeout=2) as client:
|
|
if (client.command('query-name') != {'name': context['name']}
|
|
or client.command('query-uuid') != {'UUID': context['uuid']}):
|
|
raise SmokeError('Refusing lifecycle cleanup for foreign live QMP')
|
|
if cli(context, 'stop', context['name'], '--force').get('status') != 'stopped':
|
|
raise SmokeError('Owned lifecycle VM did not stop')
|
|
elif cleanup_status != 'stopped':
|
|
raise SmokeError('Cannot safely verify the lifecycle VM for cleanup')
|
|
else:
|
|
raise SmokeError('Unknown lifecycle mode')
|
|
'''
|
|
|
|
|
|
def lifecycle_guest_script():
|
|
helpers = (lifecycle_socket, lifecycle_snapshot, verify_lifecycle_survival)
|
|
return ('import json, os, socket, stat, subprocess, sys, time, uuid\n'
|
|
'from pathlib import Path\nsys.path.insert(0, "/usr/lib/hydra")\n'
|
|
'from hydra.qmp import QMPClient, QMPError\n'
|
|
'class SmokeError(RuntimeError): pass\n' +
|
|
'\n'.join(inspect.getsource(helper) for helper in helpers) + LIFECYCLE_BODY)
|
|
|
|
|
|
def installed_admin(console, password):
|
|
console.buffer = b''
|
|
prompt_encoded = base64.b64encode(b'HYDRA_SUDO_PASSWORD: ').decode()
|
|
console.send(f'sudo -k -p "$(printf %s {prompt_encoded} | base64 -d)" -i\n')
|
|
console.expect(rb'HYDRA_SUDO_PASSWORD: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'root[^\r\n]*[#$] ', 30)
|
|
|
|
|
|
def installed_logout_probe(console, password):
|
|
# Exit only the guest sudo shell, then work in its actual getty PAM login.
|
|
console.send('exit\n')
|
|
console.expect(rb'hydratest[^\r\n]*[#$] ', 30)
|
|
if console.command('id -un', 10).strip() != b'hydratest':
|
|
raise SmokeError('Logout probe did not return to installed user login')
|
|
base = '/home/hydratest/hydra-logout-' + secrets.token_hex(4)
|
|
script = base + '/probe.py'
|
|
preparation = (f'umask 077; mkdir {shlex.quote(base)} && cat > {shlex.quote(script)} '
|
|
"<<'HYDRA_LIFECYCLE_SOURCE'\n" + lifecycle_guest_script() + '\nHYDRA_LIFECYCLE_SOURCE\n' +
|
|
f'python3 {shlex.quote(script)} start {shlex.quote(base)}')
|
|
ready = console.command(preparation, 120)
|
|
if ready.startswith(b'SKIP:'):
|
|
installed_admin(console, password)
|
|
return ready
|
|
if b'READY:' not in ready:
|
|
raise SmokeError('Lifecycle VM did not become ready before logout')
|
|
console.buffer = b''
|
|
console.send('exit\n')
|
|
console.expect(rb'login: ', 30)
|
|
# This is intentionally a period at the actual login prompt with no
|
|
# replacement login. The guest monitor independently confirms no sessions.
|
|
time.sleep(20)
|
|
console.send('hydratest\n')
|
|
console.expect(rb'Password: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'hydratest[^\r\n]*[#$] ', 60)
|
|
result = console.command(f'python3 {shlex.quote(script)} finish {shlex.quote(base)}', 90)
|
|
installed_admin(console, password)
|
|
return result
|
|
|
|
|
|
def reserve_disk(path):
|
|
try:
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
os.close(fd)
|
|
except OSError as exc:
|
|
raise SmokeError('Test disk must be a newly created regular file') from exc
|
|
|
|
|
|
def owned_file(path, scratch):
|
|
path, scratch = Path(path), Path(scratch)
|
|
try:
|
|
info = path.lstat()
|
|
except OSError as exc:
|
|
raise SmokeError('Missing disposable drive') from exc
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1
|
|
or path.parent.resolve() != scratch.resolve()
|
|
or info.st_uid != os.getuid()):
|
|
raise SmokeError('Drive must be an owned, unlinked regular file directly in scratch')
|
|
|
|
|
|
def qemu_path(path):
|
|
# QEMU keyval arguments use commas as separators. Refuse ambiguous paths.
|
|
if any(c in str(path) for c in ',\n\r\x00'):
|
|
raise SmokeError('QEMU file paths cannot contain comma or control characters')
|
|
return str(path)
|
|
|
|
|
|
def qemu_argv(qemu, scratch, disk, code, variables, iso, memory):
|
|
scratch = Path(scratch)
|
|
owned_file(disk, scratch)
|
|
owned_file(variables, scratch)
|
|
argv = [qemu, '-machine', 'q35,accel=kvm', '-cpu', 'host', '-smp', '2',
|
|
'-m', str(memory), '-display', 'none', '-monitor', 'none',
|
|
'-no-reboot', '-nic', 'user,model=virtio-net-pci',
|
|
'-drive', f'if=pflash,format=raw,readonly=on,file={qemu_path(code)}',
|
|
'-drive', f'if=pflash,format=raw,file={qemu_path(variables)}',
|
|
'-drive', f'if=virtio,format=qcow2,file={qemu_path(disk)}',
|
|
'-chardev', f'socket,id=console,path={qemu_path(scratch / "serial.sock")},server=on,wait=off',
|
|
'-serial', 'chardev:console', '-qmp', f'unix:{qemu_path(scratch / "qmp.sock")},server=on,wait=off']
|
|
if iso is not None:
|
|
if not Path(iso).is_file():
|
|
raise SmokeError('Trusted ISO must exist as a regular file')
|
|
argv += ['-drive', f'file={qemu_path(iso)},media=cdrom,format=raw,readonly=on',
|
|
'-boot', 'order=d,strict=on']
|
|
else:
|
|
argv += ['-boot', 'order=c,strict=on']
|
|
return argv
|
|
|
|
|
|
def frame_command(command, token):
|
|
if not re.fullmatch(r'[a-zA-Z0-9]+', token):
|
|
raise SmokeError('Invalid framing token')
|
|
encoded = base64.b64encode(command.encode()).decode()
|
|
# Linux canonical serial TTY input is bounded per physical line. Continue
|
|
# the shell expression across short lines for substantial guest probes.
|
|
chunks = ' \\\n'.join(shlex.quote(encoded[i:i + 768]) for i in range(0, len(encoded), 768)) or "''"
|
|
# Literal control bytes appear only in output, never in the echoed command.
|
|
return (f"printf '\\036{token}:BEGIN\\037'; "
|
|
f"sh -c \"$(printf %s {chunks} | base64 -d)\"; "
|
|
f"rc=$?; printf '\\036{token}:END:%s\\037' \"$rc\"")
|
|
|
|
|
|
def parse_frame(data, token):
|
|
begin = b'\x1e' + token.encode() + b':BEGIN\x1f'
|
|
start = data.find(begin)
|
|
if start < 0:
|
|
return None
|
|
end = re.search(rb'\x1e' + token.encode() + rb':END:(\d+)\x1f', data[start + len(begin):])
|
|
if end is None:
|
|
return None
|
|
output = data[start + len(begin):start + len(begin) + end.start()]
|
|
return int(end.group(1)), output
|
|
|
|
|
|
class Console:
|
|
def __init__(self, connection, logfile, passwords):
|
|
self.sock = connection
|
|
self.log = open(logfile, 'ab', buffering=0)
|
|
self.passwords = [p.encode() for p in passwords if p]
|
|
self.pending_log = b''
|
|
self.buffer = b''
|
|
self.closed = False
|
|
|
|
def record(self, data, final=False):
|
|
self.pending_log += data
|
|
for password in self.passwords:
|
|
self.pending_log = self.pending_log.replace(password, b'[REDACTED]')
|
|
keep = 0
|
|
if not final:
|
|
for password in self.passwords:
|
|
for length in range(1, len(password)):
|
|
if self.pending_log.endswith(password[:length]):
|
|
keep = max(keep, length)
|
|
if keep:
|
|
self.log.write(self.pending_log[:-keep])
|
|
self.pending_log = self.pending_log[-keep:]
|
|
else:
|
|
self.log.write(self.pending_log)
|
|
self.pending_log = b''
|
|
|
|
def close(self):
|
|
if not self.closed:
|
|
self.record(b'', final=True)
|
|
self.log.close()
|
|
self.sock.close()
|
|
self.closed = True
|
|
|
|
def send(self, data):
|
|
self.sock.sendall(data.encode() if isinstance(data, str) else data)
|
|
|
|
def read(self, timeout):
|
|
ready, _, _ = select.select([self.sock], [], [], max(0, timeout))
|
|
if not ready:
|
|
return b''
|
|
data = self.sock.recv(65536)
|
|
if not data:
|
|
raise SmokeError('Guest console closed')
|
|
self.record(data)
|
|
self.buffer += data
|
|
if len(self.buffer) > 16 * 1024 * 1024:
|
|
raise SmokeError('Console exceeded bounded capture size')
|
|
return data
|
|
|
|
def expect(self, pattern, timeout):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
match = re.search(pattern, self.buffer)
|
|
if match:
|
|
self.buffer = self.buffer[match.end():]
|
|
return match
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise SmokeError('Timed out waiting for guest console state')
|
|
self.read(min(remaining, 1))
|
|
|
|
def command(self, command, timeout=60):
|
|
token = secrets.token_hex(12)
|
|
self.buffer = b''
|
|
self.send(frame_command(command, token) + '\n')
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
parsed = parse_frame(self.buffer, token)
|
|
if parsed is not None:
|
|
status, output = parsed
|
|
self.buffer = b''
|
|
if status:
|
|
raise SmokeError(f'Guest probe failed with status {status}: {output.decode(errors="replace")[-3000:]}')
|
|
return output.replace(b'\r\n', b'\n')
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise SmokeError('Timed out waiting for framed guest command')
|
|
self.read(min(remaining, 1))
|
|
|
|
|
|
def connect(path, process, timeout=30):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if process.poll() is not None:
|
|
raise SmokeError(f'Test QEMU exited with status {process.returncode}')
|
|
connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
try:
|
|
connection.connect(str(path))
|
|
return connection
|
|
except OSError:
|
|
connection.close()
|
|
time.sleep(0.1)
|
|
raise SmokeError('Timed out connecting to private test socket')
|
|
|
|
|
|
class VM:
|
|
def __init__(self, args, scratch, disk, iso, password, phase):
|
|
self.scratch = scratch
|
|
for name in ('serial.sock', 'qmp.sock'):
|
|
(scratch / name).unlink(missing_ok=True)
|
|
self.stderr = open(scratch / f'{phase}-qemu.log', 'ab', buffering=0)
|
|
self.process = subprocess.Popen(qemu_argv(args.qemu, scratch, disk,
|
|
args.ovmf_code, scratch / 'OVMF_VARS.fd', iso, args.memory),
|
|
stdin=subprocess.DEVNULL, stdout=self.stderr, stderr=self.stderr)
|
|
self.console = None
|
|
self.qmp = None
|
|
try:
|
|
self.console = Console(connect(scratch / 'serial.sock', self.process),
|
|
scratch / f'{phase}-serial.log', [password])
|
|
ready = connect(scratch / 'qmp.sock', self.process)
|
|
ready.close()
|
|
self.qmp = QMPClient(scratch / 'qmp.sock', timeout=5)
|
|
self.kvm = self.qmp.command('query-kvm')
|
|
if not self.kvm.get('enabled'):
|
|
raise SmokeError('Disposable QEMU is not using KVM')
|
|
except BaseException:
|
|
self.close()
|
|
raise
|
|
|
|
def wait_poweroff(self, timeout):
|
|
try:
|
|
rc = self.process.wait(timeout=timeout)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise SmokeError('Guest failed to power off in time') from exc
|
|
if rc:
|
|
raise SmokeError(f'Test QEMU failed during poweroff ({rc})')
|
|
|
|
def close(self):
|
|
# Only the Popen child owned by this harness is signalled.
|
|
if self.qmp:
|
|
if self.process.poll() is None:
|
|
try:
|
|
self.qmp.command('quit')
|
|
self.process.wait(timeout=5)
|
|
except (QMPError, subprocess.TimeoutExpired):
|
|
pass
|
|
self.qmp.close()
|
|
if self.process.poll() is None:
|
|
self.process.terminate()
|
|
try:
|
|
self.process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
self.process.kill()
|
|
self.process.wait(timeout=10)
|
|
if self.console:
|
|
self.console.close()
|
|
self.stderr.close()
|
|
|
|
|
|
LIVE_CHECK = r'''set -eu
|
|
[ "$(id -un)" = hydra-live ]
|
|
test -e /sys/firmware/efi
|
|
test -f /etc/hydra/live-mode/active
|
|
findmnt -n -o FSTYPE / | grep -qx overlay
|
|
test -f /run/initramfs/live/LiveOS/squashfs.img
|
|
sudo -n true
|
|
for tool in hydra hydra-install qemu-system-x86_64 qemu-img virt-viewer foot labwc; do command -v "$tool"; done
|
|
for service in NetworkManager hydra-network hydra-dnsmasq; do systemctl is-active "$service"; done
|
|
'''
|
|
|
|
INSTALLED_CHECK = r'''set -eu
|
|
assert_no_match() {
|
|
if grep "$@"; then
|
|
printf 'Forbidden live configuration remains\n' >&2
|
|
exit 1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || exit "$status"
|
|
fi
|
|
}
|
|
[ "$(findmnt -n -o FSTYPE /)" = ext4 ]
|
|
[ "$(readlink -f "$(findmnt -n -o SOURCE /)")" = /dev/vda2 ]
|
|
assert_no_match -Eq 'root=live:|rd\.live' /proc/cmdline
|
|
test ! -e /etc/hydra/live-mode
|
|
test ! -L /etc/hydra/live-mode
|
|
if getent passwd hydra-live; then
|
|
printf 'Live account remains\n' >&2
|
|
exit 1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 2 ] || exit "$status"
|
|
fi
|
|
remaining=$(find /etc/systemd/system /etc/sudoers.d \( -name 90-hydra-live.conf -o -name 90-hydra-live \) -print)
|
|
[ -z "$remaining" ]
|
|
test ! -e /etc/greetd/live.toml
|
|
test ! -L /etc/greetd/live.toml
|
|
for directory in /etc/systemd/system/getty@*.service.d /etc/systemd/system/serial-getty@*.service.d /etc/sudoers.d; do
|
|
if [ -d "$directory" ]; then
|
|
assert_no_match -R -E -- '--autologin|NOPASSWD' "$directory"
|
|
elif [ -e "$directory" ] || [ -L "$directory" ]; then
|
|
printf 'Unexpected non-directory configuration path\n' >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
getent shadow root | cut -d: -f2 | grep -Eq '^[!*]'
|
|
getent shadow hydratest | cut -d: -f2 | grep -Eq '^\$'
|
|
for service in NetworkManager greetd hydra-network hydra-dnsmasq; do systemctl is-active "$service"; done
|
|
[ "$(systemctl show hydra-bridge-permissions -p Result --value)" = success ]
|
|
[ "$(systemctl show hydra-bridge-permissions -p ExecMainStatus --value)" = 0 ]
|
|
ip -4 -o addr show hydrabr0 | grep -F '10.203.0.1/24'
|
|
ip -4 route show default | grep '^default '
|
|
nft list table ip hydra_nat
|
|
nmcli -t -f STATE general | grep -E '^connected'
|
|
getent ahostsv4 gentoo.org
|
|
for tool in hydra hydra-install qemu-system-x86_64 qemu-img virt-viewer foot labwc; do command -v "$tool"; done
|
|
hydra --help
|
|
qemu-system-x86_64 --version
|
|
qemu-img --version
|
|
test -r /etc/greetd/config.toml
|
|
find /usr/share -name OVMF_CODE.fd -print -quit | grep .
|
|
'''
|
|
|
|
NETWORK_CHECK = r'''set -eu
|
|
[ "$(cat /proc/sys/net/ipv4/ip_forward)" = 1 ]
|
|
python3 - <<'NETWORK'
|
|
import pathlib, secrets, subprocess, time
|
|
tag = secrets.token_hex(3)
|
|
lan, guest = 'hl' + tag, 'hv' + tag
|
|
uplink, lanpeer = 'hu' + tag, 'hp' + tag
|
|
bridgepeer, guestpeer = 'hb' + tag, 'hg' + tag
|
|
created = []
|
|
links = []
|
|
server = None
|
|
def run(*args):
|
|
return subprocess.run(args, check=True, capture_output=True, text=True, timeout=20).stdout
|
|
try:
|
|
for name in (lan, guest):
|
|
run('ip', 'netns', 'add', name)
|
|
created.append(name)
|
|
run('ip', 'link', 'add', uplink, 'type', 'veth', 'peer', 'name', lanpeer)
|
|
links.append(uplink)
|
|
run('ip', 'link', 'set', lanpeer, 'netns', lan)
|
|
run('ip', 'addr', 'add', '198.18.0.1/30', 'dev', uplink)
|
|
run('ip', 'link', 'set', uplink, 'up')
|
|
run('ip', '-n', lan, 'addr', 'add', '198.18.0.2/30', 'dev', lanpeer)
|
|
run('ip', '-n', lan, 'link', 'set', lanpeer, 'up')
|
|
run('ip', '-n', lan, 'link', 'set', 'lo', 'up')
|
|
run('ip', '-n', lan, 'route', 'add', '10.203.0.0/24', 'via', '198.18.0.1')
|
|
run('ip', 'link', 'add', bridgepeer, 'type', 'veth', 'peer', 'name', guestpeer)
|
|
links.append(bridgepeer)
|
|
run('ip', 'link', 'set', guestpeer, 'netns', guest)
|
|
run('ip', 'link', 'set', bridgepeer, 'master', 'hydrabr0')
|
|
run('ip', 'link', 'set', bridgepeer, 'up')
|
|
run('ip', '-n', guest, 'addr', 'add', '10.203.0.250/24', 'dev', guestpeer)
|
|
run('ip', '-n', guest, 'link', 'set', guestpeer, 'up')
|
|
run('ip', '-n', guest, 'link', 'set', 'lo', 'up')
|
|
run('ip', '-n', guest, 'route', 'add', 'default', 'via', '10.203.0.1')
|
|
port = 43117
|
|
server_code = "import socket; s=socket.socket(); s.settimeout(15); s.bind(('10.203.0.250',43117)); s.listen(1); c,a=s.accept(); c.sendall(b'HYDRA_LAN_TO_VM_OK'); c.close(); s.close()"
|
|
server = subprocess.Popen(['ip', 'netns', 'exec', guest, 'python3', '-c', server_code], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
client_code = "import socket,time; deadline=time.monotonic()+10\nwhile True:\n try:\n s=socket.create_connection(('10.203.0.250',43117),1); break\n except OSError:\n if time.monotonic()>deadline: raise\n time.sleep(.1)\nexpected=b'HYDRA_LAN_TO_VM_OK'; received=b''\nwhile len(received)<len(expected):\n remaining=deadline-time.monotonic()\n if remaining<=0: raise TimeoutError('TCP marker deadline exceeded')\n s.settimeout(remaining)\n part=s.recv(len(expected)-len(received))\n if not part: raise RuntimeError('Truncated TCP marker')\n received+=part\nassert received==expected; s.close(); print('PASS: routed new LAN-to-VM TCP connection on arbitrary port 43117')"
|
|
print(run('ip', 'netns', 'exec', lan, 'python3', '-c', client_code), end='')
|
|
assert server.wait(timeout=10) == 0
|
|
finally:
|
|
if server is not None and server.poll() is None:
|
|
server.terminate()
|
|
try: server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired: server.kill(); server.wait(timeout=5)
|
|
for link in reversed(links):
|
|
subprocess.run(['ip', 'link', 'del', link], check=False, capture_output=True, timeout=10)
|
|
for name in reversed(created):
|
|
subprocess.run(['ip', 'netns', 'del', name], check=False, capture_output=True, timeout=10)
|
|
NETWORK
|
|
'''
|
|
|
|
# A nested VM hardware smoke proves KVM initialisation/QMP, not guest OS boot.
|
|
NESTED_CHECK = r'''set -eu
|
|
if ! test -c /dev/kvm; then printf 'SKIP: guest /dev/kvm unavailable\n'; exit 0; fi
|
|
python3 - <<'NESTED'
|
|
import json, pathlib, socket, subprocess, tempfile, time
|
|
with tempfile.TemporaryDirectory(prefix='hydra-nested-') as tmp:
|
|
root = pathlib.Path(tmp)
|
|
disk = root / 'fresh.qcow2'
|
|
subprocess.run(['qemu-img', 'create', '-f', 'qcow2', str(disk), '64M'], check=True, timeout=30)
|
|
log = open(root / 'qemu.log', 'wb')
|
|
proc = subprocess.Popen(['qemu-system-x86_64', '-machine', 'q35,accel=kvm', '-cpu', 'host', '-m', '128', '-S', '-display', 'none', '-nodefaults', '-nic', 'bridge,br=hydrabr0,model=virtio-net-pci', '-drive', 'if=virtio,format=qcow2,file=' + str(disk), '-qmp', 'unix:' + str(root / 'qmp') + ',server=on,wait=off'], stdin=subprocess.DEVNULL, stdout=log, stderr=log)
|
|
sock = socket.socket(socket.AF_UNIX)
|
|
sock.settimeout(10)
|
|
try:
|
|
deadline = time.monotonic() + 15
|
|
while True:
|
|
try: sock.connect(str(root / 'qmp')); break
|
|
except OSError:
|
|
if proc.poll() is not None or time.monotonic() >= deadline: raise RuntimeError('Nested KVM start failed')
|
|
time.sleep(.1)
|
|
stream = sock.makefile('rwb', buffering=0)
|
|
assert 'QMP' in json.loads(stream.readline())
|
|
for request in ({'execute':'qmp_capabilities'}, {'execute':'query-kvm'}, {'execute':'quit'}):
|
|
stream.write(json.dumps(request).encode() + b'\n')
|
|
while True:
|
|
response = json.loads(stream.readline())
|
|
if 'return' in response or 'error' in response: break
|
|
assert 'error' not in response, response
|
|
if request['execute'] == 'query-kvm':
|
|
assert response['return']['enabled'], response
|
|
print('PASS: normal-user nested KVM VM and bridge helper; fresh qcow2; private QMP')
|
|
assert proc.wait(timeout=10) == 0
|
|
finally:
|
|
sock.close()
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
try: proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired: proc.kill(); proc.wait(timeout=5)
|
|
log.close()
|
|
NESTED
|
|
'''
|
|
|
|
|
|
def digest(path):
|
|
result = hashlib.sha256()
|
|
with open(path, 'rb') as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b''):
|
|
result.update(block)
|
|
return result.hexdigest()
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--iso', required=True, type=Path)
|
|
parser.add_argument('--ovmf-code', required=True, type=Path)
|
|
parser.add_argument('--ovmf-vars', required=True, type=Path)
|
|
parser.add_argument('--qemu', default='qemu-system-x86_64')
|
|
parser.add_argument('--qemu-img', default='qemu-img')
|
|
parser.add_argument('--scratch-parent', type=Path, default=Path(tempfile.gettempdir()))
|
|
parser.add_argument('--memory', type=int, default=4096)
|
|
parser.add_argument('--boot-timeout', type=int, default=360)
|
|
parser.add_argument('--install-timeout', type=int, default=1800)
|
|
args = parser.parse_args(argv)
|
|
for path in (args.iso, args.ovmf_code, args.ovmf_vars):
|
|
if not path.is_file():
|
|
parser.error(f'Missing input file: {path}')
|
|
if args.memory < 2048 or args.boot_timeout <= 0 or args.install_timeout <= 0:
|
|
parser.error('Memory must be >=2048 MiB and timeouts positive')
|
|
args.iso = args.iso.resolve()
|
|
args.ovmf_code = args.ovmf_code.resolve()
|
|
args.ovmf_vars = args.ovmf_vars.resolve()
|
|
scratch = Path(tempfile.mkdtemp(prefix='hydra-iso-smoke-', dir=args.scratch_parent))
|
|
if len(str(scratch / 'serial.sock').encode()) >= 104:
|
|
parser.error('Scratch path too long for Unix sockets; use /tmp')
|
|
print(f'Acceptance artifacts: {scratch}', flush=True)
|
|
report = {'schema': 1, 'status': 'running', 'scratch': str(scratch),
|
|
'iso': str(args.iso), 'iso_sha256': digest(args.iso),
|
|
'checks': {}, 'physical_hardware': 'pending',
|
|
'nested_scope': 'Normal-user KVM VM launch, bridge helper and QMP; no nested guest OS boot',
|
|
'ovmf_code_sha256': digest(args.ovmf_code),
|
|
'ovmf_vars_input_sha256': digest(args.ovmf_vars)}
|
|
password = secrets.token_urlsafe(24)
|
|
sentinel = secrets.token_hex(32)
|
|
vm = None
|
|
def checkpoint(name, output):
|
|
report['checks'][name] = {'status': 'skipped' if output.startswith(b'SKIP:') else 'passed',
|
|
'output': output.decode(errors='replace')}
|
|
(scratch / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
try:
|
|
disk = scratch / 'fresh-install.qcow2'
|
|
reserve_disk(disk)
|
|
subprocess.run([args.qemu_img, 'create', '-f', 'qcow2', str(disk), '32G'], check=True, timeout=30,
|
|
stdout=subprocess.DEVNULL)
|
|
shutil.copyfile(args.ovmf_vars, scratch / 'OVMF_VARS.fd')
|
|
os.chmod(scratch / 'OVMF_VARS.fd', 0o600)
|
|
vm = VM(args, scratch, disk, args.iso, password, 'live')
|
|
checkpoint('live_qmp_kvm', json.dumps(vm.kvm).encode())
|
|
console = vm.console
|
|
console.expect(rb'hydra-live[^\r\n]*[#$] ', args.boot_timeout)
|
|
checkpoint('live_boot', console.command(LIVE_CHECK, 90))
|
|
console.buffer = b''
|
|
console.send('sudo -n hydra-install --disk /dev/vda --user hydratest --execute --confirm-erase /dev/vda\n')
|
|
console.expect(rb'Installed account password: ', 120)
|
|
console.send(password + '\n')
|
|
console.expect(rb'Repeat password: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'Installation completed\. Diagnostic log: [^\r\n]+', args.install_timeout)
|
|
checkpoint('real_installer', b'Installer completed via guest controlling TTY')
|
|
console.command("sudo -n mkdir -p /mnt/hydra-validation && sudo -n mount /dev/vda2 /mnt/hydra-validation && printf %s " +
|
|
shlex.quote(sentinel) + " | sudo -n tee /mnt/hydra-validation/var/lib/hydra-acceptance-sentinel >/dev/null && sudo -n umount /mnt/hydra-validation", 60)
|
|
console.send('sudo -n poweroff\n')
|
|
vm.wait_poweroff(90)
|
|
vm.close()
|
|
vm = None
|
|
checkpoint('live_poweroff', b'QEMU exited before ISO-free reboot')
|
|
vm = VM(args, scratch, disk, None, password, 'installed')
|
|
checkpoint('installed_qmp_kvm', json.dumps(vm.kvm).encode())
|
|
console = vm.console
|
|
console.expect(rb'login: ', args.boot_timeout)
|
|
console.send('hydratest\n')
|
|
console.expect(rb'Password: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'hydratest[^\r\n]*[#$] ', 60)
|
|
output = console.command('id -un; if sudo -n true 2>/dev/null; then exit 1; fi', 30)
|
|
if output.strip() != b'hydratest':
|
|
raise SmokeError('Installed account identity mismatch')
|
|
checkpoint('installed_password_login', output)
|
|
console.buffer = b''
|
|
prompt_encoded = base64.b64encode(b'HYDRA_SUDO_PASSWORD: ').decode()
|
|
console.send(f'sudo -p "$(printf %s {prompt_encoded} | base64 -d)" -i\n')
|
|
console.expect(rb'HYDRA_SUDO_PASSWORD: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'root[^\r\n]*[#$] ', 30)
|
|
observed = console.command('cat /var/lib/hydra-acceptance-sentinel', 30)
|
|
if observed != sentinel.encode():
|
|
raise SmokeError('Persistence sentinel did not survive installed disk boot')
|
|
checkpoint('disk_persistence', b'Sentinel content verified from command output frame')
|
|
checkpoint('installed_contract', console.command(INSTALLED_CHECK, 120))
|
|
checkpoint('lan_to_vm_tcp', console.command(NETWORK_CHECK, 120))
|
|
checkpoint('nested_kvm', console.command('runuser -u hydratest -- sh -c ' + shlex.quote(NESTED_CHECK), 90))
|
|
checkpoint('installed_user_logout', installed_logout_probe(console, password))
|
|
console.send('poweroff\n')
|
|
vm.wait_poweroff(90)
|
|
checkpoint('installed_poweroff', b'Normal installed guest shutdown')
|
|
report['status'] = 'passed'
|
|
return 0
|
|
except (SmokeError, QMPError, OSError, subprocess.SubprocessError) as exc:
|
|
report['status'] = 'failed'
|
|
report['error'] = str(exc).replace(password, '[REDACTED]')
|
|
print(f'Acceptance failed: {report["error"]}', file=sys.stderr)
|
|
return 1
|
|
finally:
|
|
if vm:
|
|
vm.close()
|
|
report['completed_utc'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
|
(scratch / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
print(f'Report: {scratch / "report.json"}', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|