1160 lines
56 KiB
Python
1160 lines
56 KiB
Python
#!/usr/bin/env python3
|
|
"""Boot/install/reboot acceptance on a fresh, isolated UEFI/KVM VM only.
|
|
|
|
No existing disk option, host sudo, SSH, host TTY, or GUI is used. Artifacts and
|
|
logs remain in a new private scratch directory even after failure.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import inspect
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import select
|
|
import shlex
|
|
import shutil
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from hydra.qmp import QMPClient, QMPError
|
|
|
|
|
|
class SmokeError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def lifecycle_socket(path, *, connect_socket=True):
|
|
path = Path(path)
|
|
info = path.lstat()
|
|
if not stat.S_ISSOCK(info.st_mode) or info.st_uid != os.getuid():
|
|
raise SmokeError('Lifecycle socket is not owned by the installed user')
|
|
identity = [info.st_dev, info.st_ino, info.st_uid, info.st_mode]
|
|
if connect_socket:
|
|
with socket.socket(socket.AF_UNIX) as connection:
|
|
connection.settimeout(2)
|
|
connection.connect(str(path))
|
|
after = path.lstat()
|
|
if identity != [after.st_dev, after.st_ino, after.st_uid, after.st_mode]:
|
|
raise SmokeError('Lifecycle socket changed while connecting')
|
|
return identity
|
|
|
|
|
|
def lifecycle_snapshot(context):
|
|
"""Prove live identity and owned runtime endpoints, never trust PID alone."""
|
|
try:
|
|
if context['uid'] != os.getuid():
|
|
raise SmokeError('Lifecycle probe is not the installed normal user')
|
|
runtime = Path(context['runtime'])
|
|
run = runtime / context['name']
|
|
directories = {}
|
|
for directory in (runtime, run):
|
|
info = directory.lstat()
|
|
if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid()
|
|
or stat.S_IMODE(info.st_mode) != 0o700):
|
|
raise SmokeError('Lifecycle runtime must remain private and owned')
|
|
directories[str(directory)] = [info.st_dev, info.st_ino, info.st_mode]
|
|
metadata = json.loads((run / 'identity.json').read_text())
|
|
if metadata != {'name': context['name'], 'uuid': context['uuid']}:
|
|
raise SmokeError('Lifecycle metadata is stale or foreign')
|
|
pid = int((run / 'qemu.pid').read_text())
|
|
process = Path('/proc') / str(pid)
|
|
if process.stat().st_uid != os.getuid():
|
|
raise SmokeError('Lifecycle process belongs to another user')
|
|
fields = (process / 'stat').read_text().rsplit(')', 1)[1].split()
|
|
if fields[0] == 'Z':
|
|
raise SmokeError('Lifecycle process has exited')
|
|
argv = (process / 'cmdline').read_bytes().split(b'\0')
|
|
for option, expected in ((b'-name', context['name']), (b'-uuid', context['uuid'])):
|
|
if argv[argv.index(option) + 1] != expected.encode():
|
|
raise SmokeError('Lifecycle PID is not the owned VM identity')
|
|
endpoints = {'qmp': lifecycle_socket(run / 'qmp.sock', connect_socket=False)}
|
|
with QMPClient(run / 'qmp.sock', timeout=2) as client:
|
|
if (client.command('query-name') != {'name': context['name']}
|
|
or client.command('query-uuid') != {'UUID': context['uuid']}):
|
|
raise SmokeError('Lifecycle QMP identity is foreign')
|
|
if client.command('query-status').get('status') != 'running':
|
|
raise SmokeError('Lifecycle VM is not running')
|
|
if not client.command('query-kvm').get('enabled'):
|
|
raise SmokeError('Lifecycle VM did not initialise nested KVM')
|
|
endpoints['spice'] = lifecycle_socket(run / 'spice.sock')
|
|
audio = Path(context['user_runtime'])
|
|
endpoints['pipewire'] = lifecycle_socket(audio / 'pipewire-0')
|
|
endpoints['pulse'] = lifecycle_socket(audio / 'pulse/native')
|
|
services = {}
|
|
for service in context.get('audio_services', []):
|
|
subprocess.run(['systemctl', '--user', 'is-active', service], check=True,
|
|
capture_output=True, timeout=5)
|
|
services[service] = subprocess.run(['systemctl', '--user', 'show', service,
|
|
'-p', 'InvocationID', '-p', 'ExecMainPID'], check=True,
|
|
capture_output=True, text=True, timeout=5).stdout
|
|
return {'name': context['name'], 'uuid': context['uuid'], 'pid': pid,
|
|
'process_start': fields[19], 'directories': directories,
|
|
'sockets': endpoints, 'audio_services': services}
|
|
except (OSError, ValueError, KeyError, IndexError, QMPError, subprocess.SubprocessError) as exc:
|
|
raise SmokeError('Lifecycle runtime or process was lost: ' + str(exc)) from exc
|
|
|
|
|
|
def verify_lifecycle_survival(before, after):
|
|
if before != after:
|
|
raise SmokeError('VM identity, private runtime, or audio endpoints changed across logout')
|
|
|
|
|
|
def login_session_records(uid):
|
|
"""Read logind's public properties, avoiding display-table row semantics."""
|
|
def query(*args):
|
|
return subprocess.run(['loginctl', *args], check=True, capture_output=True,
|
|
text=True, timeout=5).stdout
|
|
try:
|
|
ids = query('show-user', str(uid), '-p', 'Sessions', '--value').split()
|
|
records = []
|
|
for ident in ids:
|
|
output = query('show-session', ident, '--all', '--no-pager',
|
|
'-p', 'Id', '-p', 'User', '-p', 'State', '-p', 'Class',
|
|
'-p', 'Leader', '-p', 'TTY')
|
|
record = dict(line.split('=', 1) for line in output.splitlines())
|
|
if not {'Id', 'User', 'State', 'Class', 'Leader', 'TTY'} <= record.keys():
|
|
raise SmokeError('Incomplete logind session identity')
|
|
record['User'], record['Leader'] = int(record['User']), int(record['Leader'])
|
|
if record['Id'] != ident or record['User'] != uid:
|
|
raise SmokeError('Logind returned a foreign session identity')
|
|
records.append(record)
|
|
return records
|
|
except (OSError, ValueError, subprocess.SubprocessError) as exc:
|
|
raise SmokeError('Cannot establish guest login session state: ' + str(exc)) from exc
|
|
|
|
|
|
def is_open_login(record):
|
|
# systemd v258: closing session scopes can remain while user processes
|
|
# survive. Manager-class records describe bookkeeping, not a PAM login.
|
|
users = {'user', 'user-early', 'user-incomplete', 'user-light', 'user-early-light'}
|
|
bookkeeping = {'manager', 'manager-early', 'background', 'background-light',
|
|
'greeter', 'lock-screen', 'none'}
|
|
if record['State'] not in {'opening', 'online', 'active', 'closing'}:
|
|
raise SmokeError('Unknown logind session state')
|
|
if record['Class'] not in users | bookkeeping:
|
|
raise SmokeError('Unknown logind session class')
|
|
return record['Class'] in users and record['State'] != 'closing'
|
|
|
|
|
|
def require_initial_login(uid, ident):
|
|
opened = [record for record in login_session_records(uid) if is_open_login(record)]
|
|
if (len(opened) != 1 or opened[0]['Id'] != ident or opened[0]['TTY'] != 'ttyS0'
|
|
or opened[0]['Leader'] <= 0 or opened[0]['State'] not in {'online', 'active'}):
|
|
raise SmokeError('Lifecycle test requires exactly its actual serial PAM login')
|
|
return opened[0]
|
|
|
|
|
|
def no_open_logins(context):
|
|
original = context['login_session']
|
|
records = login_session_records(context['uid'])
|
|
for record in records:
|
|
if record['Id'] == original['Id']:
|
|
if (record['User'] != original['User'] or record['TTY'] != original['TTY']
|
|
or record['Class'] != original['Class']
|
|
or record['Leader'] not in {0, original['Leader']}):
|
|
raise SmokeError('Original PAM session identity was replaced')
|
|
if record['State'] != 'closing':
|
|
return False
|
|
if is_open_login(record):
|
|
return False
|
|
return True
|
|
|
|
|
|
def capture_logout_snapshot(context):
|
|
if not no_open_logins(context):
|
|
raise SmokeError('An open login overlaps the logout sample')
|
|
during = lifecycle_snapshot(context)
|
|
if not no_open_logins(context):
|
|
raise SmokeError('A login opened while the logout sample was running')
|
|
verify_lifecycle_survival(context['before'], during)
|
|
return during
|
|
|
|
|
|
LIFECYCLE_BODY = r'''
|
|
def run(*args):
|
|
return subprocess.run(args, check=True, capture_output=True, text=True, timeout=40).stdout
|
|
|
|
def cli(context, *args):
|
|
return json.loads(run('hydra', '--config', context['config'], '--state', context['state'],
|
|
'--runtime', context['runtime'], *args))
|
|
|
|
def write_report(path, data):
|
|
temporary = path.with_suffix('.tmp')
|
|
temporary.write_text(json.dumps(data, indent=2) + '\n')
|
|
temporary.chmod(0o600)
|
|
temporary.replace(path)
|
|
|
|
mode, base_arg = sys.argv[1:]
|
|
base = Path(base_arg)
|
|
context_path = base / 'context.json'
|
|
observation_path = base / 'logout-observation.json'
|
|
if mode == 'start':
|
|
if not Path('/dev/kvm').exists():
|
|
print('SKIP: installed guest nested KVM unavailable')
|
|
raise SystemExit(0)
|
|
if os.getuid() == 0 or not os.access('/dev/kvm', os.R_OK | os.W_OK):
|
|
raise SmokeError('Lifecycle test needs installed-user nested KVM access')
|
|
user_runtime = Path(os.environ['XDG_RUNTIME_DIR'])
|
|
if user_runtime != Path('/run/user') / str(os.getuid()):
|
|
raise SmokeError('Lifecycle test needs the real login runtime directory')
|
|
if run('loginctl', 'show-user', str(os.getuid()), '-p', 'Linger', '--value').strip() != 'no':
|
|
raise SmokeError('Lingering would mask the installed logout policy')
|
|
login_session = require_initial_login(os.getuid(), os.environ['XDG_SESSION_ID'])
|
|
name = 'logout-' + base.name.rsplit('-', 1)[1]
|
|
state = base / 'state'
|
|
runtime = user_runtime / base.name
|
|
context = {'uid': os.getuid(), 'name': name, 'config': str(base / 'config'),
|
|
'state': str(state), 'runtime': str(runtime), 'user_runtime': str(user_runtime),
|
|
'uuid': str(uuid.uuid5(uuid.NAMESPACE_URL, str(state / name / 'disk.qcow2'))),
|
|
'login_session': login_session,
|
|
'audio_services': ['pipewire.service', 'pipewire-pulse.service', 'wireplumber.service']}
|
|
run('systemctl', '--user', 'start', *context['audio_services'])
|
|
cli(context, 'add', name, '--memory', '256', '--cpus', '1', '--disk-size', '64M',
|
|
'--graphics', 'software', '--network', 'bridge', '--bridge', 'hydrabr0')
|
|
if cli(context, 'start', name).get('status') != 'running':
|
|
raise SmokeError('Real installed hydra CLI failed to start lifecycle VM')
|
|
try:
|
|
disk = Path(context['state']) / name / 'disk.qcow2'
|
|
info = disk.lstat()
|
|
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_uid != os.getuid():
|
|
raise SmokeError('Lifecycle VM disk is not a new owned regular qcow2')
|
|
context['before'] = lifecycle_snapshot(context)
|
|
except Exception:
|
|
# CLI control is scoped to the newly created private identity.
|
|
cli(context, 'stop', name, '--force')
|
|
raise
|
|
write_report(context_path, context)
|
|
with open(base / 'monitor.log', 'ab', buffering=0) as log:
|
|
subprocess.Popen([sys.executable, __file__, 'monitor', str(base)],
|
|
stdin=subprocess.DEVNULL, stdout=log, stderr=log,
|
|
start_new_session=True)
|
|
print('READY: normal-user Hydra VM and audio sockets recorded before real logout')
|
|
elif mode == 'monitor':
|
|
context = json.loads(context_path.read_text())
|
|
try:
|
|
deadline = time.monotonic() + 35
|
|
while not no_open_logins(context):
|
|
if time.monotonic() >= deadline:
|
|
raise SmokeError('Serial session did not actually log out')
|
|
time.sleep(.2)
|
|
# Exceed systemd's usual 10-second user-manager stop delay. No helper
|
|
# login or lingering is allowed to keep the tested user session alive.
|
|
time.sleep(15)
|
|
during = capture_logout_snapshot(context)
|
|
write_report(observation_path, {'status': 'passed', 'no_open_logins': True,
|
|
'survival_seconds': 15, 'during_logout': during})
|
|
except Exception as exc:
|
|
write_report(observation_path, {'status': 'failed', 'error': str(exc)})
|
|
raise
|
|
elif mode == 'finish':
|
|
context = json.loads(context_path.read_text())
|
|
try:
|
|
if os.environ.get('XDG_RUNTIME_DIR') != context['user_runtime']:
|
|
raise SmokeError('Relogin replaced the user runtime path')
|
|
deadline = time.monotonic() + 10
|
|
while not observation_path.is_file():
|
|
if time.monotonic() >= deadline:
|
|
raise SmokeError('Logout monitor died or did not record survival')
|
|
time.sleep(.2)
|
|
observation = json.loads(observation_path.read_text())
|
|
if observation.get('status') != 'passed' or observation.get('no_open_logins') is not True:
|
|
raise SmokeError('No-open-login observation failed: ' + json.dumps(observation))
|
|
after = lifecycle_snapshot(context)
|
|
verify_lifecycle_survival(context['before'], after)
|
|
if cli(context, 'status', context['name']).get('status') != 'running':
|
|
raise SmokeError('Real installed hydra CLI could not reconnect after login')
|
|
print(json.dumps({'status': 'passed', 'real_logout': True, 'no_open_logins': True,
|
|
'survival_seconds': observation['survival_seconds'], 'same_vm': after,
|
|
'audio_scope': 'Socket and service continuity only; signal/playback untested'}))
|
|
finally:
|
|
# Real product CLI verifies metadata and live QMP name/UUID before quit.
|
|
# Never signal a saved PID or control a VM outside these private paths.
|
|
cleanup_status = cli(context, 'status', context['name']).get('status')
|
|
if cleanup_status == 'running':
|
|
run_path = Path(context['runtime']) / context['name']
|
|
if json.loads((run_path / 'identity.json').read_text()) != {'name': context['name'], 'uuid': context['uuid']}:
|
|
raise SmokeError('Refusing lifecycle cleanup for a foreign identity')
|
|
with QMPClient(run_path / 'qmp.sock', timeout=2) as client:
|
|
if (client.command('query-name') != {'name': context['name']}
|
|
or client.command('query-uuid') != {'UUID': context['uuid']}):
|
|
raise SmokeError('Refusing lifecycle cleanup for foreign live QMP')
|
|
if cli(context, 'stop', context['name'], '--force').get('status') != 'stopped':
|
|
raise SmokeError('Owned lifecycle VM did not stop')
|
|
elif cleanup_status != 'stopped':
|
|
raise SmokeError('Cannot safely verify the lifecycle VM for cleanup')
|
|
else:
|
|
raise SmokeError('Unknown lifecycle mode')
|
|
'''
|
|
|
|
|
|
def lifecycle_guest_script():
|
|
helpers = (lifecycle_socket, lifecycle_snapshot, verify_lifecycle_survival,
|
|
login_session_records, is_open_login, require_initial_login,
|
|
no_open_logins, capture_logout_snapshot)
|
|
return ('import json, os, socket, stat, subprocess, sys, time, uuid\n'
|
|
'from pathlib import Path\nsys.path.insert(0, "/usr/lib/hydra")\n'
|
|
'from hydra.qmp import QMPClient, QMPError\n'
|
|
'class SmokeError(RuntimeError): pass\n' +
|
|
'\n'.join(inspect.getsource(helper) for helper in helpers) + LIFECYCLE_BODY)
|
|
|
|
|
|
def terminal_ux_iso():
|
|
"""An empty ISO9660 filesystem, intentionally without an El Torito boot image."""
|
|
def both(value, width):
|
|
return value.to_bytes(width, 'little') + value.to_bytes(width, 'big')
|
|
|
|
def directory_record(identifier):
|
|
record = bytearray(34)
|
|
record[0] = 34
|
|
record[2:10] = both(20, 4)
|
|
record[10:18] = both(2048, 4)
|
|
record[18:25] = bytes((126, 10, 13, 0, 0, 0, 0))
|
|
record[25] = 2
|
|
record[28:32] = both(1, 2)
|
|
record[32] = 1
|
|
record[33] = identifier
|
|
return record
|
|
|
|
data = bytearray(21 * 2048)
|
|
primary = bytearray(2048)
|
|
primary[:7] = b'\x01CD001\x01'
|
|
primary[8:40] = b'HYDRA OS'.ljust(32, b' ')
|
|
primary[40:72] = b'HYDRA_UX_PROBE'.ljust(32, b' ')
|
|
primary[80:88] = both(21, 4)
|
|
primary[120:124] = both(1, 2)
|
|
primary[124:128] = both(1, 2)
|
|
primary[128:132] = both(2048, 2)
|
|
primary[132:140] = both(10, 4)
|
|
primary[140:144] = (18).to_bytes(4, 'little')
|
|
primary[148:152] = (19).to_bytes(4, 'big')
|
|
primary[156:190] = directory_record(0)
|
|
primary[190:813] = b' ' * 623
|
|
primary[813:830] = b'2026101300000000\x00'
|
|
primary[830:847] = b'2026101300000000\x00'
|
|
primary[847:881] = b'0000000000000000\x00' * 2
|
|
primary[881] = 1
|
|
data[16 * 2048:17 * 2048] = primary
|
|
data[17 * 2048:17 * 2048 + 7] = b'\xffCD001\x01'
|
|
for sector, order in ((18, 'little'), (19, 'big')):
|
|
data[sector * 2048:sector * 2048 + 10] = (
|
|
b'\x01\x00' + (20).to_bytes(4, order) + (1).to_bytes(2, order) + b'\x00\x00')
|
|
data[20 * 2048:20 * 2048 + 68] = directory_record(0) + directory_record(1)
|
|
return bytes(data)
|
|
|
|
|
|
def verify_terminal_ux_branding(output, errors):
|
|
rendered = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', output)
|
|
if errors.strip() or re.search(r'\b(error|failed|unknown module)\b', rendered, re.I):
|
|
raise SmokeError('Installed default Fastfetch emitted errors: ' + (errors or rendered)[-1500:])
|
|
if not re.search(r'\bOS\s*:\s*Hydra OS\b', rendered):
|
|
raise SmokeError('Installed default Fastfetch did not render the Hydra OS pretty name')
|
|
if 'H Y D R A O S' not in rendered or rendered.count('(o >') != 3:
|
|
raise SmokeError('Installed default Fastfetch did not render the three-headed Hydra logo')
|
|
return rendered
|
|
|
|
|
|
def terminal_ux_snapshot(context):
|
|
"""Verify CLI-owned live process/QMP identity, excluding OS boot or audio claims."""
|
|
try:
|
|
if context['uid'] != os.getuid():
|
|
raise SmokeError('Terminal UX process is not owned by the installed normal user')
|
|
runtime = Path(context['runtime'])
|
|
run = runtime / context['name']
|
|
for directory in (runtime, run):
|
|
info = directory.lstat()
|
|
if (not stat.S_ISDIR(info.st_mode) or info.st_uid != context['uid']
|
|
or stat.S_IMODE(info.st_mode) != 0o700):
|
|
raise SmokeError('Terminal UX runtime is not private and owned')
|
|
if json.loads((run / 'identity.json').read_text()) != {
|
|
'name': context['name'], 'uuid': context['uuid']}:
|
|
raise SmokeError('Terminal UX metadata belongs to a different VM')
|
|
pid = int((run / 'qemu.pid').read_text())
|
|
process = Path('/proc') / str(pid)
|
|
fields = (process / 'stat').read_text().rsplit(')', 1)[1].split()
|
|
if process.stat().st_uid != context['uid'] or fields[0] == 'Z':
|
|
raise SmokeError('Terminal UX VM process exited or changed owner')
|
|
argv = (process / 'cmdline').read_bytes().split(b'\0')
|
|
for option, expected in ((b'-name', context['name']), (b'-uuid', context['uuid'])):
|
|
if argv[argv.index(option) + 1] != expected.encode():
|
|
raise SmokeError('Terminal UX PID belongs to a different VM')
|
|
lifecycle_socket(run / 'qmp.sock', connect_socket=False)
|
|
lifecycle_socket(run / 'spice.sock')
|
|
with QMPClient(run / 'qmp.sock', timeout=2) as client:
|
|
if (client.command('query-name') != {'name': context['name']}
|
|
or client.command('query-uuid') != {'UUID': context['uuid']}):
|
|
raise SmokeError('Terminal UX QMP belongs to a different VM')
|
|
if client.command('query-status').get('status') != 'running':
|
|
raise SmokeError('Terminal UX VM is not running')
|
|
if not client.command('query-kvm').get('enabled'):
|
|
raise SmokeError('Terminal UX VM did not initialise nested KVM')
|
|
return {'name': context['name'], 'uuid': context['uuid'], 'pid': pid,
|
|
'process_start': fields[19], 'kvm': True}
|
|
except (OSError, ValueError, KeyError, IndexError, QMPError) as exc:
|
|
raise SmokeError('Terminal UX cannot establish owned live identity: ' + str(exc)) from exc
|
|
|
|
|
|
TERMINAL_UX_BODY = r'''
|
|
if os.getuid() == 0 or subprocess.check_output(['id', '-un'], text=True).strip() != 'hydratest':
|
|
raise SmokeError('Terminal UX probe must run as the installed normal hydratest user')
|
|
if not os.access('/dev/kvm', os.R_OK | os.W_OK):
|
|
raise SmokeError('Terminal UX needs installed-user nested KVM access')
|
|
if not Path('/usr/bin/hydra').is_file():
|
|
raise SmokeError('Installed product launcher /usr/bin/hydra is missing')
|
|
fetch = subprocess.run(['fastfetch', '--pipe', 'false', '--show-errors'],
|
|
capture_output=True, text=True, timeout=30)
|
|
if fetch.returncode:
|
|
raise SmokeError('Installed default Fastfetch failed: ' + fetch.stderr[-1500:])
|
|
fastfetch_output = verify_terminal_ux_branding(fetch.stdout, fetch.stderr)
|
|
config_hash = hashlib.sha256(Path('/etc/fastfetch/config.jsonc').read_bytes()).hexdigest()
|
|
logo_hash = hashlib.sha256(Path('/usr/share/hydra/fastfetch.txt').read_bytes()).hexdigest()
|
|
base = Path(tempfile.mkdtemp(prefix='hux-', dir='/tmp'))
|
|
name = 'ux-' + secrets.token_hex(4)
|
|
context = {'name': name, 'uid': os.getuid(), 'config': str(base / 'config'),
|
|
'state': str(base / 'state'), 'runtime': str(base / 'run')}
|
|
context['uuid'] = str(uuid.uuid5(uuid.NAMESPACE_URL, str(base / 'state' / name / 'disk.qcow2')))
|
|
environment = dict(os.environ, XDG_CACHE_HOME=str(base / 'cache'))
|
|
environment.pop('HYDRA_SESSION_MANIFEST', None)
|
|
command = ['/usr/bin/hydra', '--config', context['config'], '--state', context['state'],
|
|
'--runtime', context['runtime']]
|
|
|
|
def cli(*arguments, input=None):
|
|
result = subprocess.run(command + list(arguments), input=input, env=environment,
|
|
capture_output=True, text=True, timeout=60)
|
|
if result.returncode:
|
|
raise SmokeError('Installed hydra ' + arguments[0] + ' failed: ' +
|
|
(result.stdout + result.stderr)[-2000:])
|
|
return result.stdout
|
|
|
|
def cli_json(*arguments):
|
|
return json.loads(cli(*arguments))
|
|
|
|
fixture = base / 'fixture.iso'
|
|
fixture.write_bytes(terminal_ux_iso())
|
|
iso_hash = hashlib.sha256(fixture.read_bytes()).hexdigest()
|
|
requests = []
|
|
class ISOHandler(http.server.BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
requests.append(self.path)
|
|
if self.path != '/fixture.iso':
|
|
self.send_error(404)
|
|
return
|
|
payload = fixture.read_bytes()
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', 'application/x-iso9660-image')
|
|
self.send_header('Content-Length', str(len(payload)))
|
|
self.end_headers()
|
|
self.wfile.write(payload)
|
|
def log_message(self, *args):
|
|
pass
|
|
server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), ISOHandler)
|
|
worker = threading.Thread(target=server.serve_forever, daemon=True)
|
|
worker.start()
|
|
url = 'http://127.0.0.1:' + str(server.server_port) + '/fixture.iso'
|
|
result = None
|
|
try:
|
|
if cli_json('list'):
|
|
raise SmokeError('Terminal UX private configuration was not initially empty')
|
|
empty_menu = cli('menu', input=':quit\n')
|
|
if ('Hydra OS' not in empty_menu or 'No VM profiles' not in empty_menu
|
|
or list(base.glob('run/*/qemu.pid')) or cli_json('list')):
|
|
raise SmokeError('Installed empty menu was unreadable or started a VM')
|
|
run_output = cli('run', url, '--name', name, '--memory', '512', '--cpus', '1',
|
|
'--disk-size', '64M', '--sha256', iso_hash, '--no-view')
|
|
if requests != ['/fixture.iso']:
|
|
raise SmokeError('Installed hydra run did not download exactly its loopback ISO')
|
|
before = terminal_ux_snapshot(context)
|
|
profiles = cli_json('list')
|
|
if len(profiles) != 1 or profiles[0]['name'] != name:
|
|
raise SmokeError('Installed hydra run did not create exactly the requested private profile')
|
|
cached_iso = Path(profiles[0]['iso'])
|
|
cache = base / 'cache' / 'hydra' / 'iso'
|
|
info = cached_iso.lstat()
|
|
if (cached_iso.parent != cache or not stat.S_ISREG(info.st_mode)
|
|
or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600
|
|
or hashlib.sha256(cached_iso.read_bytes()).hexdigest() != iso_hash
|
|
or list(cache.glob('*.part'))):
|
|
raise SmokeError('Installed hydra run cache is unsafe, incomplete or has a different checksum')
|
|
profile = tomllib.loads((base / 'config' / (name + '.toml')).read_text())
|
|
if (profile['memory'], profile['cpus'], profile['disk_size']) != (512, 1, '64M'):
|
|
raise SmokeError('Installed hydra run lost its explicit allocations')
|
|
downloaded = cli('download', url, '--sha256', iso_hash)
|
|
if (Path(downloaded.splitlines()[-1]) != cached_iso or requests != ['/fixture.iso']
|
|
or hashlib.sha256(cached_iso.read_bytes()).hexdigest() != iso_hash):
|
|
raise SmokeError('Installed hydra download did not reuse the complete cached ISO')
|
|
cli('open', name, '--no-view')
|
|
after_open = terminal_ux_snapshot(context)
|
|
if before != after_open:
|
|
raise SmokeError('Installed hydra open restarted or replaced its running VM')
|
|
menu = cli('menu', input=':quit\n')
|
|
after_menu = terminal_ux_snapshot(context)
|
|
if ('Hydra OS' not in menu or name not in menu or 'running' not in menu
|
|
or '512 MiB' not in menu or '"status":' in menu or before != after_menu):
|
|
raise SmokeError('Installed running menu was unreadable or changed VM lifetime')
|
|
result = {'status': 'passed', 'uid': os.getuid(), 'normal_user': 'hydratest',
|
|
'launcher': '/usr/bin/hydra', 'fixture_bytes': fixture.stat().st_size,
|
|
'iso_sha256': iso_hash, 'cache_reused': True, 'http_requests': requests,
|
|
'before': before, 'after_open': after_open, 'after_menu': after_menu,
|
|
'memory_mib': profile['memory'], 'cpus': profile['cpus'],
|
|
'disk_size': profile['disk_size'], 'network': profile['network'],
|
|
'empty_menu': empty_menu, 'running_menu': menu, 'run_output': run_output,
|
|
'download_output': downloaded, 'fastfetch_command': ['fastfetch', '--pipe', 'false', '--show-errors'],
|
|
'fastfetch_default_config_sha256': config_hash, 'fastfetch_logo_sha256': logo_hash,
|
|
'fastfetch_output': fastfetch_output,
|
|
'scope': 'Packaged installed CLI, cache, branding and live nested QMP/KVM; nonboot ISO fixture, no nested OS boot or performance claim'}
|
|
finally:
|
|
try:
|
|
status = cli_json('status', name).get('status')
|
|
if status == 'running':
|
|
# Never signal a saved PID: verify the exact private live identity
|
|
# and let the product CLI perform its own identity-checked QMP quit.
|
|
terminal_ux_snapshot(context)
|
|
if cli_json('stop', name, '--force').get('status') != 'stopped':
|
|
raise SmokeError('Owned terminal UX fixture VM did not stop')
|
|
elif status != 'stopped':
|
|
raise SmokeError('Refusing terminal UX cleanup of an unverified VM identity at ' + str(base))
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
worker.join(timeout=5)
|
|
if result is not None:
|
|
result['cleanup'] = 'Only the verified private fixture VM stopped through hydra stop --force'
|
|
print(json.dumps(result))
|
|
shutil.rmtree(base)
|
|
'''
|
|
|
|
|
|
def terminal_ux_guest_script():
|
|
helpers = (lifecycle_socket, terminal_ux_iso, verify_terminal_ux_branding, terminal_ux_snapshot)
|
|
return ('import hashlib, http.server, json, os, re, secrets, shutil, socket, stat, subprocess, sys, tempfile, threading, tomllib, uuid\n'
|
|
'from pathlib import Path\nsys.path.insert(0, "/usr/lib/hydra")\n'
|
|
'from hydra.qmp import QMPClient, QMPError\n'
|
|
'class SmokeError(RuntimeError): pass\n' +
|
|
'\n'.join(inspect.getsource(helper) for helper in helpers) + TERMINAL_UX_BODY)
|
|
|
|
|
|
def installed_terminal_ux_probe(console):
|
|
preparation = ("python3 - <<'HYDRA_TERMINAL_UX_SOURCE'\n" + terminal_ux_guest_script() +
|
|
'\nHYDRA_TERMINAL_UX_SOURCE')
|
|
output = console.command(preparation, 180)
|
|
try:
|
|
result = json.loads(output)
|
|
except ValueError as exc:
|
|
raise SmokeError('Installed terminal UX did not return framed JSON evidence') from exc
|
|
if result.get('status') != 'passed':
|
|
raise SmokeError('Installed terminal UX did not verify its packaged CLI')
|
|
return output
|
|
|
|
|
|
def installed_admin(console, password):
|
|
console.buffer = b''
|
|
prompt_encoded = base64.b64encode(b'HYDRA_SUDO_PASSWORD: ').decode()
|
|
console.send(f'sudo -k -p "$(printf %s {prompt_encoded} | base64 -d)" -i\n')
|
|
console.expect(rb'HYDRA_SUDO_PASSWORD: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'root[^\r\n]*[#$] ', 30)
|
|
|
|
|
|
def installed_logout_probe(console, password):
|
|
# Exit only the guest sudo shell, then work in its actual getty PAM login.
|
|
console.send('exit\n')
|
|
console.expect(rb'hydratest[^\r\n]*[#$] ', 30)
|
|
if console.command('id -un', 10).strip() != b'hydratest':
|
|
raise SmokeError('Logout probe did not return to installed user login')
|
|
base = '/home/hydratest/hydra-logout-' + secrets.token_hex(4)
|
|
script = base + '/probe.py'
|
|
preparation = (f'umask 077; mkdir {shlex.quote(base)} && cat > {shlex.quote(script)} '
|
|
"<<'HYDRA_LIFECYCLE_SOURCE'\n" + lifecycle_guest_script() + '\nHYDRA_LIFECYCLE_SOURCE\n' +
|
|
f'python3 {shlex.quote(script)} start {shlex.quote(base)}')
|
|
ready = console.command(preparation, 120)
|
|
if ready.startswith(b'SKIP:'):
|
|
installed_admin(console, password)
|
|
return ready
|
|
if b'READY:' not in ready:
|
|
raise SmokeError('Lifecycle VM did not become ready before logout')
|
|
console.buffer = b''
|
|
console.send('exit\n')
|
|
console.expect(rb'login: ', 30)
|
|
# This is intentionally a period at the actual login prompt with no
|
|
# replacement login. The guest monitor independently confirms no open logins.
|
|
time.sleep(20)
|
|
console.send('hydratest\n')
|
|
console.expect(rb'Password: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'hydratest[^\r\n]*[#$] ', 60)
|
|
result = console.command(f'python3 {shlex.quote(script)} finish {shlex.quote(base)}', 90)
|
|
installed_admin(console, password)
|
|
return result
|
|
|
|
|
|
def reserve_disk(path):
|
|
try:
|
|
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
|
|
os.close(fd)
|
|
except OSError as exc:
|
|
raise SmokeError('Test disk must be a newly created regular file') from exc
|
|
|
|
|
|
def owned_file(path, scratch):
|
|
path, scratch = Path(path), Path(scratch)
|
|
try:
|
|
info = path.lstat()
|
|
except OSError as exc:
|
|
raise SmokeError('Missing disposable drive') from exc
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_nlink != 1
|
|
or path.parent.resolve() != scratch.resolve()
|
|
or info.st_uid != os.getuid()):
|
|
raise SmokeError('Drive must be an owned, unlinked regular file directly in scratch')
|
|
|
|
|
|
def qemu_path(path):
|
|
# QEMU keyval arguments use commas as separators. Refuse ambiguous paths.
|
|
if any(c in str(path) for c in ',\n\r\x00'):
|
|
raise SmokeError('QEMU file paths cannot contain comma or control characters')
|
|
return str(path)
|
|
|
|
|
|
def qemu_argv(qemu, scratch, disk, code, variables, iso, memory):
|
|
scratch = Path(scratch)
|
|
owned_file(disk, scratch)
|
|
owned_file(variables, scratch)
|
|
argv = [qemu, '-machine', 'q35,accel=kvm', '-cpu', 'host', '-smp', '2',
|
|
'-m', str(memory), '-display', 'none', '-monitor', 'none',
|
|
'-no-reboot', '-nic', 'user,model=virtio-net-pci',
|
|
'-drive', f'if=pflash,format=raw,readonly=on,file={qemu_path(code)}',
|
|
'-drive', f'if=pflash,format=raw,file={qemu_path(variables)}',
|
|
'-drive', f'if=virtio,format=qcow2,file={qemu_path(disk)}',
|
|
'-chardev', f'socket,id=console,path={qemu_path(scratch / "serial.sock")},server=on,wait=off',
|
|
'-serial', 'chardev:console', '-qmp', f'unix:{qemu_path(scratch / "qmp.sock")},server=on,wait=off']
|
|
if iso is not None:
|
|
if not Path(iso).is_file():
|
|
raise SmokeError('Trusted ISO must exist as a regular file')
|
|
argv += ['-drive', f'file={qemu_path(iso)},media=cdrom,format=raw,readonly=on',
|
|
'-boot', 'order=d,strict=on']
|
|
else:
|
|
argv += ['-boot', 'order=c,strict=on']
|
|
return argv
|
|
|
|
|
|
def frame_command(command, token):
|
|
if not re.fullmatch(r'[a-zA-Z0-9]+', token):
|
|
raise SmokeError('Invalid framing token')
|
|
encoded = base64.b64encode(command.encode()).decode()
|
|
# Linux canonical serial TTY input is bounded per physical line. Continue
|
|
# the shell expression across short lines for substantial guest probes.
|
|
chunks = ' \\\n'.join(shlex.quote(encoded[i:i + 768]) for i in range(0, len(encoded), 768)) or "''"
|
|
# Literal control bytes appear only in output, never in the echoed command.
|
|
return (f"printf '\\036{token}:BEGIN\\037'; "
|
|
f"sh -c \"$(printf %s {chunks} | base64 -d)\"; "
|
|
f"rc=$?; printf '\\036{token}:END:%s\\037' \"$rc\"")
|
|
|
|
|
|
def parse_frame(data, token):
|
|
begin = b'\x1e' + token.encode() + b':BEGIN\x1f'
|
|
start = data.find(begin)
|
|
if start < 0:
|
|
return None
|
|
end = re.search(rb'\x1e' + token.encode() + rb':END:(\d+)\x1f', data[start + len(begin):])
|
|
if end is None:
|
|
return None
|
|
output = data[start + len(begin):start + len(begin) + end.start()]
|
|
return int(end.group(1)), output
|
|
|
|
|
|
def installer_command(token):
|
|
# A literal control-byte return frame cannot be forged by terminal echo.
|
|
# Keep the installer on this controlling TTY for getpass, then mark every
|
|
# shell return, including an unexpected status zero without completion.
|
|
return ('sudo -n hydra-install --disk /dev/vda --user hydratest --execute '
|
|
'--confirm-erase /dev/vda; rc=$?; '
|
|
f"printf '\\036{token}:INSTALL_EXIT:%s\\037' \"$rc\"")
|
|
|
|
|
|
class Console:
|
|
def __init__(self, connection, logfile, passwords):
|
|
self.sock = connection
|
|
self.log = open(logfile, 'ab', buffering=0)
|
|
self.passwords = [p.encode() for p in passwords if p]
|
|
self.pending_log = b''
|
|
self.buffer = b''
|
|
self.closed = False
|
|
|
|
def record(self, data, final=False):
|
|
self.pending_log += data
|
|
for password in self.passwords:
|
|
self.pending_log = self.pending_log.replace(password, b'[REDACTED]')
|
|
keep = 0
|
|
if not final:
|
|
for password in self.passwords:
|
|
for length in range(1, len(password)):
|
|
if self.pending_log.endswith(password[:length]):
|
|
keep = max(keep, length)
|
|
if keep:
|
|
self.log.write(self.pending_log[:-keep])
|
|
self.pending_log = self.pending_log[-keep:]
|
|
else:
|
|
self.log.write(self.pending_log)
|
|
self.pending_log = b''
|
|
|
|
def close(self):
|
|
if not self.closed:
|
|
self.record(b'', final=True)
|
|
self.log.close()
|
|
self.sock.close()
|
|
self.closed = True
|
|
|
|
def send(self, data):
|
|
self.sock.sendall(data.encode() if isinstance(data, str) else data)
|
|
|
|
def read(self, timeout):
|
|
ready, _, _ = select.select([self.sock], [], [], max(0, timeout))
|
|
if not ready:
|
|
return b''
|
|
data = self.sock.recv(65536)
|
|
if not data:
|
|
raise SmokeError('Guest console closed')
|
|
self.record(data)
|
|
self.buffer += data
|
|
if len(self.buffer) > 16 * 1024 * 1024:
|
|
raise SmokeError('Console exceeded bounded capture size')
|
|
return data
|
|
|
|
def expect(self, pattern, timeout, failure_pattern=None):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
match = re.search(pattern, self.buffer)
|
|
failure = re.search(failure_pattern, self.buffer) if failure_pattern else None
|
|
if failure and (not match or failure.start() < match.start()):
|
|
evidence = failure.group()
|
|
for password in self.passwords:
|
|
evidence = evidence.replace(password, b'[REDACTED]')
|
|
raise SmokeError('Installer exited before expected console state: ' +
|
|
evidence.decode(errors='replace'))
|
|
if match:
|
|
self.buffer = self.buffer[match.end():]
|
|
return match
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise SmokeError('Timed out waiting for guest console state')
|
|
self.read(min(remaining, 1))
|
|
|
|
def expect_installer(self, pattern, timeout, token):
|
|
failure = (rb'hydra-install: [^\r\n]*[\r\n]|\x1e' +
|
|
re.escape(token.encode()) + rb':INSTALL_EXIT:[0-9]+\x1f')
|
|
return self.expect(pattern, timeout, failure_pattern=failure)
|
|
|
|
def finish_installer(self, token, timeout=30):
|
|
# Completion text can arrive before the process exits and readline
|
|
# resumes. Input sent during that gap can be discarded by the TTY.
|
|
deadline = time.monotonic() + timeout
|
|
returned = self.expect(rb'\x1e' + re.escape(token.encode()) +
|
|
rb':INSTALL_EXIT:([0-9]+)\x1f', timeout)
|
|
status = int(returned.group(1))
|
|
if status:
|
|
raise SmokeError(f'Installer returned status {status} after completion')
|
|
self.expect(rb'hydra-live[^\r\n]*[#$] ', max(0, deadline - time.monotonic()))
|
|
|
|
def command(self, command, timeout=60):
|
|
token = secrets.token_hex(12)
|
|
self.buffer = b''
|
|
self.send(frame_command(command, token) + '\n')
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
parsed = parse_frame(self.buffer, token)
|
|
if parsed is not None:
|
|
status, output = parsed
|
|
self.buffer = b''
|
|
if status:
|
|
raise SmokeError(f'Guest probe failed with status {status}: {output.decode(errors="replace")[-3000:]}')
|
|
return output.replace(b'\r\n', b'\n')
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise SmokeError('Timed out waiting for framed guest command')
|
|
self.read(min(remaining, 1))
|
|
|
|
|
|
def connect(path, process, timeout=30):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if process.poll() is not None:
|
|
raise SmokeError(f'Test QEMU exited with status {process.returncode}')
|
|
connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
try:
|
|
connection.connect(str(path))
|
|
return connection
|
|
except OSError:
|
|
connection.close()
|
|
time.sleep(0.1)
|
|
raise SmokeError('Timed out connecting to private test socket')
|
|
|
|
|
|
class VM:
|
|
def __init__(self, args, scratch, disk, iso, password, phase):
|
|
self.scratch = scratch
|
|
for name in ('serial.sock', 'qmp.sock'):
|
|
(scratch / name).unlink(missing_ok=True)
|
|
self.stderr = open(scratch / f'{phase}-qemu.log', 'ab', buffering=0)
|
|
self.process = subprocess.Popen(qemu_argv(args.qemu, scratch, disk,
|
|
args.ovmf_code, scratch / 'OVMF_VARS.fd', iso, args.memory),
|
|
stdin=subprocess.DEVNULL, stdout=self.stderr, stderr=self.stderr)
|
|
self.console = None
|
|
self.qmp = None
|
|
try:
|
|
self.console = Console(connect(scratch / 'serial.sock', self.process),
|
|
scratch / f'{phase}-serial.log', [password])
|
|
ready = connect(scratch / 'qmp.sock', self.process)
|
|
ready.close()
|
|
self.qmp = QMPClient(scratch / 'qmp.sock', timeout=5)
|
|
self.kvm = self.qmp.command('query-kvm')
|
|
if not self.kvm.get('enabled'):
|
|
raise SmokeError('Disposable QEMU is not using KVM')
|
|
except BaseException:
|
|
self.close()
|
|
raise
|
|
|
|
def wait_poweroff(self, timeout):
|
|
deadline = time.monotonic() + timeout
|
|
serial_open = True
|
|
while self.process.poll() is None:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise SmokeError('Guest failed to power off in time')
|
|
if serial_open:
|
|
try:
|
|
self.console.read(min(remaining, 0.25))
|
|
except SmokeError as exc:
|
|
if str(exc) != 'Guest console closed':
|
|
raise
|
|
serial_open = False
|
|
else:
|
|
try:
|
|
self.process.wait(timeout=remaining)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise SmokeError('Guest failed to power off in time') from exc
|
|
# Capture bytes queued immediately before exit, even if poll observed
|
|
# the child exiting before the first serial read.
|
|
while serial_open:
|
|
try:
|
|
if not self.console.read(0):
|
|
break
|
|
except SmokeError as exc:
|
|
if str(exc) != 'Guest console closed':
|
|
raise
|
|
serial_open = False
|
|
rc = self.process.returncode
|
|
if rc:
|
|
raise SmokeError(f'Test QEMU failed during poweroff ({rc})')
|
|
|
|
def close(self):
|
|
# Only the Popen child owned by this harness is signalled.
|
|
if self.qmp:
|
|
if self.process.poll() is None:
|
|
try:
|
|
self.qmp.command('quit')
|
|
self.process.wait(timeout=5)
|
|
except (QMPError, subprocess.TimeoutExpired):
|
|
pass
|
|
self.qmp.close()
|
|
if self.process.poll() is None:
|
|
self.process.terminate()
|
|
try:
|
|
self.process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
self.process.kill()
|
|
self.process.wait(timeout=10)
|
|
if self.console:
|
|
self.console.close()
|
|
self.stderr.close()
|
|
|
|
|
|
LIVE_CHECK = r'''set -eu
|
|
[ "$(id -un)" = hydra-live ]
|
|
test -e /sys/firmware/efi
|
|
test -f /etc/hydra/live-mode/active
|
|
findmnt -n -o FSTYPE / | grep -qx overlay
|
|
test -f /run/initramfs/live/LiveOS/squashfs.img
|
|
sudo -n true
|
|
for tool in hydra hydra-install qemu-system-x86_64 qemu-img remote-viewer foot labwc; do command -v "$tool"; done
|
|
for service in NetworkManager hydra-network hydra-dnsmasq; do systemctl is-active "$service"; done
|
|
'''
|
|
|
|
INSTALLED_CHECK = r'''set -eu
|
|
assert_no_match() {
|
|
if grep "$@"; then
|
|
printf 'Forbidden live configuration remains\n' >&2
|
|
exit 1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || exit "$status"
|
|
fi
|
|
}
|
|
[ "$(findmnt -n -o FSTYPE /)" = ext4 ]
|
|
[ "$(readlink -f "$(findmnt -n -o SOURCE /)")" = /dev/vda2 ]
|
|
assert_no_match -Eq 'root=live:|rd\.live' /proc/cmdline
|
|
test ! -e /etc/hydra/live-mode
|
|
test ! -L /etc/hydra/live-mode
|
|
if getent passwd hydra-live; then
|
|
printf 'Live account remains\n' >&2
|
|
exit 1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 2 ] || exit "$status"
|
|
fi
|
|
remaining=$(find /etc/systemd/system /etc/sudoers.d \( -name 90-hydra-live.conf -o -name 90-hydra-live \) -print)
|
|
[ -z "$remaining" ]
|
|
test ! -e /etc/greetd/live.toml
|
|
test ! -L /etc/greetd/live.toml
|
|
for directory in /etc/systemd/system/getty@*.service.d /etc/systemd/system/serial-getty@*.service.d /etc/sudoers.d; do
|
|
if [ -d "$directory" ]; then
|
|
assert_no_match -R -E -- '--autologin|NOPASSWD' "$directory"
|
|
elif [ -e "$directory" ] || [ -L "$directory" ]; then
|
|
printf 'Unexpected non-directory configuration path\n' >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
getent shadow root | cut -d: -f2 | grep -Eq '^[!*]'
|
|
getent shadow hydratest | cut -d: -f2 | grep -Eq '^\$'
|
|
for service in NetworkManager greetd hydra-network hydra-dnsmasq; do systemctl is-active "$service"; done
|
|
[ "$(systemctl show hydra-bridge-permissions -p Result --value)" = success ]
|
|
[ "$(systemctl show hydra-bridge-permissions -p ExecMainStatus --value)" = 0 ]
|
|
ip -4 -o addr show hydrabr0 | grep -F '10.203.0.1/24'
|
|
ip -4 route show default | grep '^default '
|
|
nft list table ip hydra_nat
|
|
nmcli -t -f STATE general | grep -E '^connected'
|
|
getent ahostsv4 gentoo.org
|
|
for tool in hydra hydra-install qemu-system-x86_64 qemu-img remote-viewer foot labwc; do command -v "$tool"; done
|
|
hydra --help
|
|
qemu-system-x86_64 --version
|
|
qemu-img --version
|
|
test -r /etc/greetd/config.toml
|
|
find /usr/share -name OVMF_CODE.fd -print -quit | grep .
|
|
'''
|
|
|
|
NETWORK_CHECK = r'''set -eu
|
|
[ "$(cat /proc/sys/net/ipv4/ip_forward)" = 1 ]
|
|
python3 - <<'NETWORK'
|
|
import pathlib, secrets, subprocess, time
|
|
tag = secrets.token_hex(3)
|
|
lan, guest = 'hl' + tag, 'hv' + tag
|
|
uplink, lanpeer = 'hu' + tag, 'hp' + tag
|
|
bridgepeer, guestpeer = 'hb' + tag, 'hg' + tag
|
|
created = []
|
|
links = []
|
|
server = None
|
|
def run(*args):
|
|
return subprocess.run(args, check=True, capture_output=True, text=True, timeout=20).stdout
|
|
try:
|
|
for name in (lan, guest):
|
|
run('ip', 'netns', 'add', name)
|
|
created.append(name)
|
|
run('ip', 'link', 'add', uplink, 'type', 'veth', 'peer', 'name', lanpeer)
|
|
links.append(uplink)
|
|
run('ip', 'link', 'set', lanpeer, 'netns', lan)
|
|
run('ip', 'addr', 'add', '198.18.0.1/30', 'dev', uplink)
|
|
run('ip', 'link', 'set', uplink, 'up')
|
|
run('ip', '-n', lan, 'addr', 'add', '198.18.0.2/30', 'dev', lanpeer)
|
|
run('ip', '-n', lan, 'link', 'set', lanpeer, 'up')
|
|
run('ip', '-n', lan, 'link', 'set', 'lo', 'up')
|
|
run('ip', '-n', lan, 'route', 'add', '10.203.0.0/24', 'via', '198.18.0.1')
|
|
run('ip', 'link', 'add', bridgepeer, 'type', 'veth', 'peer', 'name', guestpeer)
|
|
links.append(bridgepeer)
|
|
run('ip', 'link', 'set', guestpeer, 'netns', guest)
|
|
run('ip', 'link', 'set', bridgepeer, 'master', 'hydrabr0')
|
|
run('ip', 'link', 'set', bridgepeer, 'up')
|
|
run('ip', '-n', guest, 'addr', 'add', '10.203.0.250/24', 'dev', guestpeer)
|
|
run('ip', '-n', guest, 'link', 'set', guestpeer, 'up')
|
|
run('ip', '-n', guest, 'link', 'set', 'lo', 'up')
|
|
run('ip', '-n', guest, 'route', 'add', 'default', 'via', '10.203.0.1')
|
|
port = 43117
|
|
server_code = "import socket; s=socket.socket(); s.settimeout(15); s.bind(('10.203.0.250',43117)); s.listen(1); c,a=s.accept(); c.sendall(b'HYDRA_LAN_TO_VM_OK'); c.close(); s.close()"
|
|
server = subprocess.Popen(['ip', 'netns', 'exec', guest, 'python3', '-c', server_code], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
client_code = "import socket,time; deadline=time.monotonic()+10\nwhile True:\n try:\n s=socket.create_connection(('10.203.0.250',43117),1); break\n except OSError:\n if time.monotonic()>deadline: raise\n time.sleep(.1)\nexpected=b'HYDRA_LAN_TO_VM_OK'; received=b''\nwhile len(received)<len(expected):\n remaining=deadline-time.monotonic()\n if remaining<=0: raise TimeoutError('TCP marker deadline exceeded')\n s.settimeout(remaining)\n part=s.recv(len(expected)-len(received))\n if not part: raise RuntimeError('Truncated TCP marker')\n received+=part\nassert received==expected; s.close(); print('PASS: routed new LAN-to-VM TCP connection on arbitrary port 43117')"
|
|
print(run('ip', 'netns', 'exec', lan, 'python3', '-c', client_code), end='')
|
|
assert server.wait(timeout=10) == 0
|
|
finally:
|
|
if server is not None and server.poll() is None:
|
|
server.terminate()
|
|
try: server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired: server.kill(); server.wait(timeout=5)
|
|
for link in reversed(links):
|
|
subprocess.run(['ip', 'link', 'del', link], check=False, capture_output=True, timeout=10)
|
|
for name in reversed(created):
|
|
subprocess.run(['ip', 'netns', 'del', name], check=False, capture_output=True, timeout=10)
|
|
NETWORK
|
|
'''
|
|
|
|
# A nested VM hardware smoke proves KVM initialisation/QMP, not guest OS boot.
|
|
NESTED_CHECK = r'''set -eu
|
|
if ! test -c /dev/kvm; then printf 'SKIP: guest /dev/kvm unavailable\n'; exit 0; fi
|
|
python3 - <<'NESTED'
|
|
import json, pathlib, socket, subprocess, tempfile, time
|
|
with tempfile.TemporaryDirectory(prefix='hydra-nested-') as tmp:
|
|
root = pathlib.Path(tmp)
|
|
disk = root / 'fresh.qcow2'
|
|
subprocess.run(['qemu-img', 'create', '-f', 'qcow2', str(disk), '64M'], check=True, timeout=30)
|
|
log = open(root / 'qemu.log', 'wb')
|
|
proc = subprocess.Popen(['qemu-system-x86_64', '-machine', 'q35,accel=kvm', '-cpu', 'host', '-m', '128', '-S', '-display', 'none', '-nodefaults', '-nic', 'bridge,br=hydrabr0,model=virtio-net-pci', '-drive', 'if=virtio,format=qcow2,file=' + str(disk), '-qmp', 'unix:' + str(root / 'qmp') + ',server=on,wait=off'], stdin=subprocess.DEVNULL, stdout=log, stderr=log)
|
|
sock = socket.socket(socket.AF_UNIX)
|
|
sock.settimeout(10)
|
|
try:
|
|
deadline = time.monotonic() + 15
|
|
while True:
|
|
try: sock.connect(str(root / 'qmp')); break
|
|
except OSError:
|
|
if proc.poll() is not None or time.monotonic() >= deadline: raise RuntimeError('Nested KVM start failed')
|
|
time.sleep(.1)
|
|
stream = sock.makefile('rwb', buffering=0)
|
|
assert 'QMP' in json.loads(stream.readline())
|
|
for request in ({'execute':'qmp_capabilities'}, {'execute':'query-kvm'}, {'execute':'quit'}):
|
|
stream.write(json.dumps(request).encode() + b'\n')
|
|
while True:
|
|
response = json.loads(stream.readline())
|
|
if 'return' in response or 'error' in response: break
|
|
assert 'error' not in response, response
|
|
if request['execute'] == 'query-kvm':
|
|
assert response['return']['enabled'], response
|
|
print('PASS: normal-user nested KVM VM and bridge helper; fresh qcow2; private QMP')
|
|
assert proc.wait(timeout=10) == 0
|
|
finally:
|
|
sock.close()
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
try: proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired: proc.kill(); proc.wait(timeout=5)
|
|
log.close()
|
|
NESTED
|
|
'''
|
|
|
|
|
|
def digest(path):
|
|
result = hashlib.sha256()
|
|
with open(path, 'rb') as source:
|
|
for block in iter(lambda: source.read(1024 * 1024), b''):
|
|
result.update(block)
|
|
return result.hexdigest()
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--iso', required=True, type=Path)
|
|
parser.add_argument('--ovmf-code', required=True, type=Path)
|
|
parser.add_argument('--ovmf-vars', required=True, type=Path)
|
|
parser.add_argument('--qemu', default='qemu-system-x86_64')
|
|
parser.add_argument('--qemu-img', default='qemu-img')
|
|
parser.add_argument('--scratch-parent', type=Path, default=Path(tempfile.gettempdir()))
|
|
parser.add_argument('--memory', type=int, default=4096)
|
|
parser.add_argument('--boot-timeout', type=int, default=360)
|
|
parser.add_argument('--install-timeout', type=int, default=1800)
|
|
args = parser.parse_args(argv)
|
|
for path in (args.iso, args.ovmf_code, args.ovmf_vars):
|
|
if not path.is_file():
|
|
parser.error(f'Missing input file: {path}')
|
|
if args.memory < 2048 or args.boot_timeout <= 0 or args.install_timeout <= 0:
|
|
parser.error('Memory must be >=2048 MiB and timeouts positive')
|
|
args.iso = args.iso.resolve()
|
|
args.ovmf_code = args.ovmf_code.resolve()
|
|
args.ovmf_vars = args.ovmf_vars.resolve()
|
|
scratch = Path(tempfile.mkdtemp(prefix='hydra-iso-smoke-', dir=args.scratch_parent))
|
|
if len(str(scratch / 'serial.sock').encode()) >= 104:
|
|
parser.error('Scratch path too long for Unix sockets; use /tmp')
|
|
print(f'Acceptance artifacts: {scratch}', flush=True)
|
|
report = {'schema': 1, 'status': 'running', 'scratch': str(scratch),
|
|
'iso': str(args.iso), 'iso_sha256': digest(args.iso),
|
|
'checks': {}, 'physical_hardware': 'pending',
|
|
'nested_scope': 'Normal-user KVM VM launch, bridge helper and QMP; no nested guest OS boot',
|
|
'ovmf_code_sha256': digest(args.ovmf_code),
|
|
'ovmf_vars_input_sha256': digest(args.ovmf_vars)}
|
|
password = secrets.token_urlsafe(24)
|
|
sentinel = secrets.token_hex(32)
|
|
vm = None
|
|
def checkpoint(name, output):
|
|
report['checks'][name] = {'status': 'skipped' if output.startswith(b'SKIP:') else 'passed',
|
|
'output': output.decode(errors='replace')}
|
|
(scratch / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
try:
|
|
disk = scratch / 'fresh-install.qcow2'
|
|
reserve_disk(disk)
|
|
subprocess.run([args.qemu_img, 'create', '-f', 'qcow2', str(disk), '32G'], check=True, timeout=30,
|
|
stdout=subprocess.DEVNULL)
|
|
shutil.copyfile(args.ovmf_vars, scratch / 'OVMF_VARS.fd')
|
|
os.chmod(scratch / 'OVMF_VARS.fd', 0o600)
|
|
vm = VM(args, scratch, disk, args.iso, password, 'live')
|
|
checkpoint('live_qmp_kvm', json.dumps(vm.kvm).encode())
|
|
console = vm.console
|
|
console.expect(rb'hydra-live[^\r\n]*[#$] ', args.boot_timeout)
|
|
checkpoint('live_boot', console.command(LIVE_CHECK, 90))
|
|
console.buffer = b''
|
|
installer_token = secrets.token_hex(12)
|
|
console.send(installer_command(installer_token) + '\n')
|
|
console.expect_installer(rb'Installed account password: ', 120, installer_token)
|
|
console.send(password + '\n')
|
|
console.expect_installer(rb'Repeat password: ', 30, installer_token)
|
|
console.send(password + '\n')
|
|
console.expect_installer(rb'Installation completed\. Diagnostic log: [^\r\n]+',
|
|
args.install_timeout, installer_token)
|
|
console.finish_installer(installer_token)
|
|
checkpoint('real_installer', b'Installer completed via guest controlling TTY')
|
|
console.command("sudo -n mkdir -p /mnt/hydra-validation && sudo -n mount /dev/vda2 /mnt/hydra-validation && printf %s " +
|
|
shlex.quote(sentinel) + " | sudo -n tee /mnt/hydra-validation/var/lib/hydra-acceptance-sentinel >/dev/null && sudo -n umount /mnt/hydra-validation", 60)
|
|
console.send('sudo -n poweroff\n')
|
|
vm.wait_poweroff(90)
|
|
vm.close()
|
|
vm = None
|
|
checkpoint('live_poweroff', b'QEMU exited before ISO-free reboot')
|
|
vm = VM(args, scratch, disk, None, password, 'installed')
|
|
checkpoint('installed_qmp_kvm', json.dumps(vm.kvm).encode())
|
|
console = vm.console
|
|
console.expect(rb'login: ', args.boot_timeout)
|
|
console.send('hydratest\n')
|
|
console.expect(rb'Password: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'hydratest[^\r\n]*[#$] ', 60)
|
|
output = console.command('id -un; if sudo -n true 2>/dev/null; then exit 1; fi', 30)
|
|
if output.strip() != b'hydratest':
|
|
raise SmokeError('Installed account identity mismatch')
|
|
checkpoint('installed_password_login', output)
|
|
checkpoint('installed_terminal_ux', installed_terminal_ux_probe(console))
|
|
console.buffer = b''
|
|
prompt_encoded = base64.b64encode(b'HYDRA_SUDO_PASSWORD: ').decode()
|
|
console.send(f'sudo -p "$(printf %s {prompt_encoded} | base64 -d)" -i\n')
|
|
console.expect(rb'HYDRA_SUDO_PASSWORD: ', 30)
|
|
console.send(password + '\n')
|
|
console.expect(rb'root[^\r\n]*[#$] ', 30)
|
|
observed = console.command('cat /var/lib/hydra-acceptance-sentinel', 30)
|
|
if observed != sentinel.encode():
|
|
raise SmokeError('Persistence sentinel did not survive installed disk boot')
|
|
checkpoint('disk_persistence', b'Sentinel content verified from command output frame')
|
|
checkpoint('installed_contract', console.command(INSTALLED_CHECK, 120))
|
|
checkpoint('lan_to_vm_tcp', console.command(NETWORK_CHECK, 120))
|
|
checkpoint('nested_kvm', console.command('runuser -u hydratest -- sh -c ' + shlex.quote(NESTED_CHECK), 90))
|
|
checkpoint('installed_user_logout', installed_logout_probe(console, password))
|
|
console.send('poweroff\n')
|
|
vm.wait_poweroff(90)
|
|
checkpoint('installed_poweroff', b'Normal installed guest shutdown')
|
|
report['status'] = 'passed'
|
|
return 0
|
|
except (SmokeError, QMPError, OSError, subprocess.SubprocessError) as exc:
|
|
report['status'] = 'failed'
|
|
report['error'] = str(exc).replace(password, '[REDACTED]')
|
|
print(f'Acceptance failed: {report["error"]}', file=sys.stderr)
|
|
return 1
|
|
finally:
|
|
if vm:
|
|
vm.close()
|
|
report['completed_utc'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
|
(scratch / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
print(f'Report: {scratch / "report.json"}', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|