Continue guest-update rollback after individual restore failures

2026-10-13 15:29:28 +03:00
parent dcf1fbae40
commit 61a5628f9e
3 changed files with 111 additions and 16 deletions
@@ -179,17 +179,30 @@ def apply_bundle(bundle, root=Path('/'), *, execute=False):
        for name, (data, mode) in payload.items():
            atomic_write(checked_path(root, name), data, mode)
            changed.append(name)
-    except BaseException:
+    except BaseException as apply_error:
+        rollback_errors = {}
        for name in reversed(changed):
-            dest = checked_path(root, name)
-            original = records[name]
-            if 'symlink' in original:
-                dest.unlink()
-                dest.symlink_to(original['symlink'])
-            elif original['existed']:
-                atomic_write(dest, (backup / name).read_bytes(), original['mode'])
-            else:
-                dest.unlink()
+            try:
+                dest = checked_path(root, name)
+                original = records[name]
+                if 'symlink' in original:
+                    dest.unlink()
+                    dest.symlink_to(original['symlink'])
+                elif original['existed']:
+                    atomic_write(dest, (backup / name).read_bytes(), original['mode'])
+                else:
+                    dest.unlink(missing_ok=True)
+            except BaseException as restore_error:
+                # A full/unavailable filesystem may prevent one restoration
+                # while unlinking independently created files still succeeds.
+                rollback_errors[name] = f'{type(restore_error).__name__}: {restore_error}'
+        if rollback_errors:
+            report = ('Hydra OS update: rollback incomplete; the runtime may contain mixed versions.\n'
+                      f'Retained backup: {backup}\n'
+                      'Manual recovery records: ' + str(backup / 'backup.json') + '\n' +
+                      '\n'.join(f'  {name}: {error}' for name, error in rollback_errors.items()))
+            apply_error.add_note(report)
+            print(report, file=sys.stderr)
        raise
    result.update(applied=True, backup=str(backup))
    return result
@@ -44,8 +44,10 @@ hydra download 'https://example.org/distribution.iso'
 ```

 Add `--sha256` followed by the publisher's 64 hexadecimal digits to check the
-download. Format validation alone checks that the file looks like an ISO,
-not who published it. Validated files are cached under
+download. Supported images are ISO9660 and ISO9660/UDF hybrids; pure UDF
+images are currently refused because their completeness cannot be validated.
+Format validation checks the declared ISO9660 volume fits the file; it does not
+verify the publisher or optional trailing hybrid data. Validated files are cached under
 `~/.cache/hydra/iso` (or `$XDG_CACHE_HOME/hydra/iso`). Ctrl+C cancels an active
 download without publishing its partial file. HTTP is available for a local
 test transfer; prefer the publisher's HTTPS endpoint for Internet downloads.
@@ -113,10 +115,51 @@ python3 hydra-os-update.pyz
 sudo python3 hydra-os-update.pyz --apply
 ```

-The first command only prints the file inventory. After applying, log out of
-Hydra's graphical session and log in again so the long-running session process
-loads the new code. Reopening a terminal alone cannot reload that process.
-Running VMs remain independent of the graphical session.
+The first command only prints the file inventory. After applying, open the
+current Hydra recovery terminal with Ctrl+Alt+Shift+Escape, enter `:shell`, then
+run this command **inside that installed Hydra graphical session**:
+
+```sh
+labwc --exit
+```
+
+At the login screen, log in again as the same Hydra user. This restarts the
+long-running session process with the updated code. Use this from the recovery
+shell, not an SSH login or the outer host terminal. Reopening a terminal alone
+cannot reload that process. Running VMs remain independent of the graphical
+session.
+
+To reverse a successful update, or finish recovery after a reported partial
+rollback, use the exact retained backup path printed by the updater. Correct
+the reported filesystem problem first. Inspect its records inside Hydra:
+
+```sh
+backup='/var/lib/hydra/updates/REPLACE_WITH_THE_REPORTED_BACKUP'
+sudo python3 -m json.tool "$backup/backup.json"
+```
+
+Process every listed path relative to `/`. For an entry with `"existed": true`,
+restore that path from the same relative location in the backup. For example,
+after confirming the CLI entry originally existed:
+
+```sh
+relative='usr/lib/hydra/hydra/cli.py'
+sudo cp --no-dereference --remove-destination --preserve=mode,timestamps -- "$backup/$relative" "/$relative"
+```
+
+Use the same operation for other originally existing files. When `"symlink"`
+is recorded, restore the symlink itself with this command; do not copy its
+referent. In particular, restoring `etc/os-release` must recreate its recorded
+vendor symlink while leaving `/usr/lib/os-release` intact. Check the restored
+link with `readlink /etc/os-release` and compare it to `backup.json`.
+
+For each entry with `"existed": false`, remove only that exact installed file
+using `sudo rm -f -- /THE_RECORDED_RELATIVE_PATH`; no backup copy exists for it.
+This includes newly added modules, logos or the runtime-update receipt when
+listed as originally absent. Copying the backup directory over `/` alone will
+leave these additions behind. Confirm restored regular-file modes and SHA-256
+values against their records, retain the backup, then exit the Hydra graphical
+session and log in again as described above.

 This bundle contains the Fastfetch configuration and logo, but no executable
 packages. If the old installation lacks Fastfetch, install it inside Hydra:
@@ -143,3 +143,42 @@ class GuestUpdateTests(unittest.TestCase):
        self.assertEqual(self.old.read_text(), 'OLD = True\n')
        self.assertFalse((self.root / 'etc/motd').exists())
        self.assertFalse((self.root / 'usr/lib/hydra/bin/hydra').exists())
+
+    def test_persistent_restore_failures_attempt_all_cleanup_and_preserve_apply_error(self):
+        import contextlib
+        import io
+        launcher = self.root / 'usr/lib/hydra/bin/hydra'
+        launcher.parent.mkdir(parents=True)
+        launcher.write_text('OLD LAUNCHER\n')
+        self.build()
+        actual_write = self.mod.atomic_write
+        original_error = OSError('original apply failure: disk remains full')
+        calls = 0
+
+        def persistent_failure(*args):
+            nonlocal calls
+            calls += 1
+            if calls == 5:
+                raise original_error
+            if calls > 5:
+                raise OSError('persistent restore failure: disk remains full')
+            return actual_write(*args)
+
+        errors = io.StringIO()
+        with contextlib.redirect_stderr(errors), patch.object(self.mod, 'atomic_write', side_effect=persistent_failure):
+            with self.assertRaises(OSError) as raised:
+                self.mod.apply_bundle(self.bundle, self.root, execute=True)
+        self.assertIs(raised.exception, original_error)
+        self.assertEqual(self.old.read_text(), 'NEW = True\n')
+        self.assertFalse((self.root / 'etc/motd').exists())
+        self.assertFalse((self.root / 'usr/lib/hydra/hydra/__init__.py').exists())
+        backups = list((self.root / 'var/lib/hydra/updates').iterdir())
+        self.assertEqual(len(backups), 1)
+        backup = backups[0]
+        self.assertEqual((backup / 'usr/lib/hydra/hydra/cli.py').read_text(), 'OLD = True\n')
+        self.assertEqual((backup / 'usr/lib/hydra/bin/hydra').read_text(), 'OLD LAUNCHER\n')
+        self.assertIn(str(backup), errors.getvalue())
+        self.assertIn('rollback incomplete', errors.getvalue().lower())
+        self.assertIn('usr/lib/hydra/hydra/cli.py', errors.getvalue())
+        self.assertIn('usr/lib/hydra/bin/hydra', errors.getvalue())
+        self.assertIn('persistent restore failure', errors.getvalue())