Automatically run release validation tests in CI against multiple base systems

.github/workflows/release-validation.yml

@@ -2,10 +2,24 @@ name: "Release Validation"
 permissions:
   contents: read
 on:
+  push:
+    branches:
+      - "release/*"
+
   workflow_dispatch:
 jobs:
   release-validation-tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # hard-coding for now, can alternatively expose diskImageURLs via
+        # passthru and then dynamically take e.g. first 2 and last 2
+        baseSystemVersion:
+          - "2023.9.1-DISK"
+          - "2025.3.1"
+          - "previous" # uses default in release-validation.nix
+
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/prep-build-env
@@ -14,4 +28,9 @@ jobs:
         NIX_CACHE_ACCESS_KEY_ID: ${{ secrets.NIX_CACHE_ACCESS_KEY_ID }}
         NIX_CACHE_SECRET_ACCESS_KEY: ${{ secrets.NIX_CACHE_SECRET_ACCESS_KEY }}
         NIX_BINARY_CACHE_PRIVATE_KEY: ${{ secrets.NIX_BINARY_CACHE_PRIVATE_KEY }}
-    - run: NIXPKGS_ALLOW_UNFREE=1 nix-build testing/release-validation.nix
+    - run: |
+        baseSystemVersionFlag=""
+        if ! [[ "${{ matrix.baseSystemVersion }}" == "previous" ]]; then
+          baseSystemVersionFlag="--arg baseSystemVersion \"${{ matrix.baseSystemVersion }}\""
+        fi
+        NIXPKGS_ALLOW_UNFREE=1 nix-build ${baseSystemVersionFlag} testing/release-validation.nix

testing/release-validation.nix

@@ -250,7 +250,9 @@ with TestPrecondition("dnsmasq hands out an IP to playos"):
 
 with TestPrecondition("kiosk is open with kiosk URL") as t:
     wait_until_passes(
-        lambda: t.assertIn("Hello world", screenshot_and_ocr(playos))
+        lambda: t.assertIn("Hello world", screenshot_and_ocr(playos)),
+        retries=60, # can take quite long on CI
+        sleep=2
     )
 
 # move mouse to bottom right corner so it doesn't accidentally cover
@@ -264,9 +266,10 @@ with TestPrecondition("controller GUI is visible") as t:
     def t_check():
         screen_text = screenshot_and_ocr(playos)
         t.assertIn("Information", screen_text)
+        time.sleep(2) # ensure page fully loaded
         return screen_text
 
-    screen_text = wait_until_passes(t_check, retries=3)
+    screen_text = wait_until_passes(t_check, retries=10)
 
     t.assertIn("Version", screen_text)
     t.assertIn("${baseSystemVersion}", screen_text)
@@ -278,9 +281,9 @@ with TestPrecondition("controller GUI is visible") as t:
 # and then mouse_move'ing there for a click
 def navigate_to_system_status():
     for _ in range(4):
-        playos.send_key("tab")
-    playos.send_key("ret")
-    time.sleep(1)
+        playos.send_key("tab", delay=0.2)
+    playos.send_key("ret", delay=0.2)
+    time.sleep(2)
 
 with TestPrecondition("Navigate to System Status page") as t:
     navigate_to_system_status()
@@ -325,30 +328,19 @@ with TestCase("controller has downloaded and installed the bundle") as t:
         t.fail(f"Update process failed with an error, last screen text: {screen_text}")
 
 # Reboot to new system
-
-
-with TestPrecondition("TEMP WORKAROUND: Try to trigger an fsync/unmount of /boot"):
-    # Avoid FAT corruption of /boot/status.ini, which can happen despite the
-    # presence of `statusfile-recovery.service`
-    #
-    # The workaround is problematic, since this can actually happen on a real
-    # system, but we need it to be able to check if slot status is marked as
-    # good after the reboot.
-    time.sleep(30) # opportunistically wait for a sync
-
-    # simulate a Power key long-press to initiate a clean shutdown
-    long_press_duration_seconds = 5.5 # empirically determined :-)
-    playos.send_monitor_command(f"sendkey power {round(long_press_duration_seconds*1000)}")
-    # let the VM partially shut down, but not all the way, otherwise
-    # system_reset will not work
-    time.sleep(long_press_duration_seconds + 0.1)
-
-playos.send_monitor_command("system_reset")
+# Note: done via root shell on tty1, since a QEMU system_reset corrupts the
+# /boot/status.ini due to unclean unmount + FAT
+playos.send_key("ctrl-alt-f8", delay=2) # direct switch to tty1 prevented by limit-vtes.nix
+playos.send_key("ctrl-alt-f1", delay=2)
+playos.send_chars("root\n")
+time.sleep(2)
+playos.send_chars("systemctl reboot\n")
 
 with TestCase("kiosk is open with kiosk URL after reboot") as t:
     wait_until_passes(
         lambda: t.assertIn("Hello world", screenshot_and_ocr(playos)),
-        retries=60
+        retries=60,
+        sleep=2
     )
 
 playos.send_monitor_command("mouse_move 2000 2000")
@@ -357,7 +349,8 @@ with TestCase("controller GUI with new version is visible") as t:
     # switch to controller
     playos.send_key("ctrl-shift-f12")
     wait_until_passes(
-        lambda: t.assertIn("${nextSystemVersion}", screenshot_and_ocr(playos))
+        lambda: t.assertIn("${nextSystemVersion}", screenshot_and_ocr(playos)),
+        retries=10
     )
 
 with TestCase("The new booted version reaches a Good state") as t: