andrewlukoshko · Sep 24, 2026 · Sep 23, 2026
diff --git a/.github/actions/shared-steps/action.yml b/.github/actions/shared-steps/action.yml
@@ -9,7 +9,7 @@ inputs:
     description: 'AlmaLinux variant to build (e.g., 9, 10, kitten)'
     required: true
   arch:
-    description: 'Target architecture (x86_64, aarch64, ppc64le or s390x - the last two built under QEMU TCG on an x86_64 runner)'
+    description: 'Target architecture (x86_64, aarch64, ppc64le or s390x - ppc64le and s390x always, and aarch64 on an x86_64 runner, built under QEMU TCG)'
     required: true
   S3_ACCESS_KEY_ID:
     description: 'AWS S3 access key ID for uploading images'
@@ -129,6 +129,23 @@ runs:
           packer_opts="${packer_opts} -var ppc64le_grub_hold=true"
         fi
 
+        # aarch64 on an x86_64 runner (outside the AlmaLinux organisation there
+        # is no arm64 runner): the same sources, switched from KVM to TCG
+        # full-system emulation. -cpu max exposes every feature TCG emulates;
+        # pauth-impdef=on swaps the architected pointer-authentication
+        # algorithm, very slow to emulate, for QEMU's cheap one. The GRUB hold
+        # absorbs the emulated firmware's unpredictable start-up (see
+        # local.aarch64_boot_prefix), console=ttyAMA0 puts anaconda's output
+        # on the captured serial console, and SSH comes up only after the whole
+        # emulated install.
+        if [ "${{ inputs.arch }}" = "aarch64" ] && [ "$(uname -m)" != "aarch64" ]; then
+          packer_opts="${packer_opts} -var aarch64_accelerator=tcg -var aarch64_cpu_model=max,pauth-impdef=on"
+          packer_opts="${packer_opts} -var aarch64_grub_hold=true -var aarch64_extra_kernel_args=console=ttyAMA0"
+          packer_opts="${packer_opts} -var aarch64_console_log=${{ github.workspace }}/aarch64-console.log"
+          packer_opts="${packer_opts} -var boot_wait=3s -var ssh_timeout=5h"
+          echo "AARCH64_TCG=true" >> $GITHUB_ENV
+        fi
+
         # Image type e.g. Azure, Vagrant Libvirt, ...
         image_type=$(echo ${{ inputs.type }} | sed -E 's/_/ /g;s/\b(.)/\u\1/g')
 
@@ -183,11 +200,17 @@ runs:
         # Install KVM
         case ${{ env.runner_os }} in
           ubuntu)
-            # qemu-system-x86, not qemu-kvm: on Ubuntu 26.04 qemu-kvm is a
-            # purely virtual package with two providers (qemu-system-x86 and
-            # qemu-system-x86-hwe) and apt refuses to pick one; on 24.04 it is a
-            # transitional package for qemu-system-x86 anyway
-            sudo apt-get -y install qemu-system-x86
+            # The runner's own system emulator, by name rather than qemu-kvm:
+            # on Ubuntu 26.04 qemu-kvm is a purely virtual package with two
+            # providers (qemu-system-x86 and qemu-system-x86-hwe) and apt
+            # refuses to pick one, and on arm64 it would not be qemu-system-x86
+            case "$(uname -m)" in
+              x86_64)  sudo apt-get -y install qemu-system-x86 ;;
+              aarch64) sudo apt-get -y install qemu-system-arm qemu-efi-aarch64 ;;
+            esac
+            # aarch64 images on an x86_64 runner are emulated (TCG): the arm
+            # system emulator and the AAVMF firmware
+            [ "${{ inputs.arch }}" = "aarch64" ] && sudo apt-get -y install qemu-system-arm qemu-efi-aarch64
             # s390x is emulated (TCG) - needs the s390x system emulator and
             # qemu-img for the blank target disk
             [ "${{ inputs.arch }}" = "s390x" ] && sudo apt-get -y install qemu-system-s390x qemu-utils
@@ -376,6 +399,14 @@ runs:
           sudo tail -n +1 -F "${console_log}" | sed -u 's/^/[ppc64le console] /' &
           console_tail=$!
         fi
+        if [ "${{ env.AARCH64_TCG }}" = "true" ]; then
+          # Same as ppc64le: stream the emulated guest's serial console
+          # (anaconda) into the job log so a stuck install is visible live
+          console_log=${{ github.workspace }}/aarch64-console.log
+          sudo touch "${console_log}"
+          sudo tail -n +1 -F "${console_log}" | sed -u 's/^/[aarch64 console] /' &
+          console_tail=$!
+        fi
         sudo sh -c "/usr/bin/packer build ${{ env.PACKER_OPTS }} -only=${{ env.packer_source }} ."
         [ -n "${console_tail:-}" ] && { sleep 2; kill "${console_tail}" 2>/dev/null || true; }
         true
@@ -388,6 +419,13 @@ runs:
         boot_dir=$(echo 'var.s390x_boot_dir' | sudo /usr/bin/packer console . 2>/dev/null || echo s390x-boot)
         sudo tail -n 200 "${boot_dir}/console.log" 2>/dev/null || echo "[Debug] no console log at ${boot_dir}/console.log"
 
+    - name: Show the aarch64 installer console
+      if: failure() && env.AARCH64_TCG == 'true'
+      shell: bash
+      run: |
+        # Last part of the emulated guest's serial console of the failed build
+        sudo tail -n 200 ${{ github.workspace }}/aarch64-console.log 2>/dev/null || echo "[Debug] no console log"
+
     - name: Show the ppc64le installer console
       if: failure() && inputs.arch == 'ppc64le'
       shell: bash

diff --git a/.github/workflows/gencloud-build-test.yml b/.github/workflows/gencloud-build-test.yml
@@ -14,6 +14,10 @@ run-name: >-
 #
 # Every built image is tested; for a build-only run use gencloud-build.yml.
 #
+# aarch64 outside the AlmaLinux org: no arm64 runner, so the image is built
+# on a GitHub-hosted x86_64 runner under QEMU TCG (the same sources, switched
+# through the aarch64_* variables) and gets the offline validation only.
+#
 # s390x (input s390x, on by default so scheduled runs build it too): there
 # are no s390x runners, so the image is built on an x86_64 runner under
 # QEMU TCG full-system emulation (see the s390x sources in the gencloud
@@ -54,7 +58,7 @@ on:
             - 8
 
         self-hosted:
-          description: "Build aarch64 images on self-hosted runner"
+          description: "Build aarch64 images (RunsOn a1.metal in the AlmaLinux org, QEMU TCG emulation on a GitHub-hosted x86_64 runner elsewhere)"
           required: true
           type: boolean
           default: true
@@ -233,58 +237,32 @@ jobs:
 
   # The job is to start self-hosted runner on AWS EC2 instance if not in the almalinux org
   # It does nothing if in the almalinux org, so 'Setup and start runner' step is skipped
-  start-self-hosted-runner:
-    name: ${{ inputs.version_major }} gencloud-aarch64 runner
-    if: ${{ inputs.self-hosted }}
-    runs-on: ubuntu-24.04
-    needs: [init-data]
-
-    steps:
-    - name: Setup and start runner
-      if: github.repository_owner != 'AlmaLinux'
-      uses: unblocked/ec2-action-builder@v1.12
-      with:
-        github_token: ${{ secrets.GIT_HUB_TOKEN }}
-        aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws_region: ${{ vars.AWS_REGION }}
-        ec2_ami_id: ${{ secrets.EC2_AMI_ID_AL9_AARCH64 }}
-
-        ec2_subnet_id: ${{ secrets.EC2_SUBNET_ID}}                  # Subnet and Security Group should match
-        ec2_security_group_id: ${{ secrets.EC2_SECURITY_GROUP_ID }} # Availability Zones list for 'a1.metal' Instance Type
-        ec2_instance_type: a1.metal
-
-        ec2_root_disk_size_gb: "16"         # override default size which is too small for actions and tests stuff
-        ec2_root_disk_ebs_class: "gp3"      # use faster and cheeper storage instead of default 'gp2'
-        ec2_instance_ttl: 30                # Optional (default is 60 minutes)
-        ec2_spot_instance_strategy: None    # Other options are: SpotOnly, BestEffort, MaxPerformance
-        ec2_instance_tags: >                # Required for IAM role resource permission scoping
-          [
-            {"Key": "Project", "Value": "GitHub Actions Self-hosted Runners"}
-          ]
-
   build-self-hosted:
     name: ${{ inputs.version_major }} ${{ matrix.subtype }}-aarch64 build+test
     permissions:
       id-token: write
       contents: read
     if: ${{ inputs.self-hosted }}
-    needs: [init-data, start-self-hosted-runner]
+    needs: [init-data]
     # AlmaLinux org: RunsOn a1.metal with the Ubuntu arm64 image (NOT
     # almalinux-9-aarch64) so the in-job QEMU test can run - shared-steps
     # builds fine on Ubuntu via its runner_os detection, and the bare-metal
     # a1 has /dev/kvm.
     #
-    # Forks fall back to the self-hosted EC2 a1.metal runner. That AMI is
-    # AlmaLinux 9 (EL9), where the apt-based test composite cannot run - the
-    # in-job test step will fail there; fork CI should test via gencloud-test.yml.
+    # Elsewhere (forks, CI mirrors) there is no arm64 runner to rely on, so
+    # the job runs on a GitHub-hosted x86_64 runner and shared-steps builds
+    # the image under QEMU TCG full-system emulation (it switches whenever
+    # the runner is not arm64). That takes hours rather than minutes, so the
+    # timeout is the hosted runner's maximum, and the in-job boot test is
+    # skipped: gencloud-test-steps needs KVM for the image's architecture.
     runs-on: >-
       ${{
         github.repository_owner == 'AlmaLinux' &&
           format('runs-on={0}/family=a1.metal/image=ubuntu24-full-arm64/volume=40g/spot=false', github.run_id)
         ||
-        github.run_id
+        'ubuntu-24.04'
       }}
+    timeout-minutes: 360
     strategy:
       fail-fast: false
       matrix:
@@ -314,12 +292,14 @@ jobs:
           upload_to_s3: ${{ inputs.upload_to_s3 }}
           notify_mattermost: ${{ inputs.notify_mattermost }}
           run_test: 'false'
-          runner: aws-ec2
+          runner: ${{ github.repository_owner == 'AlmaLinux' && 'aws-ec2' || 'gh_hosted' }}
         env:
           PACKER_GITHUB_API_TOKEN: ${{ secrets.GIT_HUB_TOKEN }}
 
       - uses: ./.github/actions/gencloud-test-steps
         name: Test ${{ inputs.version_major }} ${{ matrix.subtype }}-aarch64 image
+        # needs /dev/kvm for aarch64: only on the arm64 runner (see runs-on)
+        if: ${{ github.repository_owner == 'AlmaLinux' }}
         with:
           image_file: ${{ env.IMAGE_FILE }}
           image_url: ${{ inputs.upload_to_s3 && format('https://{0}.s3-accelerate.dualstack.amazonaws.com/{1}/{2}', vars.AWS_S3_BUCKET, env.AWS_S3_PATH, env.IMAGE_NAME) || '' }}

diff --git a/GENCLOUD_BUILD_TEST.md b/GENCLOUD_BUILD_TEST.md
@@ -64,8 +64,9 @@ There is no `run_test` input: the test always runs.
 ```
 init-data
  |- build-gh-hosted (x86_64 matrix: subtype x variant)   -. shared-steps build,
- |- start-self-hosted-runner (fork EC2)                    | then gencloud-test-steps
- |- build-self-hosted (aarch64 matrix: subtype)          -' in-job on the local qcow2
+ |- build-self-hosted (aarch64 matrix: subtype)          -' then gencloud-test-steps
+                                                             in-job on the local qcow2
+                                                             (forks: TCG, no test)
  |- build-s390x-tcg (s390x, default on)                  -  shared-steps build under
                                                              TCG, offline validation only
  '- build-ppc64le-tcg (default on; ppc64le matrix: subtype) shared-steps build under
@@ -101,20 +102,33 @@ The change is backward-compatible - `gencloud-test.yml` keeps passing
 | Job | Runner (AlmaLinux org) | Runner (forks) |
 | :--- | :--- | :--- |
 | `build-gh-hosted` | `c7i.metal-24xl+c7a.metal-48xl+*8gd.metal*`, `image=ubuntu24-full-x64` | `ubuntu-24.04` (GitHub-hosted, has nested `/dev/kvm`) |
-| `build-self-hosted` | `a1.metal`, `image=ubuntu24-full-arm64`, `volume=40g` | self-hosted EC2 `a1.metal` (`EC2_AMI_ID_AL9_AARCH64`) |
+| `build-self-hosted` | `a1.metal`, `image=ubuntu24-full-arm64`, `volume=40g` | `ubuntu-24.04` (GitHub-hosted x86_64; the image is built under QEMU TCG, see below) |
 | `build-s390x-tcg` | same x86_64 metal family as `build-gh-hosted` (KVM unused - TCG) | `ubuntu-24.04` |
 | `build-ppc64le-tcg` | same x86_64 metal family as `build-gh-hosted`, `image=ubuntu26-full-x64` (QEMU 10.2; KVM unused, TCG is CPU-bound) | `ubuntu-26.04` |
 
 Both org runners are bare metal, so `/dev/kvm` is present for the in-job
 QEMU test. The composite installs `qemu-system-*` + `cloud-image-utils`
 via `apt-get`, which is why the aarch64 leg must be on an Ubuntu image.
 
-**Fork caveat:** the fork aarch64 fallback is the EC2 `a1.metal` runner
-built from `EC2_AMI_ID_AL9_AARCH64` (AlmaLinux 9). The apt-based test
-composite cannot run there, so the in-job aarch64 test step will fail on a
-fork. The AlmaLinux-org path (`ubuntu24-full-arm64`) is the target; fork
-CI should test aarch64 via the standalone `gencloud-test.yml` on
-`ubuntu-24.04-arm`.
+**aarch64 outside the AlmaLinux org.** Forks and CI mirrors have no arm64
+runner, so `build-self-hosted` runs on a GitHub-hosted x86_64 runner and
+shared-steps builds the image under QEMU TCG full-system emulation; it
+switches automatically whenever the runner is not arm64. The sources are the
+same, switched through variables:
+
+| Variable | arm64 host default | TCG value |
+| :--- | :--- | :--- |
+| `aarch64_accelerator` | `kvm` | `tcg` |
+| `aarch64_cpu_model` | `host` | `max,pauth-impdef=on` (every emulated feature; QEMU's cheap pointer-authentication algorithm instead of the architected one, which is very slow to emulate) |
+| `aarch64_grub_hold` | `false` | `true`: 90 s of once-a-second keypresses GRUB ignores from right after the VM starts, so the emulated UEFI firmware's unpredictable start-up does not matter, then a move to the plain "Install" entry (the default entry's `rd.live.check` would hash the whole ISO under emulation) |
+| `aarch64_extra_kernel_args` | empty | `console=ttyAMA0`, so anaconda's output reaches the captured serial console; the aarch64 kickstarts' `%post` removes it again from the installed boot loader configuration, undoes the serial GRUB terminal (`GRUB_TERMINAL`/`GRUB_SERIAL_COMMAND`) anaconda sets in `/etc/default/grub` because of it, and regenerates `grub.cfg`, which anaconda wrote with both before `%post` ran |
+| `aarch64_console_log` | empty | `<workspace>/aarch64-console.log`, streamed into the job log as `[aarch64 console]` lines. The serial port is teed into the file (`-chardev vc,logfile=`), not redirected: on the `virt` machine the firmware and GRUB console is the serial port on QEMU's VNC text console, which is where Packer types the boot command |
+| `boot_wait`, `ssh_timeout` | `30s`, `3600s` | `3s`, `5h` |
+
+The in-job boot test is skipped there (`gencloud-test-steps` needs KVM for
+the image's architecture), and the job timeout is the hosted runner's
+maximum of 6 hours. The AlmaLinux org keeps the RunsOn `a1.metal` runner
+with KVM and the in-job test, unchanged.
 
 ## s390x under TCG (experimental)
 
@@ -214,7 +228,6 @@ Kitten.
 | `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | S3 upload (build stage) |
 | `MATTERMOST_WEBHOOK_URL` | Mattermost incoming webhook URL |
 | `GIT_HUB_TOKEN` | Packer plugin GitHub API token |
-| `EC2_AMI_ID_AL9_AARCH64`, `EC2_SUBNET_ID`, `EC2_SECURITY_GROUP_ID` | fork-only aarch64 EC2 runner |
 
 ### Variables (`vars.*`)
 | Variable | Description |

diff --git a/almalinux-8-gencloud-ext4.pkr.hcl b/almalinux-8-gencloud-ext4.pkr.hcl
@@ -40,9 +40,9 @@ source "qemu" "almalinux-8-gencloud-ext4-aarch64" {
   ssh_username       = var.gencloud_ssh_username
   ssh_password       = var.gencloud_ssh_password
   ssh_timeout        = var.ssh_timeout
-  boot_command       = local.gencloud_ext4_boot_command_8_aarch64
+  boot_command       = local.aarch64_gencloud_ext4_boot_command_8
   boot_wait          = var.boot_wait
-  accelerator        = "kvm"
+  accelerator        = var.aarch64_accelerator
   firmware           = var.aavmf_code
   use_pflash         = false
   disk_interface     = "virtio-scsi"
@@ -58,12 +58,17 @@ source "qemu" "almalinux-8-gencloud-ext4-aarch64" {
   net_device         = "virtio-net"
   qemu_binary        = var.qemu_binary
   vm_name            = "AlmaLinux-8-GenericCloud-ext4-${var.os_ver_8}-${formatdate("YYYYMMDD", timestamp())}.aarch64.qcow2"
-  cpu_model          = "host"
+  cpu_model          = var.aarch64_cpu_model
   cpus               = var.cpus
-  qemuargs = [
-    ["-boot", "strict=on"],
-    ["-monitor", "none"],
-  ]
+  qemuargs = concat(
+    [["-boot", "strict=on"], ["-monitor", "none"]],
+    # Tee the serial port into a log file instead of redirecting it: on the
+    # virt machine the firmware and GRUB console IS the serial port, shown
+    # on QEMU's VNC text console, and Packer types the boot command into it.
+    # "-serial file:" would take that input away (GRUB would never see the
+    # keys and boot its default media-check entry).
+    var.aarch64_console_log != "" ? [["-chardev", "vc,id=serial0,logfile=${var.aarch64_console_log}"], ["-serial", "chardev:serial0"]] : [],
+  )
 }
 
 source "qemu" "almalinux-8-gencloud-ext4-ppc64le" {

diff --git a/almalinux-8-gencloud.pkr.hcl b/almalinux-8-gencloud.pkr.hcl
@@ -40,9 +40,9 @@ source "qemu" "almalinux-8-gencloud-aarch64" {
   ssh_username       = var.gencloud_ssh_username
   ssh_password       = var.gencloud_ssh_password
   ssh_timeout        = var.ssh_timeout
-  boot_command       = local.gencloud_boot_command_8_aarch64
+  boot_command       = local.aarch64_gencloud_boot_command_8
   boot_wait          = var.boot_wait
-  accelerator        = "kvm"
+  accelerator        = var.aarch64_accelerator
   firmware           = var.aavmf_code
   use_pflash         = false
   disk_interface     = "virtio-scsi"
@@ -58,12 +58,17 @@ source "qemu" "almalinux-8-gencloud-aarch64" {
   net_device         = "virtio-net"
   qemu_binary        = var.qemu_binary
   vm_name            = "AlmaLinux-8-GenericCloud-${var.os_ver_8}-${formatdate("YYYYMMDD", timestamp())}.aarch64.qcow2"
-  cpu_model          = "host"
+  cpu_model          = var.aarch64_cpu_model
   cpus               = var.cpus
-  qemuargs = [
-    ["-boot", "strict=on"],
-    ["-monitor", "none"],
-  ]
+  qemuargs = concat(
+    [["-boot", "strict=on"], ["-monitor", "none"]],
+    # Tee the serial port into a log file instead of redirecting it: on the
+    # virt machine the firmware and GRUB console IS the serial port, shown
+    # on QEMU's VNC text console, and Packer types the boot command into it.
+    # "-serial file:" would take that input away (GRUB would never see the
+    # keys and boot its default media-check entry).
+    var.aarch64_console_log != "" ? [["-chardev", "vc,id=serial0,logfile=${var.aarch64_console_log}"], ["-serial", "chardev:serial0"]] : [],
+  )
 }
 
 source "qemu" "almalinux-8-gencloud-ppc64le" {

diff --git a/almalinux-9-gencloud-ext4.pkr.hcl b/almalinux-9-gencloud-ext4.pkr.hcl
@@ -40,9 +40,9 @@ source "qemu" "almalinux-9-gencloud-ext4-aarch64" {
   ssh_username       = var.gencloud_ssh_username
   ssh_password       = var.gencloud_ssh_password
   ssh_timeout        = var.ssh_timeout
-  boot_command       = var.gencloud_ext4_boot_command_9_aarch64
+  boot_command       = local.aarch64_gencloud_ext4_boot_command_9
   boot_wait          = var.boot_wait
-  accelerator        = "kvm"
+  accelerator        = var.aarch64_accelerator
   firmware           = var.aavmf_code
   use_pflash         = false
   disk_interface     = "virtio-scsi"
@@ -58,12 +58,17 @@ source "qemu" "almalinux-9-gencloud-ext4-aarch64" {
   net_device         = "virtio-net"
   qemu_binary        = var.qemu_binary
   vm_name            = "AlmaLinux-9-GenericCloud-ext4-${var.os_ver_9}-${formatdate("YYYYMMDD", timestamp())}.aarch64.qcow2"
-  cpu_model          = "host"
+  cpu_model          = var.aarch64_cpu_model
   cpus               = var.cpus
-  qemuargs = [
-    ["-boot", "strict=on"],
-    ["-monitor", "none"],
-  ]
+  qemuargs = concat(
+    [["-boot", "strict=on"], ["-monitor", "none"]],
+    # Tee the serial port into a log file instead of redirecting it: on the
+    # virt machine the firmware and GRUB console IS the serial port, shown
+    # on QEMU's VNC text console, and Packer types the boot command into it.
+    # "-serial file:" would take that input away (GRUB would never see the
+    # keys and boot its default media-check entry).
+    var.aarch64_console_log != "" ? [["-chardev", "vc,id=serial0,logfile=${var.aarch64_console_log}"], ["-serial", "chardev:serial0"]] : [],
+  )
 }
 
 source "qemu" "almalinux-9-gencloud-ext4-ppc64le" {