ARM Assembly Part 20: Building a Bare-Metal OS Kernel

Introduction & Kernel Architecture

Bootloader Stub (boot.S)

// boot.S — first instruction executed by QEMU -kernel
// QEMU loads ELF at 0x40000000, sets x0=DTB address, jumps to _start

.section .text.boot
.global _start

_start:
    // Step 1: Only core 0 continues; park other SMP cores
    mrs  x1, mpidr_el1
    and  x1, x1, #0xFF           // MPIDR.Aff0 = core number
    cbnz x1, .park               // cores 1+ spin here

    // Step 2: Set up EL1 stack (grows down from _stack_top)
    adrp x2, _stack_top
    add  x2, x2, :lo12:_stack_top
    mov  sp, x2

    // Step 3: Zero .bss
    adrp x3, _bss_start
    add  x3, x3, :lo12:_bss_start
    adrp x4, _bss_end
    add  x4, x4, :lo12:_bss_end
.bss_zero:
    cmp  x3, x4
    b.ge .bss_done
    str  xzr, [x3], #8
    b    .bss_zero
.bss_done:

    // Step 4: Disable trapping of FP/SIMD registers at EL1
    mov  x5, #(3 << 20)          // CPACR_EL1.FPEN = 11 (no trap)
    msr  cpacr_el1, x5
    isb

    // Step 5: Jump to C kernel_main (x0 still = DTB pointer from QEMU)
    bl   kernel_main

    // If kernel_main returns, halt
.halt:
    wfi
    b    .halt

.park:
    wfi
    b    .park

UART Driver in Assembly

// uart.S — PL011 UART0 driver for QEMU virt machine
// UART0 base: 0x09000000
// UARTDR   at base + 0x000  (data register: write = TX, read = RX)
// UARTFR   at base + 0x018  (flag register: bit 5 = TXFF transmit full)
// UARTIBRD at base + 0x024  (integer baud rate divisor)
// UARTLCR_H at base + 0x02C (line control)
// UARTCR   at base + 0x030  (control: TXE=bit8, RXE=bit9, UARTEN=bit0)

.equ UART0_BASE,   0x09000000
.equ UARTDR,       0x000
.equ UARTFR,       0x018
.equ UARTIBRD,     0x024
.equ UARTLCR_H,    0x02C
.equ UARTCR,       0x030
.equ UARTFR_TXFF,  (1 << 5)   // Transmit FIFO full flag

.global uart_init
.global uart_putc
.global uart_puts

// uart_init() — configure 115200 baud, 8N1, FIFOs enabled
uart_init:
    mov  x0, #UART0_BASE

    // Disable UART
    str  wzr, [x0, #UARTCR]

    // Set baud: IBRD = 26 → 24 MHz / (16 × 57600) = 26.04 → 115200 baud approx
    // For QEMU 24 MHz UART clock: IBRD=13, FBRD=1 → 115200
    mov  w1, #13
    str  w1, [x0, #UARTIBRD]

    // 8N1, FIFOs enabled: WLEN=11 (8-bit), FEN=1, STP2=0, PEN=0
    mov  w1, #((0b11 << 5) | (1 << 4))   // WLEN=3, FEN=1
    str  w1, [x0, #UARTLCR_H]

    // Enable UART: UARTEN|TXE|RXE
    mov  w1, #((1 << 9) | (1 << 8) | 1)
    str  w1, [x0, #UARTCR]
    ret

// uart_putc(char c) — x0 = character to send
uart_putc:
    mov  x1, #UART0_BASE
.wait_tx:
    ldr  w2, [x1, #UARTFR]
    tst  w2, #UARTFR_TXFF
    b.ne .wait_tx              // Spin while TX FIFO full
    str  w0, [x1, #UARTDR]    // Write character to data register
    ret

// uart_puts(const char *s) — x0 = null-terminated string
uart_puts:
    stp  x29, x30, [sp, #-16]!
    mov  x29, sp
    mov  x2, x0               // Save string pointer
.puts_loop:
    ldrb w0, [x2], #1         // Load byte, advance pointer
    cbz  w0, .puts_done       // Null terminator? exit
    bl   uart_putc
    b    .puts_loop
.puts_done:
    ldp  x29, x30, [sp], #16
    ret

Exception Vector Table

// vectors.S — AArch64 exception vector table (must be 2KB aligned)
// Layout: 4 groups × 4 entries × 128 bytes each = 2048 bytes

.section .text.vectors
.balign 2048
.global exception_vectors

exception_vectors:
    // ── EL1 with SP_EL0 (from current EL, using SP0) ──
    .balign 128
    b sync_sp0_handler       // Synchronous
    .balign 128
    b irq_sp0_handler        // IRQ
    .balign 128
    b fiq_sp0_handler        // FIQ
    .balign 128
    b serror_sp0_handler     // SError

    // ── EL1 with SP_EL1 (from current EL, using SPx) ──
    .balign 128
    b sync_spx_handler       // Synchronous (most common: fault/SVC)
    .balign 128
    b irq_spx_handler        // IRQ
    .balign 128
    b fiq_spx_handler        // FIQ
    .balign 128
    b serror_spx_handler     // SError

    // ── Lower EL AArch64 (from EL0, 64-bit) ──
    .balign 128
    b sync_el0_64_handler    // SVC, data abort from user
    .balign 128
    b irq_el0_64_handler
    .balign 128
    b fiq_el0_64_handler
    .balign 128
    b serror_el0_64_handler

    // ── Lower EL AArch32 (from EL0, 32-bit) ──
    .balign 128; b unhandled   // Sync
    .balign 128; b unhandled   // IRQ
    .balign 128; b unhandled   // FIQ
    .balign 128; b unhandled   // SError

// Minimal synchronous handler: print ESR + ELR then halt
sync_spx_handler:
    mrs  x0, esr_el1
    mrs  x1, elr_el1
    mrs  x2, far_el1
    bl   exception_report     // C function: void exception_report(u64 esr, u64 elr, u64 far)
    b    .                    // Infinite loop

// Install vector table at EL1
.global install_vectors
install_vectors:
    adrp x0, exception_vectors
    add  x0, x0, :lo12:exception_vectors
    msr  vbar_el1, x0
    isb
    ret

Bump Allocator & Memory Map

// mm.c — bump allocator, no free()
#include <stdint.h>
#include <stddef.h>

extern char _heap_start[];    // Symbol from linker script
static char *bump_ptr;

void mm_init(void) {
    bump_ptr = _heap_start;
}

// Align bump_ptr up to `align` (must be power of 2)
void *mm_alloc(size_t size, size_t align) {
    uintptr_t addr = (uintptr_t)bump_ptr;
    addr = (addr + align - 1) & ~(align - 1);  // Align up
    bump_ptr = (char *)(addr + size);
    // Zero-fill the allocation
    char *p = (char *)addr;
    for (size_t i = 0; i < size; i++) p[i] = 0;
    return (void *)addr;
}

Cooperative Scheduler & Context Switch

// context.S — save/restore CPU context for cooperative scheduling
// Task control block (TCB) layout (C struct layout, 64-bit):
//   offset 0:   x19
//   offset 8:   x20  ... (callee-saved registers x19–x28)
//   offset 80:  x29 (frame pointer)
//   offset 88:  x30 (LR — resume address)
//   offset 96:  sp_el1 (kernel stack pointer)
// Total TCB size: 104 bytes

.global context_switch    // void context_switch(struct tcb *from, struct tcb *to)

context_switch:
    // Save current task's callee-saved registers to 'from' TCB
    stp  x19, x20, [x0, #0]
    stp  x21, x22, [x0, #16]
    stp  x23, x24, [x0, #32]
    stp  x25, x26, [x0, #48]
    stp  x27, x28, [x0, #64]
    stp  x29, x30, [x0, #80]
    mov  x2, sp
    str  x2,       [x0, #96]

    // Restore next task's callee-saved registers from 'to' TCB
    ldp  x19, x20, [x1, #0]
    ldp  x21, x22, [x1, #16]
    ldp  x23, x24, [x1, #32]
    ldp  x25, x26, [x1, #48]
    ldp  x27, x28, [x1, #64]
    ldp  x29, x30, [x1, #80]
    ldr  x2,       [x1, #96]
    mov  sp, x2

    ret                    // Returns to saved LR = task's resume address

// sched.c — cooperative round-robin scheduler
#include <stdint.h>
#include <stddef.h>

#define MAX_TASKS 8
#define STACK_SIZE 4096

struct tcb {
    uint64_t x19, x20, x21, x22, x23, x24;
    uint64_t x25, x26, x27, x28;
    uint64_t x29, x30;       // frame ptr, link register
    uint64_t sp_el1;          // kernel stack pointer
};

extern void context_switch(struct tcb *from, struct tcb *to);
extern void *mm_alloc(size_t size, size_t align);

static struct tcb tcb_table[MAX_TASKS];
static int task_count = 0;
static int current_task = 0;

// Create a new task: allocate stack, set LR = entry, set SP top of stack
void task_create(void (*entry)(void)) {
    int i = task_count++;
    char *stack = (char *)mm_alloc(STACK_SIZE, 16);
    uint64_t stack_top = (uint64_t)(stack + STACK_SIZE);
    // Align stack to 16 bytes per ABI
    stack_top &= ~15ULL;
    tcb_table[i].sp_el1 = stack_top;
    tcb_table[i].x30    = (uint64_t)entry;  // LR = first resume address
    tcb_table[i].x29    = stack_top;        // FP = stack top initially
}

// yield() — save current task, switch to next (cooperative switch)
void yield(void) {
    int from = current_task;
    int to   = (from + 1) % task_count;
    current_task = to;
    context_switch(&tcb_table[from], &tcb_table[to]);
}

// Start the scheduler (runs task 0, sets up first context)
void sched_start(void) {
    // No 'from' for the very first switch; use a dummy TCB
    static struct tcb idle_tcb;
    current_task = 0;
    context_switch(&idle_tcb, &tcb_table[0]);
}

Build & QEMU Run

# Cross-compile for aarch64 bare-metal
CROSS=aarch64-linux-gnu-
${CROSS}gcc -nostdlib -nostartfiles -ffreestanding \
    -march=armv8-a -O2 \
    -T linker.ld \
    boot.S uart.S vectors.S context.S \
    kernel.c mm.c sched.c \
    -o kernel.elf

# Extract raw binary (QEMU -kernel accepts ELF directly)
${CROSS}objcopy -O binary kernel.elf kernel.bin

# Run in QEMU virt machine (ARM Cortex-A57)
qemu-system-aarch64 \
    -machine virt,gic-version=2 \
    -cpu cortex-a57 \
    -m 128M \
    -kernel kernel.elf \
    -serial stdio \
    -display none

# Debug with GDB over QEMU GDB stub
qemu-system-aarch64 \
    -machine virt,gic-version=2 -cpu cortex-a57 -m 128M \
    -kernel kernel.elf -serial stdio -display none \
    -s -S &   # -s = GDB port 1234, -S = pause at boot

aarch64-linux-gnu-gdb kernel.elf \
    -ex "target remote :1234" \
    -ex "b _start" \
    -ex "continue"

Case Study: How Real Kernels Started

Hands-On Exercises

Conclusion & Next Steps

We built a complete, runnable ARM64 bare-metal kernel: QEMU boot stub with SMP parking, PL011 UART driver, 2KB-aligned AArch64 exception vector table, bump memory allocator, and a cooperative context-switch scheduler. Total assembly: ~150 lines. Total C: ~80 lines. Every line maps directly to concepts in Parts 1–20. The case studies show how this same pattern scales from our 230-line project to Linux's 6-billion-device reach, and the exercises guide you toward preemptive scheduling and MMU-enabled operation.