ARM Assembly Part 23: Debugging & Tooling Ecosystem

Debugging Architecture Overview

GDB Remote Protocol

# ── Connect GDB to a QEMU GDB stub (from Part 20 bare-metal kernel) ──
qemu-system-aarch64 \
    -machine virt -cpu cortex-a57 -m 128M \
    -kernel kernel.elf -serial stdio -display none \
    -s -S   # -s: GDB on port 1234, -S: halt at boot

# In a second terminal:
aarch64-linux-gnu-gdb kernel.elf
(gdb) target remote :1234
(gdb) b _start
(gdb) continue
(gdb) info registers           # Show all GP registers
(gdb) x/4i $pc                 # Disassemble 4 instructions at PC
(gdb) x/32xg $sp               # Dump 32 quadwords from stack pointer
(gdb) p/x *(uint64_t*)0x40000000   # Read physical memory
(gdb) set $x0 = 0x1234         # Modify register
(gdb) watch *(uint64_t*)0x40100000 # Hardware watchpoint on memory address

# ── GDB useful AArch64-specific commands ──
# Inspect system registers (aarch64 extension):
(gdb) monitor info registers   # QEMU: show all QEMU-visible regs

# Set conditional breakpoint (break when X0 == 5):
(gdb) b some_function if $x0 == 5

# Tui mode for source+asm side by side:
(gdb) tui enable
(gdb) layout split

# Save/load breakpoints:
(gdb) save breakpoints bp.txt
(gdb) source bp.txt

OpenOCD & JTAG/SWD

# ── Install OpenOCD and connect to a Raspberry Pi 4 via J-Link ──
# (Raspberry Pi 4 exposes JTAG on GPIO 22-27 with appropriate config)

# openocd.cfg for RPi4 via J-Link
cat > rpi4.cfg <<'EOF'
# Probe: SEGGER J-Link (SWD mode)
source [find interface/jlink.cfg]
transport select swd

# Target: ARM Cortex-A72 (AArch64)
source [find target/bcm2711.cfg]

# SWD clk: start conservative
adapter speed 1000
EOF

openocd -f rpi4.cfg &

# Then attach GDB:
aarch64-linux-gnu-gdb vmlinux
(gdb) target extended-remote :3333
(gdb) monitor reset halt
(gdb) b start_kernel
(gdb) continue

# ── OpenOCD flash programming (Cortex-M embedded, Part 14 context) ──
# Example: flash bare-metal firmware to STM32 via ST-Link
openocd \
    -f interface/stlink.cfg \
    -f target/stm32f4x.cfg \
    -c "program firmware.elf verify reset exit"

# Memory read/write from OpenOCD telnet interface:
telnet localhost 4444
> halt
> mdw 0x40000000 8   # Read 8 words from 0x40000000
> mww 0x40000000 0xDEADBEEF   # Write word

ETM Instruction Trace

// Enable ETMv4 via system registers (AArch64, EL2 access required)
// CoreSight registers are memory-mapped in the debug ROM region
// Cortex-A78 ETM base address: read from ROM table at 0xE00FF000

// In assembly (EL2 or debug monitor):
.equ ETM_BASE,     0xE0041000   // ETM for CPU0 (SoC-specific)
.equ ETM_TRCPRGCTLR, 0x004     // Programming Control Register
.equ ETM_TRCCONFIGR, 0x010     // Configuration Register
.equ ETM_TRCVICTLR, 0x080      // ViewInst Control Register (which PCs to trace)
.equ ETM_TRCOSLAR,  0x300      // OS Lock Access Register

// Step 1: Unlock OS Lock (prevents ETM access if locked)
mov  x0, #ETM_BASE
str  wzr, [x0, #ETM_TRCOSLAR]  // Write 0 to unlock

// Step 2: Disable ETM for programming
ldr  w1, [x0, #ETM_TRCPRGCTLR]
bic  w1, w1, #1                 // Clear EN bit
str  w1, [x0, #ETM_TRCPRGCTLR]
dsb  sy
isb

// Step 3: Configure to trace all instructions, timestamps on
mov  w2, #(1 << 6)             // TRCONFIGR: timestamps enabled
str  w2, [x0, #ETM_TRCCONFIGR]

// Step 4: ViewInst — include all EL1 instructions (no filter)
mov  w3, #0x201                 // TRCVICTLR: EL1NS, no filter
str  w3, [x0, #ETM_TRCVICTLR]

// Step 5: Re-enable ETM
ldr  w1, [x0, #ETM_TRCPRGCTLR]
orr  w1, w1, #1
str  w1, [x0, #ETM_TRCPRGCTLR]
isb

# Decode ETM trace in Linux using Coresight subsystem
# (kernel must be built with CONFIG_CORESIGHT=y, CONFIG_CORESIGHT_ETM4X=y)

# Enable ETM tracing on CPU0:
echo 1 > /sys/bus/coresight/devices/etm0/enable_source

# Run workload:
./my_workload &
PID=$!
sleep 1
kill -STOP $PID
echo 0 > /sys/bus/coresight/devices/etm0/enable_source

# Read ETB (Embedded Trace Buffer) and decode:
cat /dev/cs_etb0 > trace.raw
# Use 'perf' with cs-etm event:
perf record -e cs_etm//@etm0/ --per-thread ./my_workload
perf report --stdio --call-graph flat

ITM Stimulus Ports

// ITM (Instrumentation Trace Macrocell) — Cortex-M / A debug printf
// ITM_STIM0 at 0xE0000000 — write a byte/word and it appears in trace
// No clock cycles wasted on flush — non-blocking when demultiplexed via SWO pin

.equ ITM_STIM0,    0xE0000000   // Stimulus port 0
.equ ITM_TER,      0xE0000E00   // Trace Enable Register (enable port 0)
.equ ITM_TCR,      0xE0000E80   // Trace Control Register
.equ ITM_LAR,      0xE0000FB0   // Lock Access Register

// itm_init: unlock and enable ITM, stimulus port 0
itm_init:
    mov  x0, #ITM_LAR
    movk x0, #0xE000, lsl #16
    mov  w1, #0xC5ACCE55        // ITM unlock key
    str  w1, [x0]               // Unlock ITM

    mov  x2, #ITM_TCR
    movk x2, #0xE000, lsl #16
    mov  w3, #0x00010005        // ITMENA=1, TraceBusID=1
    str  w3, [x2]

    mov  x4, #ITM_TER
    movk x4, #0xE000, lsl #16
    mov  w5, #1                  // Enable port 0 only
    str  w5, [x4]
    ret

// itm_putc(char c) — x0 = character, non-blocking write to stimulus port 0
itm_putc:
    mov  x1, #ITM_STIM0
    movk x1, #0xE000, lsl #16
.itm_spin:
    ldr  w2, [x1]               // FIFOREADY bit in STIM read
    tst  w2, #1
    b.eq .itm_spin              // Wait until FIFO has space
    strb w0, [x1]               // Write 1 byte to port 0
    ret

QEMU as Debug Target

# ── Debug AArch64 Linux kernel with QEMU + GDB ──
# Run QEMU with GDB stub enabled and halted at boot:
qemu-system-aarch64 \
    -machine virt -cpu cortex-a72 -m 2G \
    -kernel Image \
    -initrd initramfs.cpio.gz \
    -append "nokaslr console=ttyAMA0 debug" \
    -serial stdio -display none \
    -s -S

# Note: nokaslr disables KASLR (kernel address randomisation) — required for GDB symbols to work

# Attach GDB with kernel vmlinux symbols:
aarch64-linux-gnu-gdb vmlinux
(gdb) target remote :1234
(gdb) add-symbol-file drivers/net/virtio_net.ko 0xffff000001234000  # Load module symbols
(gdb) b net_rx_action     # Kernel breakpoint
(gdb) continue

# Trace individual syscalls using GDB catch:
(gdb) catch syscall read   # Break on every read() syscall
(gdb) commands             # Run commands on each catch
    silent
    printf "read at pc=%#lx\n", $pc
    continue
end

# ── QEMU register inspection while halted ──
# In GDB: QEMU maps ARM64 system registers via monitor interface
(gdb) monitor info registers   # All registers including PSTATE, ELR, SPSR

# In QEMU monitor (Ctrl+A C to switch from serial to monitor):
(qemu) x /10i 0xffff800010000000   # Disassemble kernel .text
(qemu) xp /4x 0x40000000           # Physical memory dump
(qemu) info mtree                   # Show full ARM memory map
(qemu) info cpus                    # All vCPUs and their states

Kernel Debugging Workflows

# ── 1. KASAN (Kernel Address Sanitizer) for ARM64 memory bugs ──
# Build kernel with:
# CONFIG_KASAN=y, CONFIG_KASAN_GENERIC=y
# All heap out-of-bounds and use-after-free bugs print:
# BUG: KASAN: slab-out-of-bounds in function+offset/size

# ── 2. KGDB — built-in kernel GDB stub over serial ──
# Boot: append "kgdboc=ttyAMA0,115200 kgdbwait"
# Connect: aarch64-linux-gnu-gdb vmlinux -ex "target remote /dev/ttyUSB0"

# ── 3. Ftrace — function tracer with zero config in sysfs ──
echo function > /sys/kernel/debug/tracing/current_tracer
echo schedule > /sys/kernel/debug/tracing/set_ftrace_filter
echo 1 > /sys/kernel/debug/tracing/tracing_on
sleep 1
echo 0 > /sys/kernel/debug/tracing/tracing_on
cat /sys/kernel/debug/tracing/trace | head -50

# ── 4. Crash dump analysis with crash tool ──
# Configure kernel: CONFIG_KEXEC=y, CONFIG_CRASH_DUMP=y
# kdump via kexec: on panic, boot capture kernel:
kdump: saved vmcore to /var/crash/vmcore
crash vmlinux /var/crash/vmcore
crash> bt         # Backtrace of panicking task
crash> log        # Kernel log buffer at crash time
crash> ps         # All tasks at crash time

Case Study: Debugging a Real-World ARM Boot Failure

Hands-On Exercises

Conclusion & Next Steps

The ARM debugging stack is deep but consistent: every layer from JTAG hardware to kernel tracing exposes the same fundamental abstractions — halt, single-step, memory read/write, and execution trace. Mastering these tools transforms opaque crashes into navigable execution histories. The boot failure case study shows how layered debugging tools complement each other, and the exercises build practical muscle memory with GDB, Ftrace, and custom debug scripting that you'll use throughout your ARM development career.