x86 Assembly Series Part 13: System Calls & Interrupts

Linux x86-64 System Calls

Syscall ABI

Common Syscalls

section .data
    msg db "Hello, World!", 10
    len equ $ - msg

section .text
global _start
_start:
    ; sys_write(1, msg, len)
    mov rax, 1          ; syscall: write
    mov rdi, 1          ; fd: stdout
    mov rsi, msg        ; buffer
    mov rdx, len        ; count
    syscall

    ; sys_exit(0)
    mov rax, 60         ; syscall: exit
    xor rdi, rdi        ; status: 0
    syscall

nasm -f elf64 syscall_hello.asm -o syscall_hello.o
ld syscall_hello.o -o syscall_hello
./syscall_hello

nasm -f macho64 syscall_hello.asm -o syscall_hello.o
ld -macos_version_min 10.13 -e _main -static syscall_hello.o -o syscall_hello

nasm -f win64 syscall_hello.asm -o syscall_hello.obj
link /subsystem:console /entry:_start syscall_hello.obj /out:syscall_hello.exe

Linux x86-32 (int 0x80)

; 32-bit Linux: int 0x80
; EAX=syscall, EBX=arg1, ECX=arg2, EDX=arg3
mov eax, 4          ; sys_write
mov ebx, 1          ; stdout
mov ecx, msg        ; buffer
mov edx, len        ; length
int 0x80            ; invoke kernel

mov eax, 1          ; sys_exit
xor ebx, ebx        ; status 0
int 0x80

Interrupts Overview

Hardware Interrupts (IRQs)

Hardware interrupts (Interrupt Requests) are signals from devices like keyboards, timers, and disk controllers:

; Enable/disable hardware interrupts
sti                     ; Set Interrupt Flag - enable interrupts
cli                     ; Clear Interrupt Flag - disable interrupts

; Acknowledge interrupt to PIC (8259)
mov al, 0x20           ; End of Interrupt command
out 0x20, al           ; Send to master PIC
; If IRQ >= 8:
out 0xA0, al           ; Also send to slave PIC

Software Interrupts

The INT n instruction triggers interrupt vector n explicitly:

; Common software interrupts
int 0x80                ; Linux system call (32-bit)
int 0x10                ; BIOS video services (real mode)
int 0x13                ; BIOS disk services (real mode)
int 0x16                ; BIOS keyboard services (real mode)
int 0x21                ; DOS function call
int 0x03                ; Debugger breakpoint

; BIOS example (real mode): Print character
mov ah, 0x0E            ; Teletype output function
mov al, 'A'             ; Character to print
int 0x10                ; Call BIOS video interrupt

Interrupt Descriptor Table (IDT)

The IDT maps interrupt vectors (0-255) to handler addresses. Each entry is 8 bytes (32-bit) or 16 bytes (64-bit):

IDT Entry (64-bit Long Mode):
┌───────────────┬───────────────┬───────────────┬───────────────┐
│ Offset 63:32  │  Reserved     │               │               │
├───────────────┼───────────────┼───────────────┼───────────────┤
│ Offset 31:16  │  Attr/Type    │  Selector     │ Offset 15:0   │
└───────────────┴───────────────┴───────────────┴───────────────┘

Type (4 bits): 0xE = Interrupt Gate, 0xF = Trap Gate
DPL (2 bits): Privilege level required to call via INT instruction

; Load IDT pointer
idt_descriptor:
    dw idt_end - idt_start - 1    ; Limit (size - 1)
    dq idt_start                   ; Base address

; Load IDT register
lidt [idt_descriptor]

; IDT entries (simplified 64-bit)
idt_start:
    ; Entry 0: Division Error (#DE)
    dw div_error_handler & 0xFFFF       ; Offset low
    dw 0x08                              ; Code segment selector
    db 0                                 ; IST (0 = none)
    db 0x8E                              ; Present, DPL=0, Interrupt Gate
    dw (div_error_handler >> 16) & 0xFFFF ; Offset mid
    dd div_error_handler >> 32           ; Offset high
    dd 0                                 ; Reserved
idt_end:

CPU Exceptions

CPU exceptions are synchronous interrupts triggered by instruction execution errors:

; Page Fault error code bits:
; Bit 0 (P)  : 0=non-present page, 1=protection violation
; Bit 1 (W/R): 0=read access, 1=write access
; Bit 2 (U/S): 0=supervisor mode, 1=user mode
; Bit 3 (RSVD): 1=reserved bit set in page entry
; Bit 4 (I/D) : 1=instruction fetch

Writing Interrupt Handlers

Interrupt handlers must save state, handle the interrupt, and return with iret:

; Interrupt handler template (64-bit)
; For exceptions WITH error code (e.g., #GP, #PF)
gp_fault_handler:
    ; CPU pushed: SS, RSP, RFLAGS, CS, RIP, Error Code
    push rax                ; Save registers we'll use
    push rbx
    push rcx
    push rdx
    push rsi
    push rdi
    push rbp
    push r8
    push r9
    push r10
    push r11
    
    ; Get error code (at RSP + 88 after our pushes)
    mov rdi, [rsp + 88]     ; Error code as first parameter
    
    ; Call C handler (if available)
    ; extern gp_fault_c_handler
    ; call gp_fault_c_handler
    
    ; Display error and halt (simple handler)
    ; ... panic code here ...
    
    ; Restore registers
    pop r11
    pop r10
    pop r9
    pop r8
    pop rbp
    pop rdi
    pop rsi
    pop rdx
    pop rcx
    pop rbx
    pop rax
    
    add rsp, 8              ; Remove error code from stack
    iretq                   ; Return from interrupt (64-bit)

; Divide by zero handler (no error code)
div_error_handler:
    ; Save all registers
    push rax
    ; ... save more as needed ...
    
    ; Handle the error (print message, terminate program, etc.)
    
    ; ... restore registers ...
    pop rax
    iretq

; IRQ1 Keyboard interrupt handler
keyboard_handler:
    push rax
    
    in al, 0x60             ; Read scan code from keyboard
    ; Store scancode somewhere for later processing
    mov [last_scancode], al
    
    ; Send EOI to PIC
    mov al, 0x20
    out 0x20, al
    
    pop rax
    iretq