Inline Hooking

Patching function prologues with JMP redirects — the 5-byte relative JMP, the trampoline pattern, x64 absolute JMP, the hotpatch variant, and byte-comparison detection

5-Byte Relative JMP Hook

The simplest inline hook patches the first 5 bytes of a function with a near JMP: opcode 0xE9 followed by a 4-byte signed offset from the next instruction to the hook function. On x86 (32-bit), this reaches anywhere in a 2 GB range. On x64, the same 5-byte JMP is limited to ±2 GB relative to the target — so the hook function must be within that range of the hooked function.

// 5-byte relative JMP hook installation
typedef struct {
    PVOID  pOriginal;     // address of hooked function
    PVOID  pHookFn;       // address of our hook
    BYTE   stolen[16];    // saved original bytes
    PVOID  pTrampoline;   // trampoline for calling original
} HookEntry;

void InstallHook5(HookEntry *h)
{
    // Save original bytes (copy enough for the JMP + any displaced instructions)
    memcpy(h->stolen, h->pOriginal, 5);

    // Calculate relative offset: JMP target = hook - (original + 5)
    INT32 rel = (INT32)((BYTE*)h->pHookFn - ((BYTE*)h->pOriginal + 5));

    // Build the 5-byte JMP patch
    BYTE patch[5] = { 0xE9, 0, 0, 0, 0 };
    *(INT32*)(patch + 1) = rel;

    // VirtualProtect, write, restore protection
    DWORD oldProt;
    VirtualProtect(h->pOriginal, 5, PAGE_EXECUTE_READWRITE, &oldProt);
    memcpy(h->pOriginal, patch, 5);
    VirtualProtect(h->pOriginal, 5, oldProt, &oldProt);
}

The Trampoline Pattern

When the hook fires and the hook function needs to call the original function (to avoid breaking the caller), it can't simply call pOriginal — that would loop back through the hook. Instead, a trampoline is built:

Original function (before hook):
  +0: 4C 8B D1    mov r10, rcx       ─┐ stolen bytes
  +3: B8 18 00    mov eax, 0x18      ─┘ (first 5 bytes)
  +5: 00 00       (rest of stub)
  +7: 0F 05       syscall
  +9: C3          ret

Trampoline (allocated in RWX memory):
  +0: 4C 8B D1    mov r10, rcx       ─┐ copied stolen bytes
  +3: B8 18 00    mov eax, 0x18      ─┘
  +5: E9 XX XX    JMP  (original+5)  ── jump back to the rest of original

Patched original function:
  +0: E9 XX XX XX XX  JMP hook_fn   ── jumps to our hook function

Hook function flow:
  hook_fn(args) {
      // analyze / log / block
      result = trampoline(args);   // calls real function via trampoline
      return result;               // return original result to caller
  }

x64 Absolute JMP (14-byte)

When the hook function cannot be placed within ±2 GB of the target (common in 64-bit DLLs loaded at arbitrary ASLR addresses), a 14-byte absolute JMP is used:

; x64 14-byte absolute JMP to anywhere in 64-bit address space:
; FF 25 00 00 00 00 = JMP QWORD PTR [RIP+0]
; 8 bytes of target address
void BuildAbsJmp64(BYTE *buf, UINT64 target)
{
    buf[0] = 0xFF;  buf[1] = 0x25;  // JMP [RIP+0]
    buf[2] = 0x00;  buf[3] = 0x00;  buf[4] = 0x00;  buf[5] = 0x00;  // RIP offset = 0
    *(UINT64*)(buf + 6) = target;           // absolute target address
}

Hotpatch Variant

Microsoft builds many system DLLs with a 2-byte hotpatch area: mov edi, edi (0x8B 0xFF, a 2-byte no-op) at the function prologue, preceded by 5 bytes of padding (0x90 NOPs or INT 3s). A hotpatch hook writes a 2-byte short JMP at the prologue and a 5-byte JMP at the padding, without touching bytes after the prologue. The upside: no need to disassemble and identify instruction boundaries; the 2-byte slot is pre-allocated. The downside: only works on hotpatch-enabled functions, and modern x64 DLLs rarely include hotpatch padding.

Hook Detection

"""
Detect inline hooks in ntdll by comparing live function bytes
against the on-disk image.
"""
import ctypes, ctypes.wintypes, pefile, os

def detect_inline_hooks(target_dll: str = "ntdll.dll"):
    sysdir = ctypes.create_unicode_buffer(260)
    ctypes.windll.kernel32.GetSystemDirectoryW(sysdir, 260)
    disk_path = os.path.join(sysdir.value, target_dll)

    # Parse on-disk PE to get .text RVA and data
    pe = pefile.PE(disk_path, fast_load=True)
    pe.parse_data_directories(directories=[pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT']])

    text_section = next((s for s in pe.sections
                    if s.Name.strip(b'\x00') == b'.text'), None)
    if not text_section: return

    disk_text = text_section.get_data()
    text_rva  = text_section.VirtualAddress
    dll_base  = ctypes.windll.kernel32.GetModuleHandleW(target_dll)
    text_va   = dll_base + text_rva

    # Read in-memory .text
    size    = len(disk_text)
    mem_buf = (ctypes.c_ubyte * size)()
    read    = ctypes.c_size_t()
    ctypes.windll.kernel32.ReadProcessMemory(
        ctypes.wintypes.HANDLE(-1),
        ctypes.c_void_p(text_va), mem_buf, size, ctypes.byref(read))
    mem_text = bytes(mem_buf[:read.value])

    # Check export function prologues (first 5 bytes)
    for exp in pe.DIRECTORY_ENTRY_EXPORT.symbols:
        if not exp.address: continue
        offset = exp.address - text_rva
        if offset < 0 or offset + 5 > len(disk_text): continue
        disk_bytes = disk_text[offset:offset+5]
        live_bytes = mem_text[offset:offset+5]
        if disk_bytes != live_bytes:
            fname = exp.name.decode() if exp.name else f"ord{exp.ordinal}"
            print(f"  [HOOK] {fname}")
            print(f"    disk: {disk_bytes.hex()}")
            print(f"    live: {live_bytes.hex()}")
            # Decode JMP target if it's a 5-byte rel JMP
            if live_bytes[0] == 0xE9:
                import struct
                rel = struct.unpack_from('<i', live_bytes, 1)[0]
                target = dll_base + exp.address + 5 + rel
                print(f"    JMP target: 0x{target:016X}")

Q & A