Anti-Analysis and Anti-Debugging

Anti-analysis techniques force analysts to work harder and automated sandboxes to fail silently. They divide into three families: debugger detection (identify the analysis tool and change behavior), timing attacks (measure execution time to detect instrumentation overhead), and obfuscation (make static analysis difficult or meaningless). Detection engineers should understand these techniques to recognize when a sample is actively evading analysis — a sample that exits cleanly in a sandbox may be using these techniques, not actually being benign.

Debugger Detection

// Layer 1: Win32 API checks — easiest to detect, easily bypassed by analyst,
// but useful as a first gate to deter casual analysis.

BOOL CheckDebugger_Win32() {
    // IsDebuggerPresent: reads PEB.BeingDebugged byte
    if (IsDebuggerPresent()) return TRUE;

    // CheckRemoteDebuggerPresent: NtQueryInformationProcess(ProcessDebugPort)
    // Returns non-zero port if kernel debugger is attached
    BOOL remote = FALSE;
    CheckRemoteDebuggerPresent(GetCurrentProcess(), &remote);
    if (remote) return TRUE;

    return FALSE;
}

// Layer 2: PEB flags — read directly, bypasses IsDebuggerPresent() hook
BOOL CheckDebugger_PEB() {
    PPEB peb = (PPEB)__readgsqword(0x60);
    if (peb->BeingDebugged) return TRUE;

    // NtGlobalFlag: debugger sets 0x70 (heap validation flags) in PEB+0xBC
    DWORD ntgf = *(DWORD*)((BYTE*)peb + 0xBC);
    if (ntgf & 0x70) return TRUE;

    // HeapFlags: debugged process heap has extra flags set at PEB→ProcessHeap+0x70
    PVOID heap = peb->ProcessHeap;
    DWORD flags = *(DWORD*)((BYTE*)heap + 0x70);
    if (flags & 0xFFFFFFFD) return TRUE;

    return FALSE;
}

// Layer 3: Exception-based detection
// INT3 (0xCC) breakpoints: if no debugger, executing INT3 raises EXCEPTION_BREAKPOINT.
// If a debugger is present, it silently catches INT3 and continues — the exception
// handler never fires. The malware sets a flag in SEH; if flag stays unset → debugger.
BOOL CheckDebugger_SEH() {
    volatile BOOL handled = FALSE;
    __try {
        __asm { int 3 }   // raises EXCEPTION_BREAKPOINT if no debugger
    } __except (EXCEPTION_EXECUTE_HANDLER) {
        handled = TRUE;   // SEH handler ran → no debugger
    }
    return !handled;      // if handler never ran → debugger swallowed INT3
}

// Layer 4: Hardware breakpoints via thread context
// DR0-DR3 are debug registers. If any contain non-zero values, hardware BPs are set.
BOOL CheckHardwareBP() {
    CONTEXT ctx = {0}; ctx.ContextFlags = CONTEXT_DEBUG_REGISTERS;
    GetThreadContext(GetCurrentThread(), &ctx);
    return (ctx.Dr0 || ctx.Dr1 || ctx.Dr2 || ctx.Dr3);
}

Timing Checks

// Debugger step-through slows execution measurably.
// RDTSC measures CPU clock cycles; delta between two reads reveals instrumentation.
// GetTickCount / QPC: coarser but sufficient for sandbox sleep-acceleration detection.

BOOL CheckTiming_RDTSC() {
    UINT64 t1 = __rdtsc();
    // Execute a loop that takes ~1000 cycles normally
    volatile int x = 0;
    for (int i = 0; i < 1000; i++) x += i;
    UINT64 t2 = __rdtsc();
    // Normally ~1000 cycles; debugger step-through = millions of cycles
    return (t2 - t1) > 500000;
}

BOOL CheckTiming_QPC() {
    LARGE_INTEGER freq, t1, t2;
    QueryPerformanceFrequency(&freq);
    QueryPerformanceCounter(&t1);
    Sleep(500);
    QueryPerformanceCounter(&t2);
    double elapsed = (double)(t2.QuadPart - t1.QuadPart) / freq.QuadPart;
    // Sandbox accelerates Sleep → elapsed << 0.5s
    return (elapsed < 0.4);
}

Anti-Disassembly Tricks

// Disassemblers parse bytes linearly or recursively. Anti-disassembly inserts
// bytes that cause the disassembler to mis-parse subsequent instructions.

; Technique 1: Unconditional jump over a junk byte
; The 0xFF byte after the JMP is never executed but confuses linear-sweep disassemblers.
; IDA/Ghidra recursive descent handles this correctly; simple tools do not.
__asm {
    jmp over_junk
    db 0xFF        ; junk byte — would mis-parse as start of a 2-byte instruction
    over_junk:
    nop
}

; Technique 2: CALL/POP — return address manipulation confuses CFG analysis
__asm {
    call next      ; pushes return address onto stack
    next:
    pop rax        ; rax = address of next instruction (useful for PIC too)
    ; Disassembler sees "call" and assumes next instruction is after the call target;
    ; the real execution flow goes to "pop rax" immediately after.
}

; Technique 3: Overlapping instructions (x86 variable-length encoding)
; Byte at offset X is both the second byte of one instruction and
; the first byte of a jump target — one physical byte = two logical instructions.
; Only possible on x86/x64 (variable-length); not on ARM (fixed 4-byte).

String Obfuscation

// Plaintext strings in a binary (API names, C2 hostnames, registry paths)
// are the easiest static detection target. Obfuscate by XOR-encrypting at compile
// time and decrypting inline at runtime.

// Macro-based compile-time XOR encryption (C++ template trick):

template<size_t N, typename T = char>
struct XStr {
    T buf[N];
    constexpr XStr(const T(&s)[N], T key) {
        for (size_t i = 0; i < N; i++)
            buf[i] = s[i] ^ key;
    }
    T* dec(T key) {
        for (size_t i = 0; i < N; i++) buf[i] ^= key;
        return buf;
    }
};

// Usage: string "CreateThread" XOR'd with 0x55 at compile time
// Binary contains only garbage bytes; at runtime, .dec(0x55) is called inline.
constexpr auto str_ct = XStr<sizeof("CreateThread")>("CreateThread", 0x55);
// At call site: GetProcAddress(hK32, str_ct.dec(0x55));

// Hash-based API resolution (avoids all string storage):
// FNV-1a hash the target function name; store only the hash.
// Walk export table, hash each name, compare. No strings needed in binary.

DWORD Fnv1a(const char* s) {
    DWORD h = 0x811c9dc5;
    while (*s) { h ^= (BYTE)*s++; h *= 0x01000193; }
    return h;
}
// e.g., Fnv1a("VirtualAlloc") == 0x97bc257b — store hash, not string

Control Flow Obfuscation

// Control flow flattening: replace structured code with a dispatcher loop.
// All basic blocks become cases in a switch dispatched by a state variable.
// Static analysis sees a maze of cases with no obvious call structure;
// dynamic analysis still works but is significantly slower.

// Original logic:
//   if (check_a) { do_x(); } else { do_y(); }
//   do_z();

// After flattening:
int state = 0;  // initial state
while (state != -1) {
    switch (state) {
        case 0:
            state = check_a() ? 1 : 2;
            break;
        case 1:
            do_x();
            state = 3;
            break;
        case 2:
            do_y();
            state = 3;
            break;
        case 3:
            do_z();
            state = -1;
            break;
    }
}
// IDA/Ghidra can de-flatten this with plugins (e.g., D810, OLLVM de-obfuscator)
// but it requires manual effort per sample.

Anti-Analysis Technique Matrix

Detection Engineering

title: Process Reads Own PEB Debug Flags Directly (Anti-Debug)
logsource:
  product: windows
  service: windefend
detection:
  selection:
    EventID: 1116
    ThreatName|contains: 'AntiDebug'
  condition: selection
level: high
tags: [attack.defense_evasion, T1622]

title: Suspicious Short-Lived Process (Anti-Analysis Gate)
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    EventID: 1
  condition: selection  # correlate with process exit within <2s = sandbox bail-out
level: medium

-- MDE KQL: process runs and exits within 2 seconds (sandbox/debugger bail-out)
let starts = DeviceProcessEvents
    | where ActionType == "ProcessCreated"
    | project DeviceName, ProcessId, StartTime=Timestamp,
              FileName, ProcessCommandLine;
let stops = DeviceProcessEvents
    | where ActionType == "ProcessTerminated"
    | project DeviceName, ProcessId, StopTime=Timestamp;
starts
| join kind=inner stops on DeviceName, ProcessId
| where (StopTime - StartTime) < 2s
| where FileName !in~ (
    "conhost.exe","splwow64.exe","WerFault.exe","svchost.exe")
| project StartTime, DeviceName, FileName,
    ProcessCommandLine, lifetime=(StopTime - StartTime)
| order by lifetime asc

-- MDE KQL: high RDTSC / GetTickCount call volume (timing probe pattern)
DeviceEvents
| where ActionType == "AntivirusDetection"
    or ActionType == "BehaviorPrevented"
| where AdditionalFields has_any ("AntiDebug", "SuspiciousBehavior")
| project Timestamp, DeviceName, InitiatingProcessFileName, AdditionalFields

Q&A