Anti-Debugging and Anti-Analysis Techniques

Anti-debug and anti-analysis checks allow a payload to detect when it is being examined — in a debugger, a sandbox, or an automated analysis environment — and change behavior accordingly. This chapter covers the full taxonomy from PEB flag checks through timing-based detection, NtQueryInformationProcess, heap flags, structured exception handling tricks, and parent process validation — with every check implemented in C and the defender's counter for each.

Anti-Analysis Taxonomy

IsDebuggerPresent and PEB Checks

// PEB.BeingDebugged (offset 0x2 on x64) is the canonical flag.
// IsDebuggerPresent() just reads this byte.
// NtGlobalFlag (PEB offset 0xBC on x64) is set to 0x70 under a debugger.

#include <windows.h>
#include <winternl.h>

BOOL CheckPebDebug(void) {
    // Method 1: standard API (trivially bypassed by ScyllaHide)
    if (IsDebuggerPresent()) return TRUE;

    // Method 2: read PEB.BeingDebugged directly via GS segment
    PPEB peb = (PPEB)__readgsqword(0x60);  // GS:[0x60] = PEB on x64
    if (peb->BeingDebugged) return TRUE;

    // Method 3: NtGlobalFlag — set to 0x70 under debugger (heap debug flags)
    DWORD ntGlobalFlag = *(PDWORD)((PBYTE)peb + 0xBC);
    if (ntGlobalFlag & 0x70) return TRUE;

    return FALSE;
}

// Anti-bypass: if using ScyllaHide, the PEB flags are patched to 0.
// Combine PEB checks with timing and NtQIP for defense-in-depth.
// Never crash on detection — corrupt a key byte silently:
uint8_t g_key[32] = { ... };
if (CheckPebDebug()) g_key[0] ^= 0xFF;  // silently corrupt → decrypt fails later

Timing Checks

// Debuggers introduce timing delays due to breakpoint handling.
// RDTSC measures CPU cycles; delta between two calls should be tiny.
// If too large: debugger stepped over instructions.

BOOL CheckTiming(void) {
    // Method 1: RDTSC — cycle count
    UINT64 t1 = __rdtsc();
    // Do a small amount of work:
    volatile int x = 0;
    for (int i = 0; i < 100; i++) x += i;
    UINT64 t2 = __rdtsc();
    if ((t2 - t1) > 500000) return TRUE;  // debugger slowed us down

    // Method 2: GetTickCount delta — coarser but works across VMs
    DWORD before = GetTickCount();
    Sleep(500);
    DWORD after = GetTickCount();
    if ((after - before) > 1000) return TRUE;  // more than 2x expected

    // Method 3: QueryPerformanceCounter for high-resolution timing
    LARGE_INTEGER f, s, e;
    QueryPerformanceFrequency(&f);
    QueryPerformanceCounter(&s);
    x = 0; for (int i = 0; i < 1000; i++) x += i;
    QueryPerformanceCounter(&e);
    double ms = (double)(e.QuadPart - s.QuadPart) / f.QuadPart * 1000.0;
    if (ms > 50.0) return TRUE;

    return FALSE;
}
// Bypass: analyst can NOP out the timing check, but only if they find it.
// Place timing check AFTER other decryption — if they step to get the key,
// the delta will be large regardless.

NtQueryInformationProcess Checks

// NtQueryInformationProcess reveals debug attachment at kernel level.
// ProcessDebugPort (7): non-zero if debugger attached via DebugActiveProcess
// ProcessDebugObjectHandle (30): valid HANDLE if debugger attached
// ProcessDebugFlags (31): 0 if debugger present (counterintuitively)

typedef NTSTATUS(NTAPI* pNtQIP)(HANDLE, PROCESSINFOCLASS,
                                    PVOID, ULONG, PULONG);

BOOL CheckNtQip(void) {
    pNtQIP NtQIP = (pNtQIP)GetProcAddress(
        GetModuleHandleA("ntdll"), "NtQueryInformationProcess");

    // ProcessDebugPort (class 7)
    DWORD_PTR port = 0;
    NtQIP(GetCurrentProcess(), (PROCESSINFOCLASS)7,
          &port, sizeof(port), NULL);
    if (port != 0) return TRUE;

    // ProcessDebugObjectHandle (class 30)
    HANDLE hDbgObj = NULL;
    NTSTATUS st = NtQIP(GetCurrentProcess(), (PROCESSINFOCLASS)30,
                        &hDbgObj, sizeof(hDbgObj), NULL);
    if (NT_SUCCESS(st) && hDbgObj) { CloseHandle(hDbgObj); return TRUE; }

    // ProcessDebugFlags (class 31) — 0 = debugger attached
    DWORD flags = 0;
    NtQIP(GetCurrentProcess(), (PROCESSINFOCLASS)31,
          &flags, sizeof(flags), NULL);
    if (flags == 0) return TRUE;

    return FALSE;
}
// Bypass (ScyllaHide): hooks NtQueryInformationProcess to return spoofed values.
// Harder bypass: use direct syscall (NtQueryInformationProcess syscall number)
// to bypass ScyllaHide's userland hook — ScyllaHide typically hooks NTDLL stubs.

Heap Flag Detection

// The Windows heap manager sets special flags in debug mode.
// HEAP_TAIL_CHECKING_ENABLED (0x20), HEAP_FREE_CHECKING_ENABLED (0x40),
// HEAP_VALIDATE_PARAMETERS_ENABLED (0x40000000)
// Access the heap header directly — no API required.

BOOL CheckHeapFlags(void) {
    PPEB peb = (PPEB)__readgsqword(0x60);

    // ProcessHeap is at PEB+0x30 on x64
    PVOID heap = *(PVOID*)((PBYTE)peb + 0x30);

    // Heap.Flags at offset 0x70 on x64; Heap.ForceFlags at 0x74
    DWORD heapFlags  = *(PDWORD)((PBYTE)heap + 0x70);
    DWORD forceFlags = *(PDWORD)((PBYTE)heap + 0x74);

    // Normal: Flags = 2 (HEAP_GROWABLE), ForceFlags = 0
    // Debugger: Flags has extra bits set, ForceFlags != 0
    if (heapFlags & ~0x2) return TRUE;
    if (forceFlags != 0)  return TRUE;
    return FALSE;
}

INT3 and SEH-Based Exception Checks

// Raise an INT3 exception. If a debugger is present, it consumes the
// exception silently (it's a breakpoint). If no debugger: SEH handler fires.
// Check which path executed — SEH handler = no debugger.

BOOL g_handlerFired = FALSE;

LONG WINAPI Int3VehHandler(PEXCEPTION_POINTERS ep) {
    if (ep->ExceptionRecord->ExceptionCode == EXCEPTION_BREAKPOINT) {
        g_handlerFired = TRUE;
        ep->ContextRecord->Rip++;  // skip the INT3 byte
        return EXCEPTION_CONTINUE_EXECUTION;
    }
    return EXCEPTION_CONTINUE_SEARCH;
}

BOOL CheckInt3(void) {
    PVOID veh = AddVectoredExceptionHandler(1, Int3VehHandler);
    g_handlerFired = FALSE;

    // Raise INT3
    __asm__ volatile("int3");

    RemoveVectoredExceptionHandler(veh);
    return !g_handlerFired;  // TRUE = debugger consumed our exception
}

// OutputDebugString trick:
// OutputDebugString raises EXCEPTION_RPC_S_SERVER_UNAVAILABLE if no debugger.
// If a debugger is attached, GetLastError() is 0 after the call.
BOOL CheckOutputDebugString(void) {
    SetLastError(0xDEAD);
    OutputDebugStringA("test");
    return GetLastError() == 0;  // TRUE = debugger consumed it
}

Parent Process Validation

// Normal execution: parent is explorer.exe, cmd.exe, or a known launcher.
// Sandbox: parent may be the sandbox orchestrator (cuckoo.exe, analyzer.exe).
// Debugger spawn: parent is the debugger (x64dbg.exe, windbg.exe, ollydbg.exe).

BOOL CheckParentProcess(void) {
    DWORD parentPid = 0;
    // Get parent PID from NtQueryInformationProcess (ProcessBasicInformation)
    typedef NTSTATUS(NTAPI* pNtQIP)(HANDLE, PROCESSINFOCLASS,
                                        PVOID, ULONG, PULONG);
    pNtQIP NtQIP = (pNtQIP)GetProcAddress(
        GetModuleHandleA("ntdll"), "NtQueryInformationProcess");
    PROCESS_BASIC_INFORMATION pbi = {0};
    NtQIP(GetCurrentProcess(), ProcessBasicInformation,
          &pbi, sizeof(pbi), NULL);
    parentPid = (DWORD)((ULONG_PTR)pbi.Reserved3);  // InheritedFromUniqueProcessId

    // Look up parent process name
    HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
    PROCESSENTRY32 pe = { .dwSize = sizeof(pe) };
    if (Process32First(snap, &pe)) {
        do {
            if (pe.th32ProcessID == parentPid) {
                // Blocklist known debuggers/analysis tools
                const char* bad[] = {
                    "x64dbg.exe", "x32dbg.exe", "windbg.exe",
                    "ollydbg.exe", "ida64.exe", "ida.exe",
                    "ghidra.exe", "cuckoo.exe", "analyzer.exe",
                    "python.exe",  // cuckoo agent
                    NULL
                };
                for (int i = 0; bad[i]; i++) {
                    if (_stricmp(pe.szExeFile, bad[i]) == 0) {
                        CloseHandle(snap);
                        return TRUE;
                    }
                }
                break;
            }
        } while (Process32Next(snap, &pe));
    }
    CloseHandle(snap);
    return FALSE;
}

Detection Engineering

title: Process Querying Own Debug State (Anti-Debug Pattern)
logsource:
  product: windows
  category: process_access
detection:
  selection:
    TargetImage: 'self'  # process opening handle to itself
    GrantedAccess|contains: 'PROCESS_QUERY_INFORMATION'
  condition: selection
level: low
note: Low fidelity alone; combine with high-entropy file and other IOCs

title: Vectored Exception Handler Added in Suspicious Process
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    Image|contains:
      - '\Temp\'
      - '\AppData\'
      - '\Downloads\'
  condition: selection
note: Trigger on image load + VEH registration from suspicious path — EDR-only

-- MDE KQL: process enumerating all processes immediately after launch (parent check)
DeviceProcessEvents
| where ActionType == "ProcessCreated"
| join kind=inner (
    DeviceEvents
    | where ActionType == "CreateRemoteThreadApiCall"
        or ActionType == "OpenProcessApiCall"
    | where Timestamp between (ago(5s) .. now())
) on DeviceName, InitiatingProcessId
| where InitiatingProcessFolderPath !startswith @"C:\Windows"
| project Timestamp, DeviceName, InitiatingProcessFileName,
          InitiatingProcessCommandLine, ActionType

Q&A