Custom VM Reverse Engineering

The challenge consists of an ELF executable that interprets a separate bytecode file. Its input checks follow a repeated pattern, so we can use symbolic execution to recover each flag character.

Recon

file and ltrace shows a.out is interpreting the bytes in virtue.

$ file a.out
a.out: ELF 64-bit LSB pie executable...

$ ./a.out virtue
Enter flag:

VM Analysis

Opening a.out in Ghidra shows:

• main reads argv[1] into a buffer, calls run(buffer, size).
• run is a VM loop.

By analyzing the switch/if-else block in run, instruction set can be reconstructed:

Opcode	Meaning	Notes
0x01	MOV	copy
0x02	ADD	arithmetic
0x03	XOR	arithmetic
0x04	CMP	sets zero flag
0x05	JMP	absolute
0x06	JNE	jump if not zero
0x07	OUT	putchar
0x08	IN	getchar
0xFF	EXIT	halt

Solution

The bytecode follows a repeated pattern: read input char (IN) -> apply ADD/XOR operations -> CMP to target value.

1. When IN executed, store a placeholder instead of a concrete value.
2. When ADD/XOR executed, record the operation history.
3. When CMP is reached, reverse the recorded operations to compute exact input byte needed to satisfy the comparison.

import sys

def solve():
    try:
        with open("virtue", "rb") as f:
            code = f.read()
    except:
        return

    regs = [0] * 256
    flag = []

    # Track input history
    class Symbolic:
        def __init__(self, idx):
            self.history = []
        def op(self, type, val):
            new = Symbolic(0)
            new.history = self.history + [(type, val)]
            return new

    pc = 0
    flag_zero = False

    while pc < len(code):
        op = code[pc]

        if op == 1:  # MOV
            regs[code[pc+1]] = code[pc+2]
            pc += 3
        elif op == 2:  # ADD
            dst, src = code[pc+1], code[pc+2]
            v_dst, v_src = regs[dst], regs[src]
            if isinstance(v_dst, Symbolic) and isinstance(v_src, int):
                regs[dst] = v_dst.op('+', v_src)
            elif isinstance(v_dst, int) and isinstance(v_src, Symbolic):
                regs[dst] = v_src.op('+', v_dst)
            elif isinstance(v_dst, int) and isinstance(v_src, int):
                regs[dst] = (v_dst + v_src) & 0xFF
            pc += 3
        elif op == 3:  # XOR
            dst, src = code[pc+1], code[pc+2]
            v_dst, v_src = regs[dst], regs[src]
            if isinstance(v_dst, Symbolic) and isinstance(v_src, int):
                regs[dst] = v_dst.op('^', v_src)
            elif isinstance(v_dst, int) and isinstance(v_src, Symbolic):
                regs[dst] = v_src.op('^', v_dst)
            elif isinstance(v_dst, int) and isinstance(v_src, int):
                regs[dst] = v_dst ^ v_src
            pc += 3
        elif op == 4:  # CMP
            reg, target = code[pc+1], code[pc+2]
            val = regs[reg]
            if isinstance(val, Symbolic):
                # Reverse the math
                curr = target
                for type, operand in reversed(val.history):
                    if type == '+':
                        curr = (curr - operand) & 0xFF
                    elif type == '^':
                        curr ^= operand
                print(chr(curr), end="")
                flag_zero = True
            else:
                flag_zero = (val == target)
            pc += 3
        elif op == 5:  # JMP
            pc = (code[pc+2] << 8) | code[pc+1]
        elif op == 6:  # JNE
            addr = (code[pc+2] << 8) | code[pc+1]
            pc = addr if not flag_zero else pc + 3
        elif op == 7:  # OUT
            pc += 2
        elif op == 8:  # IN
            regs[code[pc+1]] = Symbolic(0)
            pc += 2
        elif op == 0xFF:
            break
        else:
            break

if __name__ == "__main__":
    solve()