Virtualization can be utilized to reach various goals as monitoring system, system resources and applications as well. It can be used for full system virtualuzation, but i like apporach using it just as a tool too . This post will shortly cover implementation of mini-hypervisor (which is now available on github) for intel vt-x on x64 platform, and demonstrate concept how-to-use-it.
My concept is based on hypervisors by :
- Virtual Machine Monitor by Shawn Embleton
- BluePill
Materials worth of reading :
- intel manuals!, especialy Volume 3C
- Hypervisor Abyss by @Ivanlef0u : part1, part2, part3
- Bochs VMX implementation ;)
- Implementation :
… PoC of hypervisor is shipped just for intel x64, and is compatible with win7 and win8 as well…
At the first phase it is necessary to initialize VMCS area host && guest, which is handled by VMX.h/.cpp component, and for further explanation are most educative articles – manuals from intel!
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
__checkReturn bool CVmx::VmcsInit() { ... *((ULONG_PTR*)(m_guestState.GVmcs.pvmcs)) = rdmsr(IA32_VMX_BASIC_MSR_CODE); *((ULONG_PTR*)(m_guestState.HVmcs.pvmcs)) = rdmsr(IA32_VMX_BASIC_MSR_CODE); vmxon(&(m_guestState.HVmcs.vmcs)); vmclear(&(m_guestState.GVmcs.vmcs)); vmptrld(&(m_guestState.GVmcs.vmcs)); //GLOBALS vmwrite(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0); vmwrite(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0); vmwrite(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0); SetCRx(); SetControls(); SetDT(); SetSysCall(); //GUEST vmwrite(VMX_VMCS_GUEST_LINK_PTR_FULL, -1); vmwrite(VMX_VMCS_GUEST_LINK_PTR_HIGH, -1); vmwrite(VMX_VMCS_GUEST_DEBUGCTL_FULL, rdmsr(IA32_DEBUGCTL) & SEG_D_LIMIT); vmwrite(VMX_VMCS_GUEST_DEBUGCTL_HIGH, rdmsr(IA32_DEBUGCTL) >> 32); //SELECTORS SetSegSelector(m_guestState.Cs, VMX_VMCS16_GUEST_FIELD_CS); SetSegSelector(m_guestState.Ds, VMX_VMCS16_GUEST_FIELD_DS); SetSegSelector(m_guestState.Es, VMX_VMCS16_GUEST_FIELD_ES); SetSegSelector(m_guestState.Ss, VMX_VMCS16_GUEST_FIELD_SS); SetSegSelector(m_guestState.Fs, VMX_VMCS16_GUEST_FIELD_FS); SetSegSelector(m_guestState.Gs, VMX_VMCS16_GUEST_FIELD_GS); SetSegSelector(m_guestState.Ldtr, VMX_VMCS16_GUEST_FIELD_LDTR); SetSegSelector(m_guestState.Tr, VMX_VMCS16_GUEST_FIELD_TR); vmwrite(VMX_VMCS64_GUEST_FS_BASE, rdmsr(IA32_FS_BASE)); vmwrite(VMX_VMCS64_GUEST_GS_BASE, rdmsr(IA32_GS_BASE)); //HOST vmwrite(VMX_VMCS16_HOST_FIELD_CS, m_guestState.Cs); vmwrite(VMX_VMCS16_HOST_FIELD_DS, SEG_DATA); vmwrite(VMX_VMCS16_HOST_FIELD_ES, SEG_DATA); vmwrite(VMX_VMCS16_HOST_FIELD_SS, m_guestState.Ss); vmwrite(VMX_VMCS16_HOST_FIELD_FS, m_guestState.Fs & 0xf8); vmwrite(VMX_VMCS16_HOST_FIELD_GS, m_guestState.Gs & 0xf8); vmwrite(VMX_VMCS16_HOST_FIELD_TR, m_guestState.Tr); RtlZeroMemory((PVOID)((ULONG_PTR)m_guestState.GVmcs.pvmcs + 4), 4); ... } |
Next step is turning on hypervisor and handling enter to hypervisor mode. This is implemented by little bit ‘hacky’ way
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
__checkReturn bool CVmx::VmcsInit() { ULONG_PTR guest_rsp; ULONG_PTR guest_rip; get_guest_exit(&guest_rsp, &guest_rip); if (m_cpuActivated) return true; ... vmwrite(VMX_VMCS64_GUEST_RSP, guest_rsp); vmwrite(VMX_VMCS64_GUEST_RIP, guest_rip); vmwrite(VMX_VMCS_GUEST_RFLAGS, m_guestState.RFLAGS); vmwrite(VMX_VMCS_HOST_RSP, m_guestState.HRSP); vmwrite(VMX_VMCS_HOST_RIP, m_guestState.HRIP); m_cpuActivated = true; vmlaunch(); ... } //hacky way -compiler dependent - to obtain rsp + rip .code get_guest_exit proc lea rax, [rsp + sizeof(QWORD)] mov [rcx], rax mov rax, [rsp] mov [rdx], rax xor rax, rax ret get_guest_exit endp end |
From concept HyperVisor is an object, which has its own state { core_id, callback, exit_traps }, so i store this object to host stack for further usage :
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 |
CVirtualizedCpu::CVirtualizedCpu( __in BYTE cpuCore, __in_opt const ULONG_PTR traps[MAX_CALLBACK], __in_opt const VOID* callback, __in_opt const VOID* param ) : m_vmx(PROCID(cpuCore)), m_cpuCore(cpuCore) { LARGE_INTEGER HighestAcceptableAddress; HighestAcceptableAddress.HighPart = -1; m_hvStack = (ULONG_PTR*)MmAllocateContiguousMemory(HYPERVISOR_STACK_PAGE, HighestAcceptableAddress); if (NULL == m_hvStack) return; RtlZeroMemory(m_hvStack, HYPERVISOR_STACK_PAGE); m_hvStack[0] = kStackMark; m_hvStack[1] = (ULONG_PTR)param; ::new(m_hvStack + 2) CHyperVisor(cpuCore, traps, callback); } EXTERN_C ULONG_PTR HVExitTrampoline( __inout ULONG_PTR reg[0x10] ) { ULONG_PTR ds; vmread(VMX_VMCS16_GUEST_FIELD_DS, &ds); xchgds(&ds); ULONG_PTR* stack_top = CVirtualizedCpu::GetTopOfStack(reg); CHyperVisor* hypervisor = (CHyperVisor*)(stack_top + 2); ULONG_PTR handler = hypervisor->HVEntryPoint(reg, (stack_top + 1)); writeds(ds); return handler; } |
In this PoC is implemented as few events as possible, and all VM-exit handlers can be redefined :
|
1 2 3 4 5 6 7 8 9 10 11 12 |
virtual __checkReturn bool SetVirtualizationCallbacks(); __checkReturn bool CSysCall::SetVirtualizationCallbacks() { DbgPrint("CSysCall::SetVirtualizationCallbacks\n"); if (!CCRonos::SetVirtualizationCallbacks()) return false; m_traps[VMX_EXIT_RDMSR] = (ULONG_PTR)HookProtectionMSR; return true; } |
And also mechanism for callbacks per vm-exit is implemented :
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
ULONG_PTR CHyperVisor::HVEntryPoint( __inout ULONG_PTR reg[0x10], __in VOID* param ) { (*(void (*)(ULONG_PTR*, const void*))(m_callback))(reg, param); ULONG_PTR ExitReason; vmread(VMX_VMCS32_RO_EXIT_REASON, &ExitReason); ULONG_PTR ExitInstructionLength; vmread(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &ExitInstructionLength); ULONG_PTR GuestEIP; vmread(VMX_VMCS64_GUEST_RIP, &GuestEIP); vmwrite(VMX_VMCS64_GUEST_RIP, GuestEIP + ExitInstructionLength); if((ExitReason > VMX_EXIT_VMCALL) && (ExitReason <= VMX_EXIT_VMXON)) { ULONG_PTR GuestRFLAGS; vmread(VMX_VMCS_GUEST_RFLAGS, &GuestRFLAGS); vmwrite(VMX_VMCS_GUEST_RFLAGS, GuestRFLAGS & (~0x8d5) | 0x1); } if (VMX_EXIT_CRX_MOVE == ExitReason) HandleCrxAccess(reg); return m_hvCallbacks[ExitReason]; } |
and can be utilized as you pleased :
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
void CCRonos::PerCoreAction(__in BYTE coreId) { m_vCpuCount++; ::new(m_vCpu + coreId) CVirtualizedCpu(coreId, m_traps, HVCallback, m_callbacks); } __checkReturn bool CCRonos::SetVirtualizationCallbacks() { DbgPrint("CCRonos::SetVirtualizationCallbacks\n"); m_traps[VMX_EXIT_CPUID] = (ULONG_PTR)HVCpuid; return (RegisterCallback(m_callbacks, HVCallback1) && RegisterCallback(m_callbacks, HVCallback2) && RegisterCallback(m_callbacks, HVCallback3)); } bool CCRonos::RegisterCallback( __in CALLBACK* callbacks, __in const void* callback ) { while (NULL != callbacks->Next) callbacks = callbacks->Next; CALLBACK* t_callback = (CALLBACK*)malloc(sizeof(CALLBACK)); if (!t_callback) return false; RtlZeroMemory(t_callback, sizeof(*t_callback)); ::new(callbacks) CALLBACK(callback, t_callback); return true; } void CCRonos::HVCallback(__inout ULONG_PTR reg[0x10], __in_opt CALLBACK** callbacks) { if (!callbacks) return; CALLBACK* t_callbacks = *callbacks; if (!t_callbacks) return; while (t_callbacks->Callback != NULL) { (*(void (*)(ULONG_PTR*))(t_callbacks->Callback))(reg); t_callbacks = t_callbacks->Next; } } |
- Usage
How to use it, i demonstrates by program, which want to set own SYSCALL routine and also hunt for PatchGuard context. It is located on github, and this image describe it briefly :
In fact, it is easy as it looks :
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
__checkReturn bool CSysCall::SetVirtualizationCallbacks() { DbgPrint("CSysCall::SetVirtualizationCallbacks\n"); if (!CCRonos::SetVirtualizationCallbacks()) return false; m_traps[VMX_EXIT_RDMSR] = (ULONG_PTR)HookProtectionMSR; return true;//RegisterCallback(m_callbacks, ); } void CSysCall::PerCoreAction( __in BYTE coreId ) { CCRonos::PerCoreAction(coreId); if (coreId < sizeof(m_syscalls)) { KeSetSystemAffinityThread(PROCID(coreId)); m_syscalls[coreId] = (void*)rdmsr(IA64_SYSENTER_EIP); HookSyscallMSR(sysenter); DbgPrint("Hooked. procid [%x] <=> syscall addr [%p]\n", coreId, m_syscalls[coreId]); } } //**** //HV callback -> hook protection! void CSysCall::HookProtectionMSR( __inout ULONG_PTR reg[0x10] ) { ULONG_PTR syscall; if (IA64_SYSENTER_EIP == reg[RCX]) { syscall = (ULONG_PTR)CSysCall::GetSysCall(CVirtualizedCpu::GetCoreId(reg)); ULONG_PTR ins_len; vmread(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &ins_len); vmread(VMX_VMCS64_GUEST_RIP, ®[RCX]);//original 'ret'-addr m_sRdmsrRips.Push(reg[RCX] - ins_len); vmwrite(VMX_VMCS64_GUEST_RIP, rdmsr_hook);//rdmsr_hook is trampolie to RdmsrHookook } else { syscall = rdmsr((ULONG)reg[RCX]); } reg[RAX] = (ULONG)(syscall); reg[RDX] = (ULONG)(syscall >> (sizeof(ULONG) << 3)); } //little bit another kind of hook -virtualization-based- :P EXTERN_C void* RdmsrHook( __inout ULONG_PTR* reg ) { void* ret = (void*)reg[RCX]; DbgPrint("\nRdmsrHook %p [pethread : %p]\n", ret, PsGetCurrentThread()); reg[RCX] = IA64_SYSENTER_EIP; KeBreak(); return ret; } |
- Utilize as yo want
Hypervisor is great tool, if you have some ideas how to use it. it can be used not just as virtualization of system, malware / anti-malware solutions ..
- DBI – monitor application
- Monitor system
- altering system
- sanboxing
- …
interesting projects are HyperDbg, HDBG, and a source of knowledge can be also bochs, NOVA
You can monitor & play with system / application in your own. You can use EPT for monitoring memory access, combine with another cpu features … if you set-up your hypervisor right, you can have callback at hypervisor level (and trasfered to non-root mode for free if you want) at event you want. VM-exit switch is not so cheap, but also it is no tragedy, and goal can easly overcome it
Great insight and the draft is really deep. I see Intel could have an offset to darker sides as well. Could you cover side channel attacks which could reside in Intel chips on L3 cache?