1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2012 Sandvine, Inc.
5 * Copyright (c) 2012 NetApp, Inc.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #ifdef _KERNEL
32 #include <sys/param.h>
33 #include <sys/pcpu.h>
34 #include <sys/systm.h>
35 #include <sys/proc.h>
36
37 #include <vm/vm.h>
38 #include <vm/pmap.h>
39
40 #include <machine/vmparam.h>
41 #include <machine/vmm.h>
42 #else /* !_KERNEL */
43 #include <sys/types.h>
44 #include <sys/errno.h>
45 #include <sys/_iovec.h>
46
47 #include <machine/vmm.h>
48
49 #include <err.h>
50 #include <assert.h>
51 #include <stdbool.h>
52 #include <stddef.h>
53 #include <stdio.h>
54 #include <string.h>
55 #include <strings.h>
56 #include <vmmapi.h>
57 #define __diagused
58 #define KASSERT(exp,msg) assert((exp))
59 #define panic(...) errx(4, __VA_ARGS__)
60 #endif /* _KERNEL */
61
62 #include <machine/vmm_instruction_emul.h>
63 #include <x86/psl.h>
64 #include <x86/specialreg.h>
65
66 /* struct vie_op.op_type */
67 enum {
68 VIE_OP_TYPE_NONE = 0,
69 VIE_OP_TYPE_MOV,
70 VIE_OP_TYPE_MOVSX,
71 VIE_OP_TYPE_MOVZX,
72 VIE_OP_TYPE_AND,
73 VIE_OP_TYPE_OR,
74 VIE_OP_TYPE_SUB,
75 VIE_OP_TYPE_TWO_BYTE,
76 VIE_OP_TYPE_PUSH,
77 VIE_OP_TYPE_CMP,
78 VIE_OP_TYPE_POP,
79 VIE_OP_TYPE_MOVS,
80 VIE_OP_TYPE_GROUP1,
81 VIE_OP_TYPE_STOS,
82 VIE_OP_TYPE_BITTEST,
83 VIE_OP_TYPE_TWOB_GRP15,
84 VIE_OP_TYPE_ADD,
85 VIE_OP_TYPE_TEST,
86 VIE_OP_TYPE_BEXTR,
87 VIE_OP_TYPE_LAST
88 };
89
90 /* struct vie_op.op_flags */
91 #define VIE_OP_F_IMM (1 << 0) /* 16/32-bit immediate operand */
92 #define VIE_OP_F_IMM8 (1 << 1) /* 8-bit immediate operand */
93 #define VIE_OP_F_MOFFSET (1 << 2) /* 16/32/64-bit immediate moffset */
94 #define VIE_OP_F_NO_MODRM (1 << 3)
95 #define VIE_OP_F_NO_GLA_VERIFICATION (1 << 4)
96
97 static const struct vie_op three_byte_opcodes_0f38[256] = {
98 [0xF7] = {
99 .op_byte = 0xF7,
100 .op_type = VIE_OP_TYPE_BEXTR,
101 },
102 };
103
104 static const struct vie_op two_byte_opcodes[256] = {
105 [0xAE] = {
106 .op_byte = 0xAE,
107 .op_type = VIE_OP_TYPE_TWOB_GRP15,
108 },
109 [0xB6] = {
110 .op_byte = 0xB6,
111 .op_type = VIE_OP_TYPE_MOVZX,
112 },
113 [0xB7] = {
114 .op_byte = 0xB7,
115 .op_type = VIE_OP_TYPE_MOVZX,
116 },
117 [0xBA] = {
118 .op_byte = 0xBA,
119 .op_type = VIE_OP_TYPE_BITTEST,
120 .op_flags = VIE_OP_F_IMM8,
121 },
122 [0xBE] = {
123 .op_byte = 0xBE,
124 .op_type = VIE_OP_TYPE_MOVSX,
125 },
126 };
127
128 static const struct vie_op one_byte_opcodes[256] = {
129 [0x03] = {
130 .op_byte = 0x03,
131 .op_type = VIE_OP_TYPE_ADD,
132 },
133 [0x0F] = {
134 .op_byte = 0x0F,
135 .op_type = VIE_OP_TYPE_TWO_BYTE
136 },
137 [0x0B] = {
138 .op_byte = 0x0B,
139 .op_type = VIE_OP_TYPE_OR,
140 },
141 [0x2B] = {
142 .op_byte = 0x2B,
143 .op_type = VIE_OP_TYPE_SUB,
144 },
145 [0x39] = {
146 .op_byte = 0x39,
147 .op_type = VIE_OP_TYPE_CMP,
148 },
149 [0x3B] = {
150 .op_byte = 0x3B,
151 .op_type = VIE_OP_TYPE_CMP,
152 },
153 [0x88] = {
154 .op_byte = 0x88,
155 .op_type = VIE_OP_TYPE_MOV,
156 },
157 [0x89] = {
158 .op_byte = 0x89,
159 .op_type = VIE_OP_TYPE_MOV,
160 },
161 [0x8A] = {
162 .op_byte = 0x8A,
163 .op_type = VIE_OP_TYPE_MOV,
164 },
165 [0x8B] = {
166 .op_byte = 0x8B,
167 .op_type = VIE_OP_TYPE_MOV,
168 },
169 [0xA1] = {
170 .op_byte = 0xA1,
171 .op_type = VIE_OP_TYPE_MOV,
172 .op_flags = VIE_OP_F_MOFFSET | VIE_OP_F_NO_MODRM,
173 },
174 [0xA3] = {
175 .op_byte = 0xA3,
176 .op_type = VIE_OP_TYPE_MOV,
177 .op_flags = VIE_OP_F_MOFFSET | VIE_OP_F_NO_MODRM,
178 },
179 [0xA4] = {
180 .op_byte = 0xA4,
181 .op_type = VIE_OP_TYPE_MOVS,
182 .op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
183 },
184 [0xA5] = {
185 .op_byte = 0xA5,
186 .op_type = VIE_OP_TYPE_MOVS,
187 .op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
188 },
189 [0xAA] = {
190 .op_byte = 0xAA,
191 .op_type = VIE_OP_TYPE_STOS,
192 .op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
193 },
194 [0xAB] = {
195 .op_byte = 0xAB,
196 .op_type = VIE_OP_TYPE_STOS,
197 .op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
198 },
199 [0xC6] = {
200 /* XXX Group 11 extended opcode - not just MOV */
201 .op_byte = 0xC6,
202 .op_type = VIE_OP_TYPE_MOV,
203 .op_flags = VIE_OP_F_IMM8,
204 },
205 [0xC7] = {
206 .op_byte = 0xC7,
207 .op_type = VIE_OP_TYPE_MOV,
208 .op_flags = VIE_OP_F_IMM,
209 },
210 [0x23] = {
211 .op_byte = 0x23,
212 .op_type = VIE_OP_TYPE_AND,
213 },
214 [0x80] = {
215 /* Group 1 extended opcode */
216 .op_byte = 0x80,
217 .op_type = VIE_OP_TYPE_GROUP1,
218 .op_flags = VIE_OP_F_IMM8,
219 },
220 [0x81] = {
221 /* Group 1 extended opcode */
222 .op_byte = 0x81,
223 .op_type = VIE_OP_TYPE_GROUP1,
224 .op_flags = VIE_OP_F_IMM,
225 },
226 [0x83] = {
227 /* Group 1 extended opcode */
228 .op_byte = 0x83,
229 .op_type = VIE_OP_TYPE_GROUP1,
230 .op_flags = VIE_OP_F_IMM8,
231 },
232 [0x8F] = {
233 /* XXX Group 1A extended opcode - not just POP */
234 .op_byte = 0x8F,
235 .op_type = VIE_OP_TYPE_POP,
236 },
237 [0xF6] = {
238 /* XXX Group 3 extended opcode - not just TEST */
239 .op_byte = 0xF6,
240 .op_type = VIE_OP_TYPE_TEST,
241 .op_flags = VIE_OP_F_IMM8,
242 },
243 [0xF7] = {
244 /* XXX Group 3 extended opcode - not just TEST */
245 .op_byte = 0xF7,
246 .op_type = VIE_OP_TYPE_TEST,
247 .op_flags = VIE_OP_F_IMM,
248 },
249 [0xFF] = {
250 /* XXX Group 5 extended opcode - not just PUSH */
251 .op_byte = 0xFF,
252 .op_type = VIE_OP_TYPE_PUSH,
253 }
254 };
255
256 /* struct vie.mod */
257 #define VIE_MOD_INDIRECT 0
258 #define VIE_MOD_INDIRECT_DISP8 1
259 #define VIE_MOD_INDIRECT_DISP32 2
260 #define VIE_MOD_DIRECT 3
261
262 /* struct vie.rm */
263 #define VIE_RM_SIB 4
264 #define VIE_RM_DISP32 5
265
266 #define GB (1024 * 1024 * 1024)
267
268 static enum vm_reg_name gpr_map[16] = {
269 VM_REG_GUEST_RAX,
270 VM_REG_GUEST_RCX,
271 VM_REG_GUEST_RDX,
272 VM_REG_GUEST_RBX,
273 VM_REG_GUEST_RSP,
274 VM_REG_GUEST_RBP,
275 VM_REG_GUEST_RSI,
276 VM_REG_GUEST_RDI,
277 VM_REG_GUEST_R8,
278 VM_REG_GUEST_R9,
279 VM_REG_GUEST_R10,
280 VM_REG_GUEST_R11,
281 VM_REG_GUEST_R12,
282 VM_REG_GUEST_R13,
283 VM_REG_GUEST_R14,
284 VM_REG_GUEST_R15
285 };
286
287 static uint64_t size2mask[] = {
288 [1] = 0xff,
289 [2] = 0xffff,
290 [4] = 0xffffffff,
291 [8] = 0xffffffffffffffff,
292 };
293
294 static int
vie_read_register(struct vcpu * vcpu,enum vm_reg_name reg,uint64_t * rval)295 vie_read_register(struct vcpu *vcpu, enum vm_reg_name reg, uint64_t *rval)
296 {
297 int error;
298
299 error = vm_get_register(vcpu, reg, rval);
300
301 return (error);
302 }
303
304 static void
vie_calc_bytereg(struct vie * vie,enum vm_reg_name * reg,int * lhbr)305 vie_calc_bytereg(struct vie *vie, enum vm_reg_name *reg, int *lhbr)
306 {
307 *lhbr = 0;
308 *reg = gpr_map[vie->reg];
309
310 /*
311 * 64-bit mode imposes limitations on accessing legacy high byte
312 * registers (lhbr).
313 *
314 * The legacy high-byte registers cannot be addressed if the REX
315 * prefix is present. In this case the values 4, 5, 6 and 7 of the
316 * 'ModRM:reg' field address %spl, %bpl, %sil and %dil respectively.
317 *
318 * If the REX prefix is not present then the values 4, 5, 6 and 7
319 * of the 'ModRM:reg' field address the legacy high-byte registers,
320 * %ah, %ch, %dh and %bh respectively.
321 */
322 if (!vie->rex_present) {
323 if (vie->reg & 0x4) {
324 *lhbr = 1;
325 *reg = gpr_map[vie->reg & 0x3];
326 }
327 }
328 }
329
330 static int
vie_read_bytereg(struct vcpu * vcpu,struct vie * vie,uint8_t * rval)331 vie_read_bytereg(struct vcpu *vcpu, struct vie *vie, uint8_t *rval)
332 {
333 uint64_t val;
334 int error, lhbr;
335 enum vm_reg_name reg;
336
337 vie_calc_bytereg(vie, ®, &lhbr);
338 error = vm_get_register(vcpu, reg, &val);
339
340 /*
341 * To obtain the value of a legacy high byte register shift the
342 * base register right by 8 bits (%ah = %rax >> 8).
343 */
344 if (lhbr)
345 *rval = val >> 8;
346 else
347 *rval = val;
348 return (error);
349 }
350
351 static int
vie_write_bytereg(struct vcpu * vcpu,struct vie * vie,uint8_t byte)352 vie_write_bytereg(struct vcpu *vcpu, struct vie *vie, uint8_t byte)
353 {
354 uint64_t origval, val, mask;
355 int error, lhbr;
356 enum vm_reg_name reg;
357
358 vie_calc_bytereg(vie, ®, &lhbr);
359 error = vm_get_register(vcpu, reg, &origval);
360 if (error == 0) {
361 val = byte;
362 mask = 0xff;
363 if (lhbr) {
364 /*
365 * Shift left by 8 to store 'byte' in a legacy high
366 * byte register.
367 */
368 val <<= 8;
369 mask <<= 8;
370 }
371 val |= origval & ~mask;
372 error = vm_set_register(vcpu, reg, val);
373 }
374 return (error);
375 }
376
377 int
vie_update_register(struct vcpu * vcpu,enum vm_reg_name reg,uint64_t val,int size)378 vie_update_register(struct vcpu *vcpu, enum vm_reg_name reg,
379 uint64_t val, int size)
380 {
381 int error;
382 uint64_t origval;
383
384 switch (size) {
385 case 1:
386 case 2:
387 error = vie_read_register(vcpu, reg, &origval);
388 if (error)
389 return (error);
390 val &= size2mask[size];
391 val |= origval & ~size2mask[size];
392 break;
393 case 4:
394 val &= 0xffffffffUL;
395 break;
396 case 8:
397 break;
398 default:
399 return (EINVAL);
400 }
401
402 error = vm_set_register(vcpu, reg, val);
403 return (error);
404 }
405
406 #define RFLAGS_STATUS_BITS (PSL_C | PSL_PF | PSL_AF | PSL_Z | PSL_N | PSL_V)
407
408 /*
409 * Return the status flags that would result from doing (x - y).
410 */
411 #define GETCC(sz) \
412 static u_long \
413 getcc##sz(uint##sz##_t x, uint##sz##_t y) \
414 { \
415 u_long rflags; \
416 \
417 __asm __volatile("sub %2,%1; pushfq; popq %0" : \
418 "=r" (rflags), "+r" (x) : "m" (y)); \
419 return (rflags); \
420 } struct __hack
421
422 GETCC(8);
423 GETCC(16);
424 GETCC(32);
425 GETCC(64);
426
427 static u_long
getcc(int opsize,uint64_t x,uint64_t y)428 getcc(int opsize, uint64_t x, uint64_t y)
429 {
430 KASSERT(opsize == 1 || opsize == 2 || opsize == 4 || opsize == 8,
431 ("getcc: invalid operand size %d", opsize));
432
433 if (opsize == 1)
434 return (getcc8(x, y));
435 else if (opsize == 2)
436 return (getcc16(x, y));
437 else if (opsize == 4)
438 return (getcc32(x, y));
439 else
440 return (getcc64(x, y));
441 }
442
443 /*
444 * Macro creation of functions getaddflags{8,16,32,64}
445 */
446 #define GETADDFLAGS(sz) \
447 static u_long \
448 getaddflags##sz(uint##sz##_t x, uint##sz##_t y) \
449 { \
450 u_long rflags; \
451 \
452 __asm __volatile("add %2,%1; pushfq; popq %0" : \
453 "=r" (rflags), "+r" (x) : "m" (y)); \
454 return (rflags); \
455 } struct __hack
456
457 GETADDFLAGS(8);
458 GETADDFLAGS(16);
459 GETADDFLAGS(32);
460 GETADDFLAGS(64);
461
462 static u_long
getaddflags(int opsize,uint64_t x,uint64_t y)463 getaddflags(int opsize, uint64_t x, uint64_t y)
464 {
465 KASSERT(opsize == 1 || opsize == 2 || opsize == 4 || opsize == 8,
466 ("getaddflags: invalid operand size %d", opsize));
467
468 if (opsize == 1)
469 return (getaddflags8(x, y));
470 else if (opsize == 2)
471 return (getaddflags16(x, y));
472 else if (opsize == 4)
473 return (getaddflags32(x, y));
474 else
475 return (getaddflags64(x, y));
476 }
477
478 /*
479 * Return the status flags that would result from doing (x & y).
480 */
481 #define GETANDFLAGS(sz) \
482 static u_long \
483 getandflags##sz(uint##sz##_t x, uint##sz##_t y) \
484 { \
485 u_long rflags; \
486 \
487 __asm __volatile("and %2,%1; pushfq; popq %0" : \
488 "=r" (rflags), "+r" (x) : "m" (y)); \
489 return (rflags); \
490 } struct __hack
491
492 GETANDFLAGS(8);
493 GETANDFLAGS(16);
494 GETANDFLAGS(32);
495 GETANDFLAGS(64);
496
497 static u_long
getandflags(int opsize,uint64_t x,uint64_t y)498 getandflags(int opsize, uint64_t x, uint64_t y)
499 {
500 KASSERT(opsize == 1 || opsize == 2 || opsize == 4 || opsize == 8,
501 ("getandflags: invalid operand size %d", opsize));
502
503 if (opsize == 1)
504 return (getandflags8(x, y));
505 else if (opsize == 2)
506 return (getandflags16(x, y));
507 else if (opsize == 4)
508 return (getandflags32(x, y));
509 else
510 return (getandflags64(x, y));
511 }
512
513 static int
emulate_mov(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)514 emulate_mov(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
515 mem_region_read_t memread, mem_region_write_t memwrite, void *arg)
516 {
517 int error, size;
518 enum vm_reg_name reg;
519 uint8_t byte;
520 uint64_t val;
521
522 size = vie->opsize;
523 error = EINVAL;
524
525 switch (vie->op.op_byte) {
526 case 0x88:
527 /*
528 * MOV byte from reg (ModRM:reg) to mem (ModRM:r/m)
529 * 88/r: mov r/m8, r8
530 * REX + 88/r: mov r/m8, r8 (%ah, %ch, %dh, %bh not available)
531 */
532 size = 1; /* override for byte operation */
533 error = vie_read_bytereg(vcpu, vie, &byte);
534 if (error == 0)
535 error = memwrite(vcpu, gpa, byte, size, arg);
536 break;
537 case 0x89:
538 /*
539 * MOV from reg (ModRM:reg) to mem (ModRM:r/m)
540 * 89/r: mov r/m16, r16
541 * 89/r: mov r/m32, r32
542 * REX.W + 89/r mov r/m64, r64
543 */
544 reg = gpr_map[vie->reg];
545 error = vie_read_register(vcpu, reg, &val);
546 if (error == 0) {
547 val &= size2mask[size];
548 error = memwrite(vcpu, gpa, val, size, arg);
549 }
550 break;
551 case 0x8A:
552 /*
553 * MOV byte from mem (ModRM:r/m) to reg (ModRM:reg)
554 * 8A/r: mov r8, r/m8
555 * REX + 8A/r: mov r8, r/m8
556 */
557 size = 1; /* override for byte operation */
558 error = memread(vcpu, gpa, &val, size, arg);
559 if (error == 0)
560 error = vie_write_bytereg(vcpu, vie, val);
561 break;
562 case 0x8B:
563 /*
564 * MOV from mem (ModRM:r/m) to reg (ModRM:reg)
565 * 8B/r: mov r16, r/m16
566 * 8B/r: mov r32, r/m32
567 * REX.W 8B/r: mov r64, r/m64
568 */
569 error = memread(vcpu, gpa, &val, size, arg);
570 if (error == 0) {
571 reg = gpr_map[vie->reg];
572 error = vie_update_register(vcpu, reg, val, size);
573 }
574 break;
575 case 0xA1:
576 /*
577 * MOV from seg:moffset to AX/EAX/RAX
578 * A1: mov AX, moffs16
579 * A1: mov EAX, moffs32
580 * REX.W + A1: mov RAX, moffs64
581 */
582 error = memread(vcpu, gpa, &val, size, arg);
583 if (error == 0) {
584 reg = VM_REG_GUEST_RAX;
585 error = vie_update_register(vcpu, reg, val, size);
586 }
587 break;
588 case 0xA3:
589 /*
590 * MOV from AX/EAX/RAX to seg:moffset
591 * A3: mov moffs16, AX
592 * A3: mov moffs32, EAX
593 * REX.W + A3: mov moffs64, RAX
594 */
595 error = vie_read_register(vcpu, VM_REG_GUEST_RAX, &val);
596 if (error == 0) {
597 val &= size2mask[size];
598 error = memwrite(vcpu, gpa, val, size, arg);
599 }
600 break;
601 case 0xC6:
602 /*
603 * MOV from imm8 to mem (ModRM:r/m)
604 * C6/0 mov r/m8, imm8
605 * REX + C6/0 mov r/m8, imm8
606 */
607 size = 1; /* override for byte operation */
608 error = memwrite(vcpu, gpa, vie->immediate, size, arg);
609 break;
610 case 0xC7:
611 /*
612 * MOV from imm16/imm32 to mem (ModRM:r/m)
613 * C7/0 mov r/m16, imm16
614 * C7/0 mov r/m32, imm32
615 * REX.W + C7/0 mov r/m64, imm32 (sign-extended to 64-bits)
616 */
617 val = vie->immediate & size2mask[size];
618 error = memwrite(vcpu, gpa, val, size, arg);
619 break;
620 default:
621 break;
622 }
623
624 return (error);
625 }
626
627 static int
emulate_movx(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * arg)628 emulate_movx(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
629 mem_region_read_t memread, mem_region_write_t memwrite __unused, void *arg)
630 {
631 int error, size;
632 enum vm_reg_name reg;
633 uint64_t val;
634
635 size = vie->opsize;
636 error = EINVAL;
637
638 switch (vie->op.op_byte) {
639 case 0xB6:
640 /*
641 * MOV and zero extend byte from mem (ModRM:r/m) to
642 * reg (ModRM:reg).
643 *
644 * 0F B6/r movzx r16, r/m8
645 * 0F B6/r movzx r32, r/m8
646 * REX.W + 0F B6/r movzx r64, r/m8
647 */
648
649 /* get the first operand */
650 error = memread(vcpu, gpa, &val, 1, arg);
651 if (error)
652 break;
653
654 /* get the second operand */
655 reg = gpr_map[vie->reg];
656
657 /* zero-extend byte */
658 val = (uint8_t)val;
659
660 /* write the result */
661 error = vie_update_register(vcpu, reg, val, size);
662 break;
663 case 0xB7:
664 /*
665 * MOV and zero extend word from mem (ModRM:r/m) to
666 * reg (ModRM:reg).
667 *
668 * 0F B7/r movzx r32, r/m16
669 * REX.W + 0F B7/r movzx r64, r/m16
670 */
671 error = memread(vcpu, gpa, &val, 2, arg);
672 if (error)
673 return (error);
674
675 reg = gpr_map[vie->reg];
676
677 /* zero-extend word */
678 val = (uint16_t)val;
679
680 error = vie_update_register(vcpu, reg, val, size);
681 break;
682 case 0xBE:
683 /*
684 * MOV and sign extend byte from mem (ModRM:r/m) to
685 * reg (ModRM:reg).
686 *
687 * 0F BE/r movsx r16, r/m8
688 * 0F BE/r movsx r32, r/m8
689 * REX.W + 0F BE/r movsx r64, r/m8
690 */
691
692 /* get the first operand */
693 error = memread(vcpu, gpa, &val, 1, arg);
694 if (error)
695 break;
696
697 /* get the second operand */
698 reg = gpr_map[vie->reg];
699
700 /* sign extend byte */
701 val = (int8_t)val;
702
703 /* write the result */
704 error = vie_update_register(vcpu, reg, val, size);
705 break;
706 default:
707 break;
708 }
709 return (error);
710 }
711
712 /*
713 * Helper function to calculate and validate a linear address.
714 */
715 static int
get_gla(struct vcpu * vcpu,struct vie * vie __unused,struct vm_guest_paging * paging,int opsize,int addrsize,int prot,enum vm_reg_name seg,enum vm_reg_name gpr,uint64_t * gla,int * fault)716 get_gla(struct vcpu *vcpu, struct vie *vie __unused,
717 struct vm_guest_paging *paging, int opsize, int addrsize, int prot,
718 enum vm_reg_name seg, enum vm_reg_name gpr, uint64_t *gla, int *fault)
719 {
720 struct seg_desc desc;
721 uint64_t cr0, val, rflags;
722 int error __diagused;
723
724 error = vie_read_register(vcpu, VM_REG_GUEST_CR0, &cr0);
725 KASSERT(error == 0, ("%s: error %d getting cr0", __func__, error));
726
727 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
728 KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
729
730 error = vm_get_seg_desc(vcpu, seg, &desc);
731 KASSERT(error == 0, ("%s: error %d getting segment descriptor %d",
732 __func__, error, seg));
733
734 error = vie_read_register(vcpu, gpr, &val);
735 KASSERT(error == 0, ("%s: error %d getting register %d", __func__,
736 error, gpr));
737
738 if (vie_calculate_gla(paging->cpu_mode, seg, &desc, val, opsize,
739 addrsize, prot, gla)) {
740 if (seg == VM_REG_GUEST_SS)
741 vm_inject_ss(vcpu, 0);
742 else
743 vm_inject_gp(vcpu);
744 goto guest_fault;
745 }
746
747 if (vie_canonical_check(paging->cpu_mode, *gla)) {
748 if (seg == VM_REG_GUEST_SS)
749 vm_inject_ss(vcpu, 0);
750 else
751 vm_inject_gp(vcpu);
752 goto guest_fault;
753 }
754
755 if (vie_alignment_check(paging->cpl, opsize, cr0, rflags, *gla)) {
756 vm_inject_ac(vcpu, 0);
757 goto guest_fault;
758 }
759
760 *fault = 0;
761 return (0);
762
763 guest_fault:
764 *fault = 1;
765 return (0);
766 }
767
768 static int
emulate_movs(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,struct vm_guest_paging * paging,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)769 emulate_movs(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
770 struct vm_guest_paging *paging, mem_region_read_t memread,
771 mem_region_write_t memwrite, void *arg)
772 {
773 #ifdef _KERNEL
774 struct vm_copyinfo copyinfo[2];
775 #else
776 struct iovec copyinfo[2];
777 #endif
778 uint64_t dstaddr, srcaddr, dstgpa, srcgpa, val;
779 uint64_t rcx, rdi, rsi, rflags;
780 int error, fault, opsize, seg, repeat;
781
782 opsize = (vie->op.op_byte == 0xA4) ? 1 : vie->opsize;
783 val = 0;
784 error = 0;
785
786 /*
787 * XXX although the MOVS instruction is only supposed to be used with
788 * the "rep" prefix some guests like FreeBSD will use "repnz" instead.
789 *
790 * Empirically the "repnz" prefix has identical behavior to "rep"
791 * and the zero flag does not make a difference.
792 */
793 repeat = vie->repz_present | vie->repnz_present;
794
795 if (repeat) {
796 error = vie_read_register(vcpu, VM_REG_GUEST_RCX, &rcx);
797 KASSERT(!error, ("%s: error %d getting rcx", __func__, error));
798
799 /*
800 * The count register is %rcx, %ecx or %cx depending on the
801 * address size of the instruction.
802 */
803 if ((rcx & vie_size2mask(vie->addrsize)) == 0) {
804 error = 0;
805 goto done;
806 }
807 }
808
809 /*
810 * Source Destination Comments
811 * --------------------------------------------
812 * (1) memory memory n/a
813 * (2) memory mmio emulated
814 * (3) mmio memory emulated
815 * (4) mmio mmio emulated
816 *
817 * At this point we don't have sufficient information to distinguish
818 * between (2), (3) and (4). We use 'vm_copy_setup()' to tease this
819 * out because it will succeed only when operating on regular memory.
820 *
821 * XXX the emulation doesn't properly handle the case where 'gpa'
822 * is straddling the boundary between the normal memory and MMIO.
823 */
824
825 seg = vie->segment_override ? vie->segment_register : VM_REG_GUEST_DS;
826 error = get_gla(vcpu, vie, paging, opsize, vie->addrsize,
827 PROT_READ, seg, VM_REG_GUEST_RSI, &srcaddr, &fault);
828 if (error || fault)
829 goto done;
830
831 error = vm_copy_setup(vcpu, paging, srcaddr, opsize, PROT_READ,
832 copyinfo, nitems(copyinfo), &fault);
833 if (error == 0) {
834 if (fault)
835 goto done; /* Resume guest to handle fault */
836
837 /*
838 * case (2): read from system memory and write to mmio.
839 */
840 vm_copyin(copyinfo, &val, opsize);
841 vm_copy_teardown(copyinfo, nitems(copyinfo));
842 error = memwrite(vcpu, gpa, val, opsize, arg);
843 if (error)
844 goto done;
845 } else {
846 /*
847 * 'vm_copy_setup()' is expected to fail for cases (3) and (4)
848 * if 'srcaddr' is in the mmio space.
849 */
850
851 error = get_gla(vcpu, vie, paging, opsize, vie->addrsize,
852 PROT_WRITE, VM_REG_GUEST_ES, VM_REG_GUEST_RDI, &dstaddr,
853 &fault);
854 if (error || fault)
855 goto done;
856
857 error = vm_copy_setup(vcpu, paging, dstaddr, opsize,
858 PROT_WRITE, copyinfo, nitems(copyinfo), &fault);
859 if (error == 0) {
860 if (fault)
861 goto done; /* Resume guest to handle fault */
862
863 /*
864 * case (3): read from MMIO and write to system memory.
865 *
866 * A MMIO read can have side-effects so we
867 * commit to it only after vm_copy_setup() is
868 * successful. If a page-fault needs to be
869 * injected into the guest then it will happen
870 * before the MMIO read is attempted.
871 */
872 error = memread(vcpu, gpa, &val, opsize, arg);
873 if (error)
874 goto done;
875
876 vm_copyout(&val, copyinfo, opsize);
877 vm_copy_teardown(copyinfo, nitems(copyinfo));
878 } else {
879 /*
880 * Case (4): read from and write to mmio.
881 *
882 * Commit to the MMIO read/write (with potential
883 * side-effects) only after we are sure that the
884 * instruction is not going to be restarted due
885 * to address translation faults.
886 */
887 error = vm_gla2gpa(vcpu, paging, srcaddr,
888 PROT_READ, &srcgpa, &fault);
889 if (error || fault)
890 goto done;
891
892 error = vm_gla2gpa(vcpu, paging, dstaddr,
893 PROT_WRITE, &dstgpa, &fault);
894 if (error || fault)
895 goto done;
896
897 error = memread(vcpu, srcgpa, &val, opsize, arg);
898 if (error)
899 goto done;
900
901 error = memwrite(vcpu, dstgpa, val, opsize, arg);
902 if (error)
903 goto done;
904 }
905 }
906
907 error = vie_read_register(vcpu, VM_REG_GUEST_RSI, &rsi);
908 KASSERT(error == 0, ("%s: error %d getting rsi", __func__, error));
909
910 error = vie_read_register(vcpu, VM_REG_GUEST_RDI, &rdi);
911 KASSERT(error == 0, ("%s: error %d getting rdi", __func__, error));
912
913 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
914 KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
915
916 if (rflags & PSL_D) {
917 rsi -= opsize;
918 rdi -= opsize;
919 } else {
920 rsi += opsize;
921 rdi += opsize;
922 }
923
924 error = vie_update_register(vcpu, VM_REG_GUEST_RSI, rsi,
925 vie->addrsize);
926 KASSERT(error == 0, ("%s: error %d updating rsi", __func__, error));
927
928 error = vie_update_register(vcpu, VM_REG_GUEST_RDI, rdi,
929 vie->addrsize);
930 KASSERT(error == 0, ("%s: error %d updating rdi", __func__, error));
931
932 if (repeat) {
933 rcx = rcx - 1;
934 error = vie_update_register(vcpu, VM_REG_GUEST_RCX,
935 rcx, vie->addrsize);
936 KASSERT(!error, ("%s: error %d updating rcx", __func__, error));
937
938 /*
939 * Repeat the instruction if the count register is not zero.
940 */
941 if ((rcx & vie_size2mask(vie->addrsize)) != 0)
942 vm_restart_instruction(vcpu);
943 }
944 done:
945 KASSERT(error == 0 || error == EFAULT, ("%s: unexpected error %d",
946 __func__, error));
947 return (error);
948 }
949
950 static int
emulate_stos(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,struct vm_guest_paging * paging __unused,mem_region_read_t memread __unused,mem_region_write_t memwrite,void * arg)951 emulate_stos(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
952 struct vm_guest_paging *paging __unused, mem_region_read_t memread __unused,
953 mem_region_write_t memwrite, void *arg)
954 {
955 int error, opsize, repeat;
956 uint64_t val;
957 uint64_t rcx, rdi, rflags;
958
959 opsize = (vie->op.op_byte == 0xAA) ? 1 : vie->opsize;
960 repeat = vie->repz_present | vie->repnz_present;
961
962 if (repeat) {
963 error = vie_read_register(vcpu, VM_REG_GUEST_RCX, &rcx);
964 KASSERT(!error, ("%s: error %d getting rcx", __func__, error));
965
966 /*
967 * The count register is %rcx, %ecx or %cx depending on the
968 * address size of the instruction.
969 */
970 if ((rcx & vie_size2mask(vie->addrsize)) == 0)
971 return (0);
972 }
973
974 error = vie_read_register(vcpu, VM_REG_GUEST_RAX, &val);
975 KASSERT(!error, ("%s: error %d getting rax", __func__, error));
976
977 error = memwrite(vcpu, gpa, val, opsize, arg);
978 if (error)
979 return (error);
980
981 error = vie_read_register(vcpu, VM_REG_GUEST_RDI, &rdi);
982 KASSERT(error == 0, ("%s: error %d getting rdi", __func__, error));
983
984 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
985 KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
986
987 if (rflags & PSL_D)
988 rdi -= opsize;
989 else
990 rdi += opsize;
991
992 error = vie_update_register(vcpu, VM_REG_GUEST_RDI, rdi,
993 vie->addrsize);
994 KASSERT(error == 0, ("%s: error %d updating rdi", __func__, error));
995
996 if (repeat) {
997 rcx = rcx - 1;
998 error = vie_update_register(vcpu, VM_REG_GUEST_RCX,
999 rcx, vie->addrsize);
1000 KASSERT(!error, ("%s: error %d updating rcx", __func__, error));
1001
1002 /*
1003 * Repeat the instruction if the count register is not zero.
1004 */
1005 if ((rcx & vie_size2mask(vie->addrsize)) != 0)
1006 vm_restart_instruction(vcpu);
1007 }
1008
1009 return (0);
1010 }
1011
1012 static int
emulate_and(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)1013 emulate_and(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1014 mem_region_read_t memread, mem_region_write_t memwrite, void *arg)
1015 {
1016 int error, size;
1017 enum vm_reg_name reg;
1018 uint64_t result, rflags, rflags2, val1, val2;
1019
1020 size = vie->opsize;
1021 error = EINVAL;
1022
1023 switch (vie->op.op_byte) {
1024 case 0x23:
1025 /*
1026 * AND reg (ModRM:reg) and mem (ModRM:r/m) and store the
1027 * result in reg.
1028 *
1029 * 23/r and r16, r/m16
1030 * 23/r and r32, r/m32
1031 * REX.W + 23/r and r64, r/m64
1032 */
1033
1034 /* get the first operand */
1035 reg = gpr_map[vie->reg];
1036 error = vie_read_register(vcpu, reg, &val1);
1037 if (error)
1038 break;
1039
1040 /* get the second operand */
1041 error = memread(vcpu, gpa, &val2, size, arg);
1042 if (error)
1043 break;
1044
1045 /* perform the operation and write the result */
1046 result = val1 & val2;
1047 error = vie_update_register(vcpu, reg, result, size);
1048 break;
1049 case 0x81:
1050 case 0x83:
1051 /*
1052 * AND mem (ModRM:r/m) with immediate and store the
1053 * result in mem.
1054 *
1055 * 81 /4 and r/m16, imm16
1056 * 81 /4 and r/m32, imm32
1057 * REX.W + 81 /4 and r/m64, imm32 sign-extended to 64
1058 *
1059 * 83 /4 and r/m16, imm8 sign-extended to 16
1060 * 83 /4 and r/m32, imm8 sign-extended to 32
1061 * REX.W + 83/4 and r/m64, imm8 sign-extended to 64
1062 */
1063
1064 /* get the first operand */
1065 error = memread(vcpu, gpa, &val1, size, arg);
1066 if (error)
1067 break;
1068
1069 /*
1070 * perform the operation with the pre-fetched immediate
1071 * operand and write the result
1072 */
1073 result = val1 & vie->immediate;
1074 error = memwrite(vcpu, gpa, result, size, arg);
1075 break;
1076 default:
1077 break;
1078 }
1079 if (error)
1080 return (error);
1081
1082 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1083 if (error)
1084 return (error);
1085
1086 /*
1087 * OF and CF are cleared; the SF, ZF and PF flags are set according
1088 * to the result; AF is undefined.
1089 *
1090 * The updated status flags are obtained by subtracting 0 from 'result'.
1091 */
1092 rflags2 = getcc(size, result, 0);
1093 rflags &= ~RFLAGS_STATUS_BITS;
1094 rflags |= rflags2 & (PSL_PF | PSL_Z | PSL_N);
1095
1096 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS, rflags, 8);
1097 return (error);
1098 }
1099
1100 static int
emulate_or(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)1101 emulate_or(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1102 mem_region_read_t memread, mem_region_write_t memwrite, void *arg)
1103 {
1104 int error, size;
1105 enum vm_reg_name reg;
1106 uint64_t result, rflags, rflags2, val1, val2;
1107
1108 size = vie->opsize;
1109 error = EINVAL;
1110
1111 switch (vie->op.op_byte) {
1112 case 0x0B:
1113 /*
1114 * OR reg (ModRM:reg) and mem (ModRM:r/m) and store the
1115 * result in reg.
1116 *
1117 * 0b/r or r16, r/m16
1118 * 0b/r or r32, r/m32
1119 * REX.W + 0b/r or r64, r/m64
1120 */
1121
1122 /* get the first operand */
1123 reg = gpr_map[vie->reg];
1124 error = vie_read_register(vcpu, reg, &val1);
1125 if (error)
1126 break;
1127
1128 /* get the second operand */
1129 error = memread(vcpu, gpa, &val2, size, arg);
1130 if (error)
1131 break;
1132
1133 /* perform the operation and write the result */
1134 result = val1 | val2;
1135 error = vie_update_register(vcpu, reg, result, size);
1136 break;
1137 case 0x81:
1138 case 0x83:
1139 /*
1140 * OR mem (ModRM:r/m) with immediate and store the
1141 * result in mem.
1142 *
1143 * 81 /1 or r/m16, imm16
1144 * 81 /1 or r/m32, imm32
1145 * REX.W + 81 /1 or r/m64, imm32 sign-extended to 64
1146 *
1147 * 83 /1 or r/m16, imm8 sign-extended to 16
1148 * 83 /1 or r/m32, imm8 sign-extended to 32
1149 * REX.W + 83/1 or r/m64, imm8 sign-extended to 64
1150 */
1151
1152 /* get the first operand */
1153 error = memread(vcpu, gpa, &val1, size, arg);
1154 if (error)
1155 break;
1156
1157 /*
1158 * perform the operation with the pre-fetched immediate
1159 * operand and write the result
1160 */
1161 result = val1 | vie->immediate;
1162 error = memwrite(vcpu, gpa, result, size, arg);
1163 break;
1164 default:
1165 break;
1166 }
1167 if (error)
1168 return (error);
1169
1170 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1171 if (error)
1172 return (error);
1173
1174 /*
1175 * OF and CF are cleared; the SF, ZF and PF flags are set according
1176 * to the result; AF is undefined.
1177 *
1178 * The updated status flags are obtained by subtracting 0 from 'result'.
1179 */
1180 rflags2 = getcc(size, result, 0);
1181 rflags &= ~RFLAGS_STATUS_BITS;
1182 rflags |= rflags2 & (PSL_PF | PSL_Z | PSL_N);
1183
1184 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS, rflags, 8);
1185 return (error);
1186 }
1187
1188 static int
emulate_cmp(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * arg)1189 emulate_cmp(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1190 mem_region_read_t memread, mem_region_write_t memwrite __unused, void *arg)
1191 {
1192 int error, size;
1193 uint64_t regop, memop, op1, op2, rflags, rflags2;
1194 enum vm_reg_name reg;
1195
1196 size = vie->opsize;
1197 switch (vie->op.op_byte) {
1198 case 0x39:
1199 case 0x3B:
1200 /*
1201 * 39/r CMP r/m16, r16
1202 * 39/r CMP r/m32, r32
1203 * REX.W 39/r CMP r/m64, r64
1204 *
1205 * 3B/r CMP r16, r/m16
1206 * 3B/r CMP r32, r/m32
1207 * REX.W + 3B/r CMP r64, r/m64
1208 *
1209 * Compare the first operand with the second operand and
1210 * set status flags in EFLAGS register. The comparison is
1211 * performed by subtracting the second operand from the first
1212 * operand and then setting the status flags.
1213 */
1214
1215 /* Get the register operand */
1216 reg = gpr_map[vie->reg];
1217 error = vie_read_register(vcpu, reg, ®op);
1218 if (error)
1219 return (error);
1220
1221 /* Get the memory operand */
1222 error = memread(vcpu, gpa, &memop, size, arg);
1223 if (error)
1224 return (error);
1225
1226 if (vie->op.op_byte == 0x3B) {
1227 op1 = regop;
1228 op2 = memop;
1229 } else {
1230 op1 = memop;
1231 op2 = regop;
1232 }
1233 rflags2 = getcc(size, op1, op2);
1234 break;
1235 case 0x80:
1236 case 0x81:
1237 case 0x83:
1238 /*
1239 * 80 /7 cmp r/m8, imm8
1240 * REX + 80 /7 cmp r/m8, imm8
1241 *
1242 * 81 /7 cmp r/m16, imm16
1243 * 81 /7 cmp r/m32, imm32
1244 * REX.W + 81 /7 cmp r/m64, imm32 sign-extended to 64
1245 *
1246 * 83 /7 cmp r/m16, imm8 sign-extended to 16
1247 * 83 /7 cmp r/m32, imm8 sign-extended to 32
1248 * REX.W + 83 /7 cmp r/m64, imm8 sign-extended to 64
1249 *
1250 * Compare mem (ModRM:r/m) with immediate and set
1251 * status flags according to the results. The
1252 * comparison is performed by subtracting the
1253 * immediate from the first operand and then setting
1254 * the status flags.
1255 *
1256 */
1257 if (vie->op.op_byte == 0x80)
1258 size = 1;
1259
1260 /* get the first operand */
1261 error = memread(vcpu, gpa, &op1, size, arg);
1262 if (error)
1263 return (error);
1264
1265 rflags2 = getcc(size, op1, vie->immediate);
1266 break;
1267 default:
1268 return (EINVAL);
1269 }
1270 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1271 if (error)
1272 return (error);
1273 rflags &= ~RFLAGS_STATUS_BITS;
1274 rflags |= rflags2 & RFLAGS_STATUS_BITS;
1275
1276 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS, rflags, 8);
1277 return (error);
1278 }
1279
1280 static int
emulate_test(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * arg)1281 emulate_test(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1282 mem_region_read_t memread, mem_region_write_t memwrite __unused, void *arg)
1283 {
1284 int error, size;
1285 uint64_t op1, rflags, rflags2;
1286
1287 size = vie->opsize;
1288 error = EINVAL;
1289
1290 switch (vie->op.op_byte) {
1291 case 0xF6:
1292 /*
1293 * F6 /0 test r/m8, imm8
1294 */
1295 size = 1; /* override for byte operation */
1296 /* FALLTHROUGH */
1297 case 0xF7:
1298 /*
1299 * F7 /0 test r/m16, imm16
1300 * F7 /0 test r/m32, imm32
1301 * REX.W + F7 /0 test r/m64, imm32 sign-extended to 64
1302 *
1303 * Test mem (ModRM:r/m) with immediate and set status
1304 * flags according to the results. The comparison is
1305 * performed by anding the immediate from the first
1306 * operand and then setting the status flags.
1307 */
1308 if ((vie->reg & 7) != 0)
1309 return (EINVAL);
1310
1311 error = memread(vcpu, gpa, &op1, size, arg);
1312 if (error)
1313 return (error);
1314
1315 rflags2 = getandflags(size, op1, vie->immediate);
1316 break;
1317 default:
1318 return (EINVAL);
1319 }
1320 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1321 if (error)
1322 return (error);
1323
1324 /*
1325 * OF and CF are cleared; the SF, ZF and PF flags are set according
1326 * to the result; AF is undefined.
1327 */
1328 rflags &= ~RFLAGS_STATUS_BITS;
1329 rflags |= rflags2 & (PSL_PF | PSL_Z | PSL_N);
1330
1331 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS, rflags, 8);
1332 return (error);
1333 }
1334
1335 static int
emulate_bextr(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,struct vm_guest_paging * paging,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * arg)1336 emulate_bextr(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1337 struct vm_guest_paging *paging, mem_region_read_t memread,
1338 mem_region_write_t memwrite __unused, void *arg)
1339 {
1340 uint64_t src1, src2, dst, rflags;
1341 unsigned start, len, size;
1342 int error;
1343
1344 size = vie->opsize;
1345 error = EINVAL;
1346
1347 /*
1348 * VEX.LZ.0F38.W0 F7 /r BEXTR r32a, r/m32, r32b
1349 * VEX.LZ.0F38.W1 F7 /r BEXTR r64a, r/m64, r64b
1350 *
1351 * Destination operand is ModRM:reg. Source operands are ModRM:r/m and
1352 * Vex.vvvv.
1353 *
1354 * Operand size is always 32-bit if not in 64-bit mode (W1 is ignored).
1355 */
1356 if (size != 4 && paging->cpu_mode != CPU_MODE_64BIT)
1357 size = 4;
1358
1359 /*
1360 * Extracts contiguous bits from the first /source/ operand (second
1361 * operand) using an index and length specified in the second /source/
1362 * operand (third operand).
1363 */
1364 error = memread(vcpu, gpa, &src1, size, arg);
1365 if (error)
1366 return (error);
1367 error = vie_read_register(vcpu, gpr_map[vie->vex_reg], &src2);
1368 if (error)
1369 return (error);
1370 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1371 if (error)
1372 return (error);
1373
1374 start = (src2 & 0xff);
1375 len = (src2 & 0xff00) >> 8;
1376
1377 /* If no bits are extracted, the destination register is cleared. */
1378 dst = 0;
1379
1380 /* If START exceeds the operand size, no bits are extracted. */
1381 if (start > size * 8)
1382 goto done;
1383 /* Length is bounded by both the destination size and start offset. */
1384 if (start + len > size * 8)
1385 len = (size * 8) - start;
1386 if (len == 0)
1387 goto done;
1388
1389 if (start > 0)
1390 src1 = (src1 >> start);
1391 if (len < 64)
1392 src1 = src1 & ((1ull << len) - 1);
1393 dst = src1;
1394
1395 done:
1396 error = vie_update_register(vcpu, gpr_map[vie->reg], dst, size);
1397 if (error)
1398 return (error);
1399
1400 /*
1401 * AMD: OF, CF cleared; SF/AF/PF undefined; ZF set by result.
1402 * Intel: ZF is set by result; AF/SF/PF undefined; all others cleared.
1403 */
1404 rflags &= ~RFLAGS_STATUS_BITS;
1405 if (dst == 0)
1406 rflags |= PSL_Z;
1407 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS, rflags,
1408 8);
1409 return (error);
1410 }
1411
1412 static int
emulate_add(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * arg)1413 emulate_add(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1414 mem_region_read_t memread, mem_region_write_t memwrite __unused, void *arg)
1415 {
1416 int error, size;
1417 uint64_t nval, rflags, rflags2, val1, val2;
1418 enum vm_reg_name reg;
1419
1420 size = vie->opsize;
1421 error = EINVAL;
1422
1423 switch (vie->op.op_byte) {
1424 case 0x03:
1425 /*
1426 * ADD r/m to r and store the result in r
1427 *
1428 * 03/r ADD r16, r/m16
1429 * 03/r ADD r32, r/m32
1430 * REX.W + 03/r ADD r64, r/m64
1431 */
1432
1433 /* get the first operand */
1434 reg = gpr_map[vie->reg];
1435 error = vie_read_register(vcpu, reg, &val1);
1436 if (error)
1437 break;
1438
1439 /* get the second operand */
1440 error = memread(vcpu, gpa, &val2, size, arg);
1441 if (error)
1442 break;
1443
1444 /* perform the operation and write the result */
1445 nval = val1 + val2;
1446 error = vie_update_register(vcpu, reg, nval, size);
1447 break;
1448 default:
1449 break;
1450 }
1451
1452 if (!error) {
1453 rflags2 = getaddflags(size, val1, val2);
1454 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS,
1455 &rflags);
1456 if (error)
1457 return (error);
1458
1459 rflags &= ~RFLAGS_STATUS_BITS;
1460 rflags |= rflags2 & RFLAGS_STATUS_BITS;
1461 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS,
1462 rflags, 8);
1463 }
1464
1465 return (error);
1466 }
1467
1468 static int
emulate_sub(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * arg)1469 emulate_sub(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1470 mem_region_read_t memread, mem_region_write_t memwrite __unused, void *arg)
1471 {
1472 int error, size;
1473 uint64_t nval, rflags, rflags2, val1, val2;
1474 enum vm_reg_name reg;
1475
1476 size = vie->opsize;
1477 error = EINVAL;
1478
1479 switch (vie->op.op_byte) {
1480 case 0x2B:
1481 /*
1482 * SUB r/m from r and store the result in r
1483 *
1484 * 2B/r SUB r16, r/m16
1485 * 2B/r SUB r32, r/m32
1486 * REX.W + 2B/r SUB r64, r/m64
1487 */
1488
1489 /* get the first operand */
1490 reg = gpr_map[vie->reg];
1491 error = vie_read_register(vcpu, reg, &val1);
1492 if (error)
1493 break;
1494
1495 /* get the second operand */
1496 error = memread(vcpu, gpa, &val2, size, arg);
1497 if (error)
1498 break;
1499
1500 /* perform the operation and write the result */
1501 nval = val1 - val2;
1502 error = vie_update_register(vcpu, reg, nval, size);
1503 break;
1504 default:
1505 break;
1506 }
1507
1508 if (!error) {
1509 rflags2 = getcc(size, val1, val2);
1510 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS,
1511 &rflags);
1512 if (error)
1513 return (error);
1514
1515 rflags &= ~RFLAGS_STATUS_BITS;
1516 rflags |= rflags2 & RFLAGS_STATUS_BITS;
1517 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS,
1518 rflags, 8);
1519 }
1520
1521 return (error);
1522 }
1523
1524 static int
emulate_stack_op(struct vcpu * vcpu,uint64_t mmio_gpa,struct vie * vie,struct vm_guest_paging * paging,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)1525 emulate_stack_op(struct vcpu *vcpu, uint64_t mmio_gpa, struct vie *vie,
1526 struct vm_guest_paging *paging, mem_region_read_t memread,
1527 mem_region_write_t memwrite, void *arg)
1528 {
1529 #ifdef _KERNEL
1530 struct vm_copyinfo copyinfo[2];
1531 #else
1532 struct iovec copyinfo[2];
1533 #endif
1534 struct seg_desc ss_desc;
1535 uint64_t cr0, rflags, rsp, stack_gla, val;
1536 int error, fault, size, stackaddrsize, pushop;
1537
1538 val = 0;
1539 size = vie->opsize;
1540 pushop = (vie->op.op_type == VIE_OP_TYPE_PUSH) ? 1 : 0;
1541
1542 /*
1543 * From "Address-Size Attributes for Stack Accesses", Intel SDL, Vol 1
1544 */
1545 if (paging->cpu_mode == CPU_MODE_REAL) {
1546 stackaddrsize = 2;
1547 } else if (paging->cpu_mode == CPU_MODE_64BIT) {
1548 /*
1549 * "Stack Manipulation Instructions in 64-bit Mode", SDM, Vol 3
1550 * - Stack pointer size is always 64-bits.
1551 * - PUSH/POP of 32-bit values is not possible in 64-bit mode.
1552 * - 16-bit PUSH/POP is supported by using the operand size
1553 * override prefix (66H).
1554 */
1555 stackaddrsize = 8;
1556 size = vie->opsize_override ? 2 : 8;
1557 } else {
1558 /*
1559 * In protected or compatibility mode the 'B' flag in the
1560 * stack-segment descriptor determines the size of the
1561 * stack pointer.
1562 */
1563 error = vm_get_seg_desc(vcpu, VM_REG_GUEST_SS, &ss_desc);
1564 KASSERT(error == 0, ("%s: error %d getting SS descriptor",
1565 __func__, error));
1566 if (SEG_DESC_DEF32(ss_desc.access))
1567 stackaddrsize = 4;
1568 else
1569 stackaddrsize = 2;
1570 }
1571
1572 error = vie_read_register(vcpu, VM_REG_GUEST_CR0, &cr0);
1573 KASSERT(error == 0, ("%s: error %d getting cr0", __func__, error));
1574
1575 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1576 KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
1577
1578 error = vie_read_register(vcpu, VM_REG_GUEST_RSP, &rsp);
1579 KASSERT(error == 0, ("%s: error %d getting rsp", __func__, error));
1580 if (pushop) {
1581 rsp -= size;
1582 }
1583
1584 if (vie_calculate_gla(paging->cpu_mode, VM_REG_GUEST_SS, &ss_desc,
1585 rsp, size, stackaddrsize, pushop ? PROT_WRITE : PROT_READ,
1586 &stack_gla)) {
1587 vm_inject_ss(vcpu, 0);
1588 return (0);
1589 }
1590
1591 if (vie_canonical_check(paging->cpu_mode, stack_gla)) {
1592 vm_inject_ss(vcpu, 0);
1593 return (0);
1594 }
1595
1596 if (vie_alignment_check(paging->cpl, size, cr0, rflags, stack_gla)) {
1597 vm_inject_ac(vcpu, 0);
1598 return (0);
1599 }
1600
1601 error = vm_copy_setup(vcpu, paging, stack_gla, size,
1602 pushop ? PROT_WRITE : PROT_READ, copyinfo, nitems(copyinfo),
1603 &fault);
1604 if (error || fault)
1605 return (error);
1606
1607 if (pushop) {
1608 error = memread(vcpu, mmio_gpa, &val, size, arg);
1609 if (error == 0)
1610 vm_copyout(&val, copyinfo, size);
1611 } else {
1612 vm_copyin(copyinfo, &val, size);
1613 error = memwrite(vcpu, mmio_gpa, val, size, arg);
1614 rsp += size;
1615 }
1616 vm_copy_teardown(copyinfo, nitems(copyinfo));
1617
1618 if (error == 0) {
1619 error = vie_update_register(vcpu, VM_REG_GUEST_RSP, rsp,
1620 stackaddrsize);
1621 KASSERT(error == 0, ("error %d updating rsp", error));
1622 }
1623 return (error);
1624 }
1625
1626 static int
emulate_push(struct vcpu * vcpu,uint64_t mmio_gpa,struct vie * vie,struct vm_guest_paging * paging,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)1627 emulate_push(struct vcpu *vcpu, uint64_t mmio_gpa, struct vie *vie,
1628 struct vm_guest_paging *paging, mem_region_read_t memread,
1629 mem_region_write_t memwrite, void *arg)
1630 {
1631 int error;
1632
1633 /*
1634 * Table A-6, "Opcode Extensions", Intel SDM, Vol 2.
1635 *
1636 * PUSH is part of the group 5 extended opcodes and is identified
1637 * by ModRM:reg = b110.
1638 */
1639 if ((vie->reg & 7) != 6)
1640 return (EINVAL);
1641
1642 error = emulate_stack_op(vcpu, mmio_gpa, vie, paging, memread,
1643 memwrite, arg);
1644 return (error);
1645 }
1646
1647 static int
emulate_pop(struct vcpu * vcpu,uint64_t mmio_gpa,struct vie * vie,struct vm_guest_paging * paging,mem_region_read_t memread,mem_region_write_t memwrite,void * arg)1648 emulate_pop(struct vcpu *vcpu, uint64_t mmio_gpa, struct vie *vie,
1649 struct vm_guest_paging *paging, mem_region_read_t memread,
1650 mem_region_write_t memwrite, void *arg)
1651 {
1652 int error;
1653
1654 /*
1655 * Table A-6, "Opcode Extensions", Intel SDM, Vol 2.
1656 *
1657 * POP is part of the group 1A extended opcodes and is identified
1658 * by ModRM:reg = b000.
1659 */
1660 if ((vie->reg & 7) != 0)
1661 return (EINVAL);
1662
1663 error = emulate_stack_op(vcpu, mmio_gpa, vie, paging, memread,
1664 memwrite, arg);
1665 return (error);
1666 }
1667
1668 static int
emulate_group1(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,struct vm_guest_paging * paging __unused,mem_region_read_t memread,mem_region_write_t memwrite,void * memarg)1669 emulate_group1(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1670 struct vm_guest_paging *paging __unused, mem_region_read_t memread,
1671 mem_region_write_t memwrite, void *memarg)
1672 {
1673 int error;
1674
1675 switch (vie->reg & 7) {
1676 case 0x1: /* OR */
1677 error = emulate_or(vcpu, gpa, vie,
1678 memread, memwrite, memarg);
1679 break;
1680 case 0x4: /* AND */
1681 error = emulate_and(vcpu, gpa, vie,
1682 memread, memwrite, memarg);
1683 break;
1684 case 0x7: /* CMP */
1685 error = emulate_cmp(vcpu, gpa, vie,
1686 memread, memwrite, memarg);
1687 break;
1688 default:
1689 error = EINVAL;
1690 break;
1691 }
1692
1693 return (error);
1694 }
1695
1696 static int
emulate_bittest(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * memarg)1697 emulate_bittest(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1698 mem_region_read_t memread, mem_region_write_t memwrite __unused,
1699 void *memarg)
1700 {
1701 uint64_t val, rflags;
1702 int error, bitmask, bitoff;
1703
1704 /*
1705 * 0F BA is a Group 8 extended opcode.
1706 *
1707 * Currently we only emulate the 'Bit Test' instruction which is
1708 * identified by a ModR/M:reg encoding of 100b.
1709 */
1710 if ((vie->reg & 7) != 4)
1711 return (EINVAL);
1712
1713 error = vie_read_register(vcpu, VM_REG_GUEST_RFLAGS, &rflags);
1714 KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
1715
1716 error = memread(vcpu, gpa, &val, vie->opsize, memarg);
1717 if (error)
1718 return (error);
1719
1720 /*
1721 * Intel SDM, Vol 2, Table 3-2:
1722 * "Range of Bit Positions Specified by Bit Offset Operands"
1723 */
1724 bitmask = vie->opsize * 8 - 1;
1725 bitoff = vie->immediate & bitmask;
1726
1727 /* Copy the bit into the Carry flag in %rflags */
1728 if (val & (1UL << bitoff))
1729 rflags |= PSL_C;
1730 else
1731 rflags &= ~PSL_C;
1732
1733 error = vie_update_register(vcpu, VM_REG_GUEST_RFLAGS, rflags, 8);
1734 KASSERT(error == 0, ("%s: error %d updating rflags", __func__, error));
1735
1736 return (0);
1737 }
1738
1739 static int
emulate_twob_group15(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,mem_region_read_t memread,mem_region_write_t memwrite __unused,void * memarg)1740 emulate_twob_group15(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1741 mem_region_read_t memread, mem_region_write_t memwrite __unused,
1742 void *memarg)
1743 {
1744 int error;
1745 uint64_t buf;
1746
1747 switch (vie->reg & 7) {
1748 case 0x7: /* CLFLUSH, CLFLUSHOPT, and SFENCE */
1749 if (vie->mod == 0x3) {
1750 /*
1751 * SFENCE. Ignore it, VM exit provides enough
1752 * barriers on its own.
1753 */
1754 error = 0;
1755 } else {
1756 /*
1757 * CLFLUSH, CLFLUSHOPT. Only check for access
1758 * rights.
1759 */
1760 error = memread(vcpu, gpa, &buf, 1, memarg);
1761 }
1762 break;
1763 default:
1764 error = EINVAL;
1765 break;
1766 }
1767
1768 return (error);
1769 }
1770
1771 int
vmm_emulate_instruction(struct vcpu * vcpu,uint64_t gpa,struct vie * vie,struct vm_guest_paging * paging,mem_region_read_t memread,mem_region_write_t memwrite,void * memarg)1772 vmm_emulate_instruction(struct vcpu *vcpu, uint64_t gpa, struct vie *vie,
1773 struct vm_guest_paging *paging, mem_region_read_t memread,
1774 mem_region_write_t memwrite, void *memarg)
1775 {
1776 int error;
1777
1778 if (!vie->decoded)
1779 return (EINVAL);
1780
1781 switch (vie->op.op_type) {
1782 case VIE_OP_TYPE_GROUP1:
1783 error = emulate_group1(vcpu, gpa, vie, paging, memread,
1784 memwrite, memarg);
1785 break;
1786 case VIE_OP_TYPE_POP:
1787 error = emulate_pop(vcpu, gpa, vie, paging, memread,
1788 memwrite, memarg);
1789 break;
1790 case VIE_OP_TYPE_PUSH:
1791 error = emulate_push(vcpu, gpa, vie, paging, memread,
1792 memwrite, memarg);
1793 break;
1794 case VIE_OP_TYPE_CMP:
1795 error = emulate_cmp(vcpu, gpa, vie,
1796 memread, memwrite, memarg);
1797 break;
1798 case VIE_OP_TYPE_MOV:
1799 error = emulate_mov(vcpu, gpa, vie,
1800 memread, memwrite, memarg);
1801 break;
1802 case VIE_OP_TYPE_MOVSX:
1803 case VIE_OP_TYPE_MOVZX:
1804 error = emulate_movx(vcpu, gpa, vie,
1805 memread, memwrite, memarg);
1806 break;
1807 case VIE_OP_TYPE_MOVS:
1808 error = emulate_movs(vcpu, gpa, vie, paging, memread,
1809 memwrite, memarg);
1810 break;
1811 case VIE_OP_TYPE_STOS:
1812 error = emulate_stos(vcpu, gpa, vie, paging, memread,
1813 memwrite, memarg);
1814 break;
1815 case VIE_OP_TYPE_AND:
1816 error = emulate_and(vcpu, gpa, vie,
1817 memread, memwrite, memarg);
1818 break;
1819 case VIE_OP_TYPE_OR:
1820 error = emulate_or(vcpu, gpa, vie,
1821 memread, memwrite, memarg);
1822 break;
1823 case VIE_OP_TYPE_SUB:
1824 error = emulate_sub(vcpu, gpa, vie,
1825 memread, memwrite, memarg);
1826 break;
1827 case VIE_OP_TYPE_BITTEST:
1828 error = emulate_bittest(vcpu, gpa, vie,
1829 memread, memwrite, memarg);
1830 break;
1831 case VIE_OP_TYPE_TWOB_GRP15:
1832 error = emulate_twob_group15(vcpu, gpa, vie,
1833 memread, memwrite, memarg);
1834 break;
1835 case VIE_OP_TYPE_ADD:
1836 error = emulate_add(vcpu, gpa, vie, memread,
1837 memwrite, memarg);
1838 break;
1839 case VIE_OP_TYPE_TEST:
1840 error = emulate_test(vcpu, gpa, vie,
1841 memread, memwrite, memarg);
1842 break;
1843 case VIE_OP_TYPE_BEXTR:
1844 error = emulate_bextr(vcpu, gpa, vie, paging,
1845 memread, memwrite, memarg);
1846 break;
1847 default:
1848 error = EINVAL;
1849 break;
1850 }
1851
1852 return (error);
1853 }
1854
1855 int
vie_alignment_check(int cpl,int size,uint64_t cr0,uint64_t rf,uint64_t gla)1856 vie_alignment_check(int cpl, int size, uint64_t cr0, uint64_t rf, uint64_t gla)
1857 {
1858 KASSERT(size == 1 || size == 2 || size == 4 || size == 8,
1859 ("%s: invalid size %d", __func__, size));
1860 KASSERT(cpl >= 0 && cpl <= 3, ("%s: invalid cpl %d", __func__, cpl));
1861
1862 if (cpl != 3 || (cr0 & CR0_AM) == 0 || (rf & PSL_AC) == 0)
1863 return (0);
1864
1865 return ((gla & (size - 1)) ? 1 : 0);
1866 }
1867
1868 int
vie_canonical_check(enum vm_cpu_mode cpu_mode,uint64_t gla)1869 vie_canonical_check(enum vm_cpu_mode cpu_mode, uint64_t gla)
1870 {
1871 uint64_t mask;
1872
1873 if (cpu_mode != CPU_MODE_64BIT)
1874 return (0);
1875
1876 /*
1877 * The value of the bit 47 in the 'gla' should be replicated in the
1878 * most significant 16 bits.
1879 */
1880 mask = ~((1UL << 48) - 1);
1881 if (gla & (1UL << 47))
1882 return ((gla & mask) != mask);
1883 else
1884 return ((gla & mask) != 0);
1885 }
1886
1887 uint64_t
vie_size2mask(int size)1888 vie_size2mask(int size)
1889 {
1890 KASSERT(size == 1 || size == 2 || size == 4 || size == 8,
1891 ("vie_size2mask: invalid size %d", size));
1892 return (size2mask[size]);
1893 }
1894
1895 int
vie_calculate_gla(enum vm_cpu_mode cpu_mode,enum vm_reg_name seg,struct seg_desc * desc,uint64_t offset,int length,int addrsize,int prot,uint64_t * gla)1896 vie_calculate_gla(enum vm_cpu_mode cpu_mode, enum vm_reg_name seg,
1897 struct seg_desc *desc, uint64_t offset, int length, int addrsize,
1898 int prot, uint64_t *gla)
1899 {
1900 uint64_t firstoff, low_limit, high_limit, segbase;
1901 int glasize, type;
1902
1903 KASSERT(seg >= VM_REG_GUEST_ES && seg <= VM_REG_GUEST_GS,
1904 ("%s: invalid segment %d", __func__, seg));
1905 KASSERT(length == 1 || length == 2 || length == 4 || length == 8,
1906 ("%s: invalid operand size %d", __func__, length));
1907 KASSERT((prot & ~(PROT_READ | PROT_WRITE)) == 0,
1908 ("%s: invalid prot %#x", __func__, prot));
1909
1910 firstoff = offset;
1911 if (cpu_mode == CPU_MODE_64BIT) {
1912 KASSERT(addrsize == 4 || addrsize == 8, ("%s: invalid address "
1913 "size %d for cpu_mode %d", __func__, addrsize, cpu_mode));
1914 glasize = 8;
1915 } else {
1916 KASSERT(addrsize == 2 || addrsize == 4, ("%s: invalid address "
1917 "size %d for cpu mode %d", __func__, addrsize, cpu_mode));
1918 glasize = 4;
1919 /*
1920 * If the segment selector is loaded with a NULL selector
1921 * then the descriptor is unusable and attempting to use
1922 * it results in a #GP(0).
1923 */
1924 if (SEG_DESC_UNUSABLE(desc->access))
1925 return (-1);
1926
1927 /*
1928 * The processor generates a #NP exception when a segment
1929 * register is loaded with a selector that points to a
1930 * descriptor that is not present. If this was the case then
1931 * it would have been checked before the VM-exit.
1932 */
1933 KASSERT(SEG_DESC_PRESENT(desc->access),
1934 ("segment %d not present: %#x", seg, desc->access));
1935
1936 /*
1937 * The descriptor type must indicate a code/data segment.
1938 */
1939 type = SEG_DESC_TYPE(desc->access);
1940 KASSERT(type >= 16 && type <= 31, ("segment %d has invalid "
1941 "descriptor type %#x", seg, type));
1942
1943 if (prot & PROT_READ) {
1944 /* #GP on a read access to a exec-only code segment */
1945 if ((type & 0xA) == 0x8)
1946 return (-1);
1947 }
1948
1949 if (prot & PROT_WRITE) {
1950 /*
1951 * #GP on a write access to a code segment or a
1952 * read-only data segment.
1953 */
1954 if (type & 0x8) /* code segment */
1955 return (-1);
1956
1957 if ((type & 0xA) == 0) /* read-only data seg */
1958 return (-1);
1959 }
1960
1961 /*
1962 * 'desc->limit' is fully expanded taking granularity into
1963 * account.
1964 */
1965 if ((type & 0xC) == 0x4) {
1966 /* expand-down data segment */
1967 low_limit = desc->limit + 1;
1968 high_limit = SEG_DESC_DEF32(desc->access) ?
1969 0xffffffff : 0xffff;
1970 } else {
1971 /* code segment or expand-up data segment */
1972 low_limit = 0;
1973 high_limit = desc->limit;
1974 }
1975
1976 while (length > 0) {
1977 offset &= vie_size2mask(addrsize);
1978 if (offset < low_limit || offset > high_limit)
1979 return (-1);
1980 offset++;
1981 length--;
1982 }
1983 }
1984
1985 /*
1986 * In 64-bit mode all segments except %fs and %gs have a segment
1987 * base address of 0.
1988 */
1989 if (cpu_mode == CPU_MODE_64BIT && seg != VM_REG_GUEST_FS &&
1990 seg != VM_REG_GUEST_GS) {
1991 segbase = 0;
1992 } else {
1993 segbase = desc->base;
1994 }
1995
1996 /*
1997 * Truncate 'firstoff' to the effective address size before adding
1998 * it to the segment base.
1999 */
2000 firstoff &= vie_size2mask(addrsize);
2001 *gla = (segbase + firstoff) & vie_size2mask(glasize);
2002 return (0);
2003 }
2004
2005 /*
2006 * Prepare a partially decoded vie for a 2nd attempt.
2007 */
2008 void
vie_restart(struct vie * vie)2009 vie_restart(struct vie *vie)
2010 {
2011 _Static_assert(
2012 offsetof(struct vie, inst) < offsetof(struct vie, vie_startzero) &&
2013 offsetof(struct vie, num_valid) < offsetof(struct vie, vie_startzero),
2014 "restart should not erase instruction length or contents");
2015
2016 memset((char *)vie + offsetof(struct vie, vie_startzero), 0,
2017 sizeof(*vie) - offsetof(struct vie, vie_startzero));
2018
2019 vie->base_register = VM_REG_LAST;
2020 vie->index_register = VM_REG_LAST;
2021 vie->segment_register = VM_REG_LAST;
2022 }
2023
2024 void
vie_init(struct vie * vie,const char * inst_bytes,int inst_length)2025 vie_init(struct vie *vie, const char *inst_bytes, int inst_length)
2026 {
2027 KASSERT(inst_length >= 0 && inst_length <= VIE_INST_SIZE,
2028 ("%s: invalid instruction length (%d)", __func__, inst_length));
2029
2030 vie_restart(vie);
2031 memset(vie->inst, 0, sizeof(vie->inst));
2032 if (inst_length != 0)
2033 memcpy(vie->inst, inst_bytes, inst_length);
2034 vie->num_valid = inst_length;
2035 }
2036
2037 #ifdef _KERNEL
2038 static int
pf_error_code(int usermode,int prot,int rsvd,uint64_t pte)2039 pf_error_code(int usermode, int prot, int rsvd, uint64_t pte)
2040 {
2041 int error_code = 0;
2042
2043 if (pte & PG_V)
2044 error_code |= PGEX_P;
2045 if (prot & VM_PROT_WRITE)
2046 error_code |= PGEX_W;
2047 if (usermode)
2048 error_code |= PGEX_U;
2049 if (rsvd)
2050 error_code |= PGEX_RSV;
2051 if (prot & VM_PROT_EXECUTE)
2052 error_code |= PGEX_I;
2053
2054 return (error_code);
2055 }
2056
2057 static void
ptp_release(void ** cookie)2058 ptp_release(void **cookie)
2059 {
2060 if (*cookie != NULL) {
2061 vm_gpa_release(*cookie);
2062 *cookie = NULL;
2063 }
2064 }
2065
2066 static void *
ptp_hold(struct vcpu * vcpu,vm_paddr_t ptpphys,size_t len,void ** cookie)2067 ptp_hold(struct vcpu *vcpu, vm_paddr_t ptpphys, size_t len, void **cookie)
2068 {
2069 void *ptr;
2070
2071 ptp_release(cookie);
2072 ptr = vm_gpa_hold(vcpu, ptpphys, len, VM_PROT_RW, cookie);
2073 return (ptr);
2074 }
2075
2076 static int
_vm_gla2gpa(struct vcpu * vcpu,struct vm_guest_paging * paging,uint64_t gla,int prot,uint64_t * gpa,int * guest_fault,bool check_only)2077 _vm_gla2gpa(struct vcpu *vcpu, struct vm_guest_paging *paging,
2078 uint64_t gla, int prot, uint64_t *gpa, int *guest_fault, bool check_only)
2079 {
2080 int nlevels, pfcode, ptpshift, ptpindex, retval, usermode, writable;
2081 u_int retries;
2082 uint64_t *ptpbase, ptpphys, pte, pgsize;
2083 uint32_t *ptpbase32, pte32;
2084 void *cookie;
2085
2086 *guest_fault = 0;
2087
2088 usermode = (paging->cpl == 3 ? 1 : 0);
2089 writable = prot & VM_PROT_WRITE;
2090 cookie = NULL;
2091 retval = 0;
2092 retries = 0;
2093 restart:
2094 ptpphys = paging->cr3; /* root of the page tables */
2095 ptp_release(&cookie);
2096 if (retries++ > 0)
2097 maybe_yield();
2098
2099 if (vie_canonical_check(paging->cpu_mode, gla)) {
2100 /*
2101 * XXX assuming a non-stack reference otherwise a stack fault
2102 * should be generated.
2103 */
2104 if (!check_only)
2105 vm_inject_gp(vcpu);
2106 goto fault;
2107 }
2108
2109 if (paging->paging_mode == PAGING_MODE_FLAT) {
2110 *gpa = gla;
2111 goto done;
2112 }
2113
2114 if (paging->paging_mode == PAGING_MODE_32) {
2115 nlevels = 2;
2116 while (--nlevels >= 0) {
2117 /* Zero out the lower 12 bits. */
2118 ptpphys &= ~0xfff;
2119
2120 ptpbase32 = ptp_hold(vcpu, ptpphys, PAGE_SIZE,
2121 &cookie);
2122
2123 if (ptpbase32 == NULL)
2124 goto error;
2125
2126 ptpshift = PAGE_SHIFT + nlevels * 10;
2127 ptpindex = (gla >> ptpshift) & 0x3FF;
2128 pgsize = 1UL << ptpshift;
2129
2130 pte32 = ptpbase32[ptpindex];
2131
2132 if ((pte32 & PG_V) == 0 ||
2133 (usermode && (pte32 & PG_U) == 0) ||
2134 (writable && (pte32 & PG_RW) == 0)) {
2135 if (!check_only) {
2136 pfcode = pf_error_code(usermode, prot, 0,
2137 pte32);
2138 vm_inject_pf(vcpu, pfcode, gla);
2139 }
2140 goto fault;
2141 }
2142
2143 /*
2144 * Emulate the x86 MMU's management of the accessed
2145 * and dirty flags. While the accessed flag is set
2146 * at every level of the page table, the dirty flag
2147 * is only set at the last level providing the guest
2148 * physical address.
2149 */
2150 if (!check_only && (pte32 & PG_A) == 0) {
2151 if (atomic_cmpset_32(&ptpbase32[ptpindex],
2152 pte32, pte32 | PG_A) == 0) {
2153 goto restart;
2154 }
2155 }
2156
2157 /* XXX must be ignored if CR4.PSE=0 */
2158 if (nlevels > 0 && (pte32 & PG_PS) != 0)
2159 break;
2160
2161 ptpphys = pte32;
2162 }
2163
2164 /* Set the dirty bit in the page table entry if necessary */
2165 if (!check_only && writable && (pte32 & PG_M) == 0) {
2166 if (atomic_cmpset_32(&ptpbase32[ptpindex],
2167 pte32, pte32 | PG_M) == 0) {
2168 goto restart;
2169 }
2170 }
2171
2172 /* Zero out the lower 'ptpshift' bits */
2173 pte32 >>= ptpshift; pte32 <<= ptpshift;
2174 *gpa = pte32 | (gla & (pgsize - 1));
2175 goto done;
2176 }
2177
2178 if (paging->paging_mode == PAGING_MODE_PAE) {
2179 /* Zero out the lower 5 bits and the upper 32 bits */
2180 ptpphys &= 0xffffffe0UL;
2181
2182 ptpbase = ptp_hold(vcpu, ptpphys, sizeof(*ptpbase) * 4,
2183 &cookie);
2184 if (ptpbase == NULL)
2185 goto error;
2186
2187 ptpindex = (gla >> 30) & 0x3;
2188
2189 pte = ptpbase[ptpindex];
2190
2191 if ((pte & PG_V) == 0) {
2192 if (!check_only) {
2193 pfcode = pf_error_code(usermode, prot, 0, pte);
2194 vm_inject_pf(vcpu, pfcode, gla);
2195 }
2196 goto fault;
2197 }
2198
2199 ptpphys = pte;
2200
2201 nlevels = 2;
2202 } else if (paging->paging_mode == PAGING_MODE_64_LA57) {
2203 nlevels = 5;
2204 } else {
2205 nlevels = 4;
2206 }
2207
2208 while (--nlevels >= 0) {
2209 /* Zero out the lower 12 bits and the upper 12 bits */
2210 ptpphys >>= 12; ptpphys <<= 24; ptpphys >>= 12;
2211
2212 ptpbase = ptp_hold(vcpu, ptpphys, PAGE_SIZE, &cookie);
2213 if (ptpbase == NULL)
2214 goto error;
2215
2216 ptpshift = PAGE_SHIFT + nlevels * 9;
2217 ptpindex = (gla >> ptpshift) & 0x1FF;
2218 pgsize = 1UL << ptpshift;
2219
2220 pte = ptpbase[ptpindex];
2221
2222 if ((pte & PG_V) == 0 ||
2223 (usermode && (pte & PG_U) == 0) ||
2224 (writable && (pte & PG_RW) == 0)) {
2225 if (!check_only) {
2226 pfcode = pf_error_code(usermode, prot, 0, pte);
2227 vm_inject_pf(vcpu, pfcode, gla);
2228 }
2229 goto fault;
2230 }
2231
2232 /* Set the accessed bit in the page table entry */
2233 if (!check_only && (pte & PG_A) == 0) {
2234 if (atomic_cmpset_64(&ptpbase[ptpindex],
2235 pte, pte | PG_A) == 0) {
2236 goto restart;
2237 }
2238 }
2239
2240 if (nlevels > 0 && (pte & PG_PS) != 0) {
2241 if (pgsize > 1 * GB) {
2242 if (!check_only) {
2243 pfcode = pf_error_code(usermode, prot, 1,
2244 pte);
2245 vm_inject_pf(vcpu, pfcode, gla);
2246 }
2247 goto fault;
2248 }
2249 break;
2250 }
2251
2252 ptpphys = pte;
2253 }
2254
2255 /* Set the dirty bit in the page table entry if necessary */
2256 if (!check_only && writable && (pte & PG_M) == 0) {
2257 if (atomic_cmpset_64(&ptpbase[ptpindex], pte, pte | PG_M) == 0)
2258 goto restart;
2259 }
2260
2261 /* Zero out the lower 'ptpshift' bits and the upper 12 bits */
2262 pte >>= ptpshift; pte <<= (ptpshift + 12); pte >>= 12;
2263 *gpa = pte | (gla & (pgsize - 1));
2264 done:
2265 ptp_release(&cookie);
2266 KASSERT(retval == 0 || retval == EFAULT, ("%s: unexpected retval %d",
2267 __func__, retval));
2268 return (retval);
2269 error:
2270 retval = EFAULT;
2271 goto done;
2272 fault:
2273 *guest_fault = 1;
2274 goto done;
2275 }
2276
2277 int
vm_gla2gpa(struct vcpu * vcpu,struct vm_guest_paging * paging,uint64_t gla,int prot,uint64_t * gpa,int * guest_fault)2278 vm_gla2gpa(struct vcpu *vcpu, struct vm_guest_paging *paging,
2279 uint64_t gla, int prot, uint64_t *gpa, int *guest_fault)
2280 {
2281
2282 return (_vm_gla2gpa(vcpu, paging, gla, prot, gpa, guest_fault,
2283 false));
2284 }
2285
2286 int
vm_gla2gpa_nofault(struct vcpu * vcpu,struct vm_guest_paging * paging,uint64_t gla,int prot,uint64_t * gpa,int * guest_fault)2287 vm_gla2gpa_nofault(struct vcpu *vcpu, struct vm_guest_paging *paging,
2288 uint64_t gla, int prot, uint64_t *gpa, int *guest_fault)
2289 {
2290
2291 return (_vm_gla2gpa(vcpu, paging, gla, prot, gpa, guest_fault,
2292 true));
2293 }
2294
2295 int
vmm_fetch_instruction(struct vcpu * vcpu,struct vm_guest_paging * paging,uint64_t rip,int inst_length,struct vie * vie,int * faultptr)2296 vmm_fetch_instruction(struct vcpu *vcpu, struct vm_guest_paging *paging,
2297 uint64_t rip, int inst_length, struct vie *vie, int *faultptr)
2298 {
2299 struct vm_copyinfo copyinfo[2];
2300 int error, prot;
2301
2302 if (inst_length > VIE_INST_SIZE)
2303 panic("vmm_fetch_instruction: invalid length %d", inst_length);
2304
2305 prot = PROT_READ | PROT_EXEC;
2306 error = vm_copy_setup(vcpu, paging, rip, inst_length, prot,
2307 copyinfo, nitems(copyinfo), faultptr);
2308 if (error || *faultptr)
2309 return (error);
2310
2311 vm_copyin(copyinfo, vie->inst, inst_length);
2312 vm_copy_teardown(copyinfo, nitems(copyinfo));
2313 vie->num_valid = inst_length;
2314 return (0);
2315 }
2316 #endif /* _KERNEL */
2317
2318 static int
vie_peek(struct vie * vie,uint8_t * x)2319 vie_peek(struct vie *vie, uint8_t *x)
2320 {
2321
2322 if (vie->num_processed < vie->num_valid) {
2323 *x = vie->inst[vie->num_processed];
2324 return (0);
2325 } else
2326 return (-1);
2327 }
2328
2329 static void
vie_advance(struct vie * vie)2330 vie_advance(struct vie *vie)
2331 {
2332
2333 vie->num_processed++;
2334 }
2335
2336 static bool
segment_override(uint8_t x,int * seg)2337 segment_override(uint8_t x, int *seg)
2338 {
2339
2340 switch (x) {
2341 case 0x2E:
2342 *seg = VM_REG_GUEST_CS;
2343 break;
2344 case 0x36:
2345 *seg = VM_REG_GUEST_SS;
2346 break;
2347 case 0x3E:
2348 *seg = VM_REG_GUEST_DS;
2349 break;
2350 case 0x26:
2351 *seg = VM_REG_GUEST_ES;
2352 break;
2353 case 0x64:
2354 *seg = VM_REG_GUEST_FS;
2355 break;
2356 case 0x65:
2357 *seg = VM_REG_GUEST_GS;
2358 break;
2359 default:
2360 return (false);
2361 }
2362 return (true);
2363 }
2364
2365 static int
decode_prefixes(struct vie * vie,enum vm_cpu_mode cpu_mode,int cs_d)2366 decode_prefixes(struct vie *vie, enum vm_cpu_mode cpu_mode, int cs_d)
2367 {
2368 uint8_t x;
2369
2370 while (1) {
2371 if (vie_peek(vie, &x))
2372 return (-1);
2373
2374 if (x == 0x66)
2375 vie->opsize_override = 1;
2376 else if (x == 0x67)
2377 vie->addrsize_override = 1;
2378 else if (x == 0xF3)
2379 vie->repz_present = 1;
2380 else if (x == 0xF2)
2381 vie->repnz_present = 1;
2382 else if (segment_override(x, &vie->segment_register))
2383 vie->segment_override = 1;
2384 else
2385 break;
2386
2387 vie_advance(vie);
2388 }
2389
2390 /*
2391 * From section 2.2.1, "REX Prefixes", Intel SDM Vol 2:
2392 * - Only one REX prefix is allowed per instruction.
2393 * - The REX prefix must immediately precede the opcode byte or the
2394 * escape opcode byte.
2395 * - If an instruction has a mandatory prefix (0x66, 0xF2 or 0xF3)
2396 * the mandatory prefix must come before the REX prefix.
2397 */
2398 if (cpu_mode == CPU_MODE_64BIT && x >= 0x40 && x <= 0x4F) {
2399 vie->rex_present = 1;
2400 vie->rex_w = x & 0x8 ? 1 : 0;
2401 vie->rex_r = x & 0x4 ? 1 : 0;
2402 vie->rex_x = x & 0x2 ? 1 : 0;
2403 vie->rex_b = x & 0x1 ? 1 : 0;
2404 vie_advance(vie);
2405 }
2406
2407 /*
2408 * § 2.3.5, "The VEX Prefix", SDM Vol 2.
2409 */
2410 if ((cpu_mode == CPU_MODE_64BIT || cpu_mode == CPU_MODE_COMPATIBILITY)
2411 && x == 0xC4) {
2412 const struct vie_op *optab;
2413
2414 /* 3-byte VEX prefix. */
2415 vie->vex_present = 1;
2416
2417 vie_advance(vie);
2418 if (vie_peek(vie, &x))
2419 return (-1);
2420
2421 /*
2422 * 2nd byte: [R', X', B', mmmmm[4:0]]. Bits are inverted
2423 * relative to REX encoding.
2424 */
2425 vie->rex_r = x & 0x80 ? 0 : 1;
2426 vie->rex_x = x & 0x40 ? 0 : 1;
2427 vie->rex_b = x & 0x20 ? 0 : 1;
2428
2429 switch (x & 0x1F) {
2430 case 0x2:
2431 /* 0F 38. */
2432 optab = three_byte_opcodes_0f38;
2433 break;
2434 case 0x1:
2435 /* 0F class - nothing handled here yet. */
2436 /* FALLTHROUGH */
2437 case 0x3:
2438 /* 0F 3A class - nothing handled here yet. */
2439 /* FALLTHROUGH */
2440 default:
2441 /* Reserved (#UD). */
2442 return (-1);
2443 }
2444
2445 vie_advance(vie);
2446 if (vie_peek(vie, &x))
2447 return (-1);
2448
2449 /* 3rd byte: [W, vvvv[6:3], L, pp[1:0]]. */
2450 vie->rex_w = x & 0x80 ? 1 : 0;
2451
2452 vie->vex_reg = ((~(unsigned)x & 0x78u) >> 3);
2453 vie->vex_l = !!(x & 0x4);
2454 vie->vex_pp = (x & 0x3);
2455
2456 /* PP: 1=66 2=F3 3=F2 prefixes. */
2457 switch (vie->vex_pp) {
2458 case 0x1:
2459 vie->opsize_override = 1;
2460 break;
2461 case 0x2:
2462 vie->repz_present = 1;
2463 break;
2464 case 0x3:
2465 vie->repnz_present = 1;
2466 break;
2467 }
2468
2469 vie_advance(vie);
2470
2471 /* Opcode, sans literal prefix prefix. */
2472 if (vie_peek(vie, &x))
2473 return (-1);
2474
2475 vie->op = optab[x];
2476 if (vie->op.op_type == VIE_OP_TYPE_NONE)
2477 return (-1);
2478
2479 vie_advance(vie);
2480 }
2481
2482 /*
2483 * Section "Operand-Size And Address-Size Attributes", Intel SDM, Vol 1
2484 */
2485 if (cpu_mode == CPU_MODE_64BIT) {
2486 /*
2487 * Default address size is 64-bits and default operand size
2488 * is 32-bits.
2489 */
2490 vie->addrsize = vie->addrsize_override ? 4 : 8;
2491 if (vie->rex_w)
2492 vie->opsize = 8;
2493 else if (vie->opsize_override)
2494 vie->opsize = 2;
2495 else
2496 vie->opsize = 4;
2497 } else if (cs_d) {
2498 /* Default address and operand sizes are 32-bits */
2499 vie->addrsize = vie->addrsize_override ? 2 : 4;
2500 vie->opsize = vie->opsize_override ? 2 : 4;
2501 } else {
2502 /* Default address and operand sizes are 16-bits */
2503 vie->addrsize = vie->addrsize_override ? 4 : 2;
2504 vie->opsize = vie->opsize_override ? 4 : 2;
2505 }
2506 return (0);
2507 }
2508
2509 static int
decode_two_byte_opcode(struct vie * vie)2510 decode_two_byte_opcode(struct vie *vie)
2511 {
2512 uint8_t x;
2513
2514 if (vie_peek(vie, &x))
2515 return (-1);
2516
2517 vie->op = two_byte_opcodes[x];
2518
2519 if (vie->op.op_type == VIE_OP_TYPE_NONE)
2520 return (-1);
2521
2522 vie_advance(vie);
2523 return (0);
2524 }
2525
2526 static int
decode_opcode(struct vie * vie)2527 decode_opcode(struct vie *vie)
2528 {
2529 uint8_t x;
2530
2531 if (vie_peek(vie, &x))
2532 return (-1);
2533
2534 /* Already did this via VEX prefix. */
2535 if (vie->op.op_type != VIE_OP_TYPE_NONE)
2536 return (0);
2537
2538 vie->op = one_byte_opcodes[x];
2539
2540 if (vie->op.op_type == VIE_OP_TYPE_NONE)
2541 return (-1);
2542
2543 vie_advance(vie);
2544
2545 if (vie->op.op_type == VIE_OP_TYPE_TWO_BYTE)
2546 return (decode_two_byte_opcode(vie));
2547
2548 return (0);
2549 }
2550
2551 static int
decode_modrm(struct vie * vie,enum vm_cpu_mode cpu_mode)2552 decode_modrm(struct vie *vie, enum vm_cpu_mode cpu_mode)
2553 {
2554 uint8_t x;
2555
2556 if (vie->op.op_flags & VIE_OP_F_NO_MODRM)
2557 return (0);
2558
2559 if (cpu_mode == CPU_MODE_REAL)
2560 return (-1);
2561
2562 if (vie_peek(vie, &x))
2563 return (-1);
2564
2565 vie->mod = (x >> 6) & 0x3;
2566 vie->rm = (x >> 0) & 0x7;
2567 vie->reg = (x >> 3) & 0x7;
2568
2569 /*
2570 * A direct addressing mode makes no sense in the context of an EPT
2571 * fault. There has to be a memory access involved to cause the
2572 * EPT fault.
2573 */
2574 if (vie->mod == VIE_MOD_DIRECT)
2575 return (-1);
2576
2577 if ((vie->mod == VIE_MOD_INDIRECT && vie->rm == VIE_RM_DISP32) ||
2578 (vie->mod != VIE_MOD_DIRECT && vie->rm == VIE_RM_SIB)) {
2579 /*
2580 * Table 2-5: Special Cases of REX Encodings
2581 *
2582 * mod=0, r/m=5 is used in the compatibility mode to
2583 * indicate a disp32 without a base register.
2584 *
2585 * mod!=3, r/m=4 is used in the compatibility mode to
2586 * indicate that the SIB byte is present.
2587 *
2588 * The 'b' bit in the REX prefix is don't care in
2589 * this case.
2590 */
2591 } else {
2592 vie->rm |= (vie->rex_b << 3);
2593 }
2594
2595 vie->reg |= (vie->rex_r << 3);
2596
2597 /* SIB */
2598 if (vie->mod != VIE_MOD_DIRECT && vie->rm == VIE_RM_SIB)
2599 goto done;
2600
2601 vie->base_register = gpr_map[vie->rm];
2602
2603 switch (vie->mod) {
2604 case VIE_MOD_INDIRECT_DISP8:
2605 vie->disp_bytes = 1;
2606 break;
2607 case VIE_MOD_INDIRECT_DISP32:
2608 vie->disp_bytes = 4;
2609 break;
2610 case VIE_MOD_INDIRECT:
2611 if (vie->rm == VIE_RM_DISP32) {
2612 vie->disp_bytes = 4;
2613 /*
2614 * Table 2-7. RIP-Relative Addressing
2615 *
2616 * In 64-bit mode mod=00 r/m=101 implies [rip] + disp32
2617 * whereas in compatibility mode it just implies disp32.
2618 */
2619
2620 if (cpu_mode == CPU_MODE_64BIT)
2621 vie->base_register = VM_REG_GUEST_RIP;
2622 else
2623 vie->base_register = VM_REG_LAST;
2624 }
2625 break;
2626 }
2627
2628 done:
2629 vie_advance(vie);
2630
2631 return (0);
2632 }
2633
2634 static int
decode_sib(struct vie * vie)2635 decode_sib(struct vie *vie)
2636 {
2637 uint8_t x;
2638
2639 /* Proceed only if SIB byte is present */
2640 if (vie->mod == VIE_MOD_DIRECT || vie->rm != VIE_RM_SIB)
2641 return (0);
2642
2643 if (vie_peek(vie, &x))
2644 return (-1);
2645
2646 /* De-construct the SIB byte */
2647 vie->ss = (x >> 6) & 0x3;
2648 vie->index = (x >> 3) & 0x7;
2649 vie->base = (x >> 0) & 0x7;
2650
2651 /* Apply the REX prefix modifiers */
2652 vie->index |= vie->rex_x << 3;
2653 vie->base |= vie->rex_b << 3;
2654
2655 switch (vie->mod) {
2656 case VIE_MOD_INDIRECT_DISP8:
2657 vie->disp_bytes = 1;
2658 break;
2659 case VIE_MOD_INDIRECT_DISP32:
2660 vie->disp_bytes = 4;
2661 break;
2662 }
2663
2664 if (vie->mod == VIE_MOD_INDIRECT &&
2665 (vie->base == 5 || vie->base == 13)) {
2666 /*
2667 * Special case when base register is unused if mod = 0
2668 * and base = %rbp or %r13.
2669 *
2670 * Documented in:
2671 * Table 2-3: 32-bit Addressing Forms with the SIB Byte
2672 * Table 2-5: Special Cases of REX Encodings
2673 */
2674 vie->disp_bytes = 4;
2675 } else {
2676 vie->base_register = gpr_map[vie->base];
2677 }
2678
2679 /*
2680 * All encodings of 'index' are valid except for %rsp (4).
2681 *
2682 * Documented in:
2683 * Table 2-3: 32-bit Addressing Forms with the SIB Byte
2684 * Table 2-5: Special Cases of REX Encodings
2685 */
2686 if (vie->index != 4)
2687 vie->index_register = gpr_map[vie->index];
2688
2689 /* 'scale' makes sense only in the context of an index register */
2690 if (vie->index_register < VM_REG_LAST)
2691 vie->scale = 1 << vie->ss;
2692
2693 vie_advance(vie);
2694
2695 return (0);
2696 }
2697
2698 static int
decode_displacement(struct vie * vie)2699 decode_displacement(struct vie *vie)
2700 {
2701 int n, i;
2702 uint8_t x;
2703
2704 union {
2705 char buf[4];
2706 int8_t signed8;
2707 int32_t signed32;
2708 } u;
2709
2710 if ((n = vie->disp_bytes) == 0)
2711 return (0);
2712
2713 if (n != 1 && n != 4)
2714 panic("decode_displacement: invalid disp_bytes %d", n);
2715
2716 for (i = 0; i < n; i++) {
2717 if (vie_peek(vie, &x))
2718 return (-1);
2719
2720 u.buf[i] = x;
2721 vie_advance(vie);
2722 }
2723
2724 if (n == 1)
2725 vie->displacement = u.signed8; /* sign-extended */
2726 else
2727 vie->displacement = u.signed32; /* sign-extended */
2728
2729 return (0);
2730 }
2731
2732 static int
decode_immediate(struct vie * vie)2733 decode_immediate(struct vie *vie)
2734 {
2735 int i, n;
2736 uint8_t x;
2737 union {
2738 char buf[4];
2739 int8_t signed8;
2740 int16_t signed16;
2741 int32_t signed32;
2742 } u;
2743
2744 /* Figure out immediate operand size (if any) */
2745 if (vie->op.op_flags & VIE_OP_F_IMM) {
2746 /*
2747 * Section 2.2.1.5 "Immediates", Intel SDM:
2748 * In 64-bit mode the typical size of immediate operands
2749 * remains 32-bits. When the operand size if 64-bits, the
2750 * processor sign-extends all immediates to 64-bits prior
2751 * to their use.
2752 */
2753 if (vie->opsize == 4 || vie->opsize == 8)
2754 vie->imm_bytes = 4;
2755 else
2756 vie->imm_bytes = 2;
2757 } else if (vie->op.op_flags & VIE_OP_F_IMM8) {
2758 vie->imm_bytes = 1;
2759 }
2760
2761 if ((n = vie->imm_bytes) == 0)
2762 return (0);
2763
2764 KASSERT(n == 1 || n == 2 || n == 4,
2765 ("%s: invalid number of immediate bytes: %d", __func__, n));
2766
2767 for (i = 0; i < n; i++) {
2768 if (vie_peek(vie, &x))
2769 return (-1);
2770
2771 u.buf[i] = x;
2772 vie_advance(vie);
2773 }
2774
2775 /* sign-extend the immediate value before use */
2776 if (n == 1)
2777 vie->immediate = u.signed8;
2778 else if (n == 2)
2779 vie->immediate = u.signed16;
2780 else
2781 vie->immediate = u.signed32;
2782
2783 return (0);
2784 }
2785
2786 static int
decode_moffset(struct vie * vie)2787 decode_moffset(struct vie *vie)
2788 {
2789 int i, n;
2790 uint8_t x;
2791 union {
2792 char buf[8];
2793 uint64_t u64;
2794 } u;
2795
2796 if ((vie->op.op_flags & VIE_OP_F_MOFFSET) == 0)
2797 return (0);
2798
2799 /*
2800 * Section 2.2.1.4, "Direct Memory-Offset MOVs", Intel SDM:
2801 * The memory offset size follows the address-size of the instruction.
2802 */
2803 n = vie->addrsize;
2804 KASSERT(n == 2 || n == 4 || n == 8, ("invalid moffset bytes: %d", n));
2805
2806 u.u64 = 0;
2807 for (i = 0; i < n; i++) {
2808 if (vie_peek(vie, &x))
2809 return (-1);
2810
2811 u.buf[i] = x;
2812 vie_advance(vie);
2813 }
2814 vie->displacement = u.u64;
2815 return (0);
2816 }
2817
2818 #ifdef _KERNEL
2819 /*
2820 * Verify that the 'guest linear address' provided as collateral of the nested
2821 * page table fault matches with our instruction decoding.
2822 */
2823 static int
verify_gla(struct vcpu * vcpu,uint64_t gla,struct vie * vie,enum vm_cpu_mode cpu_mode)2824 verify_gla(struct vcpu *vcpu, uint64_t gla, struct vie *vie,
2825 enum vm_cpu_mode cpu_mode)
2826 {
2827 int error;
2828 uint64_t base, segbase, idx, gla2;
2829 enum vm_reg_name seg;
2830 struct seg_desc desc;
2831
2832 /* Skip 'gla' verification */
2833 if (gla == VIE_INVALID_GLA)
2834 return (0);
2835
2836 base = 0;
2837 if (vie->base_register != VM_REG_LAST) {
2838 error = vm_get_register(vcpu, vie->base_register, &base);
2839 if (error) {
2840 printf("verify_gla: error %d getting base reg %d\n",
2841 error, vie->base_register);
2842 return (-1);
2843 }
2844
2845 /*
2846 * RIP-relative addressing starts from the following
2847 * instruction
2848 */
2849 if (vie->base_register == VM_REG_GUEST_RIP)
2850 base += vie->num_processed;
2851 }
2852
2853 idx = 0;
2854 if (vie->index_register != VM_REG_LAST) {
2855 error = vm_get_register(vcpu, vie->index_register, &idx);
2856 if (error) {
2857 printf("verify_gla: error %d getting index reg %d\n",
2858 error, vie->index_register);
2859 return (-1);
2860 }
2861 }
2862
2863 /*
2864 * From "Specifying a Segment Selector", Intel SDM, Vol 1
2865 *
2866 * In 64-bit mode, segmentation is generally (but not
2867 * completely) disabled. The exceptions are the FS and GS
2868 * segments.
2869 *
2870 * In legacy IA-32 mode, when the ESP or EBP register is used
2871 * as the base, the SS segment is the default segment. For
2872 * other data references, except when relative to stack or
2873 * string destination the DS segment is the default. These
2874 * can be overridden to allow other segments to be accessed.
2875 */
2876 if (vie->segment_override)
2877 seg = vie->segment_register;
2878 else if (vie->base_register == VM_REG_GUEST_RSP ||
2879 vie->base_register == VM_REG_GUEST_RBP)
2880 seg = VM_REG_GUEST_SS;
2881 else
2882 seg = VM_REG_GUEST_DS;
2883 if (cpu_mode == CPU_MODE_64BIT && seg != VM_REG_GUEST_FS &&
2884 seg != VM_REG_GUEST_GS) {
2885 segbase = 0;
2886 } else {
2887 error = vm_get_seg_desc(vcpu, seg, &desc);
2888 if (error) {
2889 printf("verify_gla: error %d getting segment"
2890 " descriptor %d", error,
2891 vie->segment_register);
2892 return (-1);
2893 }
2894 segbase = desc.base;
2895 }
2896
2897 gla2 = segbase + base + vie->scale * idx + vie->displacement;
2898 gla2 &= size2mask[vie->addrsize];
2899 if (gla != gla2) {
2900 printf("verify_gla mismatch: segbase(0x%0lx)"
2901 "base(0x%0lx), scale(%d), index(0x%0lx), "
2902 "disp(0x%0lx), gla(0x%0lx), gla2(0x%0lx)\n",
2903 segbase, base, vie->scale, idx, vie->displacement,
2904 gla, gla2);
2905 return (-1);
2906 }
2907
2908 return (0);
2909 }
2910 #endif /* _KERNEL */
2911
2912 int
2913 #ifdef _KERNEL
vmm_decode_instruction(struct vcpu * vcpu,uint64_t gla,enum vm_cpu_mode cpu_mode,int cs_d,struct vie * vie)2914 vmm_decode_instruction(struct vcpu *vcpu, uint64_t gla,
2915 enum vm_cpu_mode cpu_mode, int cs_d, struct vie *vie)
2916 #else
2917 vmm_decode_instruction(enum vm_cpu_mode cpu_mode, int cs_d, struct vie *vie)
2918 #endif
2919 {
2920
2921 if (decode_prefixes(vie, cpu_mode, cs_d))
2922 return (-1);
2923
2924 if (decode_opcode(vie))
2925 return (-1);
2926
2927 if (decode_modrm(vie, cpu_mode))
2928 return (-1);
2929
2930 if (decode_sib(vie))
2931 return (-1);
2932
2933 if (decode_displacement(vie))
2934 return (-1);
2935
2936 if (decode_immediate(vie))
2937 return (-1);
2938
2939 if (decode_moffset(vie))
2940 return (-1);
2941
2942 #ifdef _KERNEL
2943 if ((vie->op.op_flags & VIE_OP_F_NO_GLA_VERIFICATION) == 0) {
2944 if (verify_gla(vcpu, gla, vie, cpu_mode))
2945 return (-1);
2946 }
2947 #endif
2948
2949 vie->decoded = 1; /* success */
2950
2951 return (0);
2952 }
2953