hollalinux
/
linux-sg2042
mirror of https://github.com/sophgo/linux-riscv.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'x86-paravirt-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 paravirt updates from Ingo Molnar:
 "A handful of paravirt patching code enhancements to make it more
  robust against patching failures, and related cleanups and not so
  related cleanups - by Thomas Gleixner and myself"

* 'x86-paravirt-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/paravirt: Rename paravirt_patch_site::instrtype to paravirt_patch_site::type
  x86/paravirt: Standardize 'insn_buff' variable names
  x86/paravirt: Match paravirt patchlet field definition ordering to initialization ordering
  x86/paravirt: Replace the paravirt patch asm magic
  x86/paravirt: Unify the 32/64 bit paravirt patching code
  x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
  x86/paravirt: Detect over-sized patching bugs in paravirt_patch_insns()
  x86/paravirt: Remove bogus extern declarations
This commit is contained in:
Linus Torvalds 2019-07-08 17:34:44 -07:00
parent 3431a940bb 46938cc8ab
commit da17702385
11 changed files with 213 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
arch/x86/events/intel/ds.c
View File

@ -337,7 +337,7 @@ static int alloc_pebs_buffer(int cpu)
struct debug_store *ds = hwev->ds; struct debug_store *ds = hwev->ds;
size_t bsiz = x86_pmu.pebs_buffer_size; size_t bsiz = x86_pmu.pebs_buffer_size;
int max, node = cpu_to_node(cpu); int max, node = cpu_to_node(cpu);
void *buffer, *ibuffer, *cea; void *buffer, *insn_buff, *cea;
if (!x86_pmu.pebs) if (!x86_pmu.pebs)
return 0; return 0;
@ -351,12 +351,12 @@ static int alloc_pebs_buffer(int cpu)
* buffer then. * buffer then.
*/ */
if (x86_pmu.intel_cap.pebs_format < 2) { if (x86_pmu.intel_cap.pebs_format < 2) {
ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node); insn_buff = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
if (!ibuffer) { if (!insn_buff) {
dsfree_pages(buffer, bsiz); dsfree_pages(buffer, bsiz);
return -ENOMEM; return -ENOMEM;
} }
per_cpu(insn_buffer, cpu) = ibuffer; per_cpu(insn_buffer, cpu) = insn_buff;
} }
hwev->ds_pebs_vaddr = buffer; hwev->ds_pebs_vaddr = buffer;
/* Update the cpu entry area mapping */ /* Update the cpu entry area mapping */

21
arch/x86/include/asm/paravirt_types.h
View File

@ -88,7 +88,7 @@ struct pv_init_ops {
* the number of bytes of code generated, as we nop pad the * the number of bytes of code generated, as we nop pad the
* rest in generic code. * rest in generic code.
*/ */
unsigned (*patch)(u8 type, void *insnbuf, unsigned (*patch)(u8 type, void *insn_buff,
unsigned long addr, unsigned len); unsigned long addr, unsigned len);
} __no_randomize_layout; } __no_randomize_layout;
@ -370,18 +370,11 @@ extern struct paravirt_patch_template pv_ops;
/* Simple instruction patching code. */ /* Simple instruction patching code. */
#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t" #define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t"
#define DEF_NATIVE(ops, name, code) \ unsigned paravirt_patch_ident_64(void *insn_buff, unsigned len);
__visible extern const char start_##ops##_##name[], end_##ops##_##name[]; \ unsigned paravirt_patch_default(u8 type, void *insn_buff, unsigned long addr, unsigned len);
asm(NATIVE_LABEL("start_", ops, name) code NATIVE_LABEL("end_", ops, name)) unsigned paravirt_patch_insns(void *insn_buff, unsigned len, const char *start, const char *end);
unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len); unsigned native_patch(u8 type, void *insn_buff, unsigned long addr, unsigned len);
unsigned paravirt_patch_default(u8 type, void *insnbuf,
unsigned long addr, unsigned len);
unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
const char *start, const char *end);
unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len);
int paravirt_disable_iospace(void); int paravirt_disable_iospace(void);
@ -679,8 +672,8 @@ u64 _paravirt_ident_64(u64);
/* These all sit in the .parainstructions section to tell us what to patch. */ /* These all sit in the .parainstructions section to tell us what to patch. */
struct paravirt_patch_site { struct paravirt_patch_site {
u8 *instr; /* original instructions */ u8 *instr; /* original instructions */
u8 instrtype; /* type of this instruction */ u8 type; /* type of this instruction */
u8 len; /* length of original instruction */ u8 len; /* length of original instruction */
}; };

4
arch/x86/kernel/Makefile
View File

@ -30,7 +30,7 @@ KASAN_SANITIZE_paravirt.o := n
OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y
OBJECT_FILES_NON_STANDARD_test_nx.o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y
OBJECT_FILES_NON_STANDARD_paravirt_patch_$(BITS).o := y OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y
ifdef CONFIG_FRAME_POINTER ifdef CONFIG_FRAME_POINTER
OBJECT_FILES_NON_STANDARD_ftrace_$(BITS).o := y OBJECT_FILES_NON_STANDARD_ftrace_$(BITS).o := y
@ -112,7 +112,7 @@ obj-$(CONFIG_AMD_NB) += amd_nb.o
obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o
obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch.o
obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o
obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o

53
arch/x86/kernel/alternative.c
View File

@ -278,7 +278,7 @@ static inline bool is_jmp(const u8 opcode)
} }
static void __init_or_module static void __init_or_module
recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf) recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insn_buff)
{ {
u8 *next_rip, *tgt_rip; u8 *next_rip, *tgt_rip;
s32 n_dspl, o_dspl; s32 n_dspl, o_dspl;
@ -287,7 +287,7 @@ recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf)
if (a->replacementlen != 5) if (a->replacementlen != 5)
return; return;
o_dspl = *(s32 *)(insnbuf + 1); o_dspl = *(s32 *)(insn_buff + 1);
/* next_rip of the replacement JMP */ /* next_rip of the replacement JMP */
next_rip = repl_insn + a->replacementlen; next_rip = repl_insn + a->replacementlen;
@ -313,9 +313,9 @@ recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf)
two_byte_jmp: two_byte_jmp:
n_dspl -= 2; n_dspl -= 2;
insnbuf[0] = 0xeb; insn_buff[0] = 0xeb;
insnbuf[1] = (s8)n_dspl; insn_buff[1] = (s8)n_dspl;
add_nops(insnbuf + 2, 3); add_nops(insn_buff + 2, 3);
repl_len = 2; repl_len = 2;
goto done; goto done;
@ -323,8 +323,8 @@ two_byte_jmp:
five_byte_jmp: five_byte_jmp:
n_dspl -= 5; n_dspl -= 5;
insnbuf[0] = 0xe9; insn_buff[0] = 0xe9;
*(s32 *)&insnbuf[1] = n_dspl; *(s32 *)&insn_buff[1] = n_dspl;
repl_len = 5; repl_len = 5;
@ -371,7 +371,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
{ {
struct alt_instr *a; struct alt_instr *a;
u8 *instr, *replacement; u8 *instr, *replacement;
u8 insnbuf[MAX_PATCH_LEN]; u8 insn_buff[MAX_PATCH_LEN];
DPRINTK("alt table %px, -> %px", start, end); DPRINTK("alt table %px, -> %px", start, end);
/* /*
@ -384,11 +384,11 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
* order. * order.
*/ */
for (a = start; a < end; a++) { for (a = start; a < end; a++) {
int insnbuf_sz = 0; int insn_buff_sz = 0;
instr = (u8 *)&a->instr_offset + a->instr_offset; instr = (u8 *)&a->instr_offset + a->instr_offset;
replacement = (u8 *)&a->repl_offset + a->repl_offset; replacement = (u8 *)&a->repl_offset + a->repl_offset;
BUG_ON(a->instrlen > sizeof(insnbuf)); BUG_ON(a->instrlen > sizeof(insn_buff));
BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
if (!boot_cpu_has(a->cpuid)) { if (!boot_cpu_has(a->cpuid)) {
if (a->padlen > 1) if (a->padlen > 1)
@ -406,8 +406,8 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr); DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement); DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
memcpy(insnbuf, replacement, a->replacementlen); memcpy(insn_buff, replacement, a->replacementlen);
insnbuf_sz = a->replacementlen; insn_buff_sz = a->replacementlen;
/* /*
* 0xe8 is a relative jump; fix the offset. * 0xe8 is a relative jump; fix the offset.
@ -415,24 +415,24 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
* Instruction length is checked before the opcode to avoid * Instruction length is checked before the opcode to avoid
* accessing uninitialized bytes for zero-length replacements. * accessing uninitialized bytes for zero-length replacements.
*/ */
if (a->replacementlen == 5 && *insnbuf == 0xe8) { if (a->replacementlen == 5 && *insn_buff == 0xe8) {
*(s32 *)(insnbuf + 1) += replacement - instr; *(s32 *)(insn_buff + 1) += replacement - instr;
DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx", DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
*(s32 *)(insnbuf + 1), *(s32 *)(insn_buff + 1),
(unsigned long)instr + *(s32 *)(insnbuf + 1) + 5); (unsigned long)instr + *(s32 *)(insn_buff + 1) + 5);
} }
if (a->replacementlen && is_jmp(replacement[0])) if (a->replacementlen && is_jmp(replacement[0]))
recompute_jump(a, instr, replacement, insnbuf); recompute_jump(a, instr, replacement, insn_buff);
if (a->instrlen > a->replacementlen) { if (a->instrlen > a->replacementlen) {
add_nops(insnbuf + a->replacementlen, add_nops(insn_buff + a->replacementlen,
a->instrlen - a->replacementlen); a->instrlen - a->replacementlen);
insnbuf_sz += a->instrlen - a->replacementlen; insn_buff_sz += a->instrlen - a->replacementlen;
} }
DUMP_BYTES(insnbuf, insnbuf_sz, "%px: final_insn: ", instr); DUMP_BYTES(insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
text_poke_early(instr, insnbuf, insnbuf_sz); text_poke_early(instr, insn_buff, insn_buff_sz);
} }
} }
@ -594,22 +594,21 @@ void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
struct paravirt_patch_site *end) struct paravirt_patch_site *end)
{ {
struct paravirt_patch_site *p; struct paravirt_patch_site *p;
char insnbuf[MAX_PATCH_LEN]; char insn_buff[MAX_PATCH_LEN];
for (p = start; p < end; p++) { for (p = start; p < end; p++) {
unsigned int used; unsigned int used;
BUG_ON(p->len > MAX_PATCH_LEN); BUG_ON(p->len > MAX_PATCH_LEN);
/* prep the buffer with the original instructions */ /* prep the buffer with the original instructions */
memcpy(insnbuf, p->instr, p->len); memcpy(insn_buff, p->instr, p->len);
used = pv_ops.init.patch(p->instrtype, insnbuf, used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);
(unsigned long)p->instr, p->len);
BUG_ON(used > p->len); BUG_ON(used > p->len);
/* Pad the rest with nops */ /* Pad the rest with nops */
add_nops(insnbuf + used, p->len - used); add_nops(insn_buff + used, p->len - used);
text_poke_early(p->instr, insnbuf, p->len); text_poke_early(p->instr, insn_buff, p->len);
} }
} }
extern struct paravirt_patch_site __start_parainstructions[], extern struct paravirt_patch_site __start_parainstructions[],

16
arch/x86/kernel/kprobes/opt.c
View File

@ -422,7 +422,7 @@ err:
void arch_optimize_kprobes(struct list_head *oplist) void arch_optimize_kprobes(struct list_head *oplist)
{ {
struct optimized_kprobe *op, *tmp; struct optimized_kprobe *op, *tmp;
u8 insn_buf[RELATIVEJUMP_SIZE]; u8 insn_buff[RELATIVEJUMP_SIZE];
list_for_each_entry_safe(op, tmp, oplist, list) { list_for_each_entry_safe(op, tmp, oplist, list) {
s32 rel = (s32)((long)op->optinsn.insn - s32 rel = (s32)((long)op->optinsn.insn -
@ -434,10 +434,10 @@ void arch_optimize_kprobes(struct list_head *oplist)
memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE, memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE,
RELATIVE_ADDR_SIZE); RELATIVE_ADDR_SIZE);
insn_buf[0] = RELATIVEJUMP_OPCODE; insn_buff[0] = RELATIVEJUMP_OPCODE;
*(s32 *)(&insn_buf[1]) = rel; *(s32 *)(&insn_buff[1]) = rel;
text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
op->optinsn.insn); op->optinsn.insn);
list_del_init(&op->list); list_del_init(&op->list);
@ -447,12 +447,12 @@ void arch_optimize_kprobes(struct list_head *oplist)
/* Replace a relative jump with a breakpoint (int3). */ /* Replace a relative jump with a breakpoint (int3). */
void arch_unoptimize_kprobe(struct optimized_kprobe *op) void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{ {
u8 insn_buf[RELATIVEJUMP_SIZE]; u8 insn_buff[RELATIVEJUMP_SIZE];
/* Set int3 to first byte for kprobes */ /* Set int3 to first byte for kprobes */
insn_buf[0] = BREAKPOINT_INSTRUCTION; insn_buff[0] = BREAKPOINT_INSTRUCTION;
memcpy(insn_buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE); memcpy(insn_buff + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
text_poke_bp(op->kp.addr, insn_buf, RELATIVEJUMP_SIZE, text_poke_bp(op->kp.addr, insn_buff, RELATIVEJUMP_SIZE,
op->optinsn.insn); op->optinsn.insn);
} }

44
arch/x86/kernel/paravirt.c
View File

@ -58,24 +58,24 @@ struct branch {
u32 delta; u32 delta;
} __attribute__((packed)); } __attribute__((packed));
static unsigned paravirt_patch_call(void *insnbuf, const void *target, static unsigned paravirt_patch_call(void *insn_buff, const void *target,
unsigned long addr, unsigned len) unsigned long addr, unsigned len)
{ {
struct branch *b = insnbuf; const int call_len = 5;
unsigned long delta = (unsigned long)target - (addr+5); struct branch *b = insn_buff;
unsigned long delta = (unsigned long)target - (addr+call_len);
if (len < 5) { if (len < call_len) {
#ifdef CONFIG_RETPOLINE pr_warn("paravirt: Failed to patch indirect CALL at %ps\n", (void *)addr);
WARN_ONCE(1, "Failing to patch indirect CALL in %ps\n", (void *)addr); /* Kernel might not be viable if patching fails, bail out: */
#endif BUG_ON(1);
return len; /* call too long for patch site */
} }
b->opcode = 0xe8; /* call */ b->opcode = 0xe8; /* call */
b->delta = delta; b->delta = delta;
BUILD_BUG_ON(sizeof(*b) != 5); BUILD_BUG_ON(sizeof(*b) != call_len);
return 5; return call_len;
} }
#ifdef CONFIG_PARAVIRT_XXL #ifdef CONFIG_PARAVIRT_XXL
@ -85,10 +85,10 @@ u64 notrace _paravirt_ident_64(u64 x)
return x; return x;
} }
static unsigned paravirt_patch_jmp(void *insnbuf, const void *target, static unsigned paravirt_patch_jmp(void *insn_buff, const void *target,
unsigned long addr, unsigned len) unsigned long addr, unsigned len)
{ {
struct branch *b = insnbuf; struct branch *b = insn_buff;
unsigned long delta = (unsigned long)target - (addr+5); unsigned long delta = (unsigned long)target - (addr+5);
if (len < 5) { if (len < 5) {
@ -113,7 +113,7 @@ void __init native_pv_lock_init(void)
static_branch_disable(&virt_spin_lock_key); static_branch_disable(&virt_spin_lock_key);
} }
unsigned paravirt_patch_default(u8 type, void *insnbuf, unsigned paravirt_patch_default(u8 type, void *insn_buff,
unsigned long addr, unsigned len) unsigned long addr, unsigned len)
{ {
/* /*
@ -125,36 +125,36 @@ unsigned paravirt_patch_default(u8 type, void *insnbuf,
if (opfunc == NULL) if (opfunc == NULL)
/* If there's no function, patch it with a ud2a (BUG) */ /* If there's no function, patch it with a ud2a (BUG) */
ret = paravirt_patch_insns(insnbuf, len, ud2a, ud2a+sizeof(ud2a)); ret = paravirt_patch_insns(insn_buff, len, ud2a, ud2a+sizeof(ud2a));
else if (opfunc == _paravirt_nop) else if (opfunc == _paravirt_nop)
ret = 0; ret = 0;
#ifdef CONFIG_PARAVIRT_XXL #ifdef CONFIG_PARAVIRT_XXL
/* identity functions just return their single argument */ /* identity functions just return their single argument */
else if (opfunc == _paravirt_ident_64) else if (opfunc == _paravirt_ident_64)
ret = paravirt_patch_ident_64(insnbuf, len); ret = paravirt_patch_ident_64(insn_buff, len);
else if (type == PARAVIRT_PATCH(cpu.iret) || else if (type == PARAVIRT_PATCH(cpu.iret) ||
type == PARAVIRT_PATCH(cpu.usergs_sysret64)) type == PARAVIRT_PATCH(cpu.usergs_sysret64))
/* If operation requires a jmp, then jmp */ /* If operation requires a jmp, then jmp */
ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); ret = paravirt_patch_jmp(insn_buff, opfunc, addr, len);
#endif #endif
else else
/* Otherwise call the function. */ /* Otherwise call the function. */
ret = paravirt_patch_call(insnbuf, opfunc, addr, len); ret = paravirt_patch_call(insn_buff, opfunc, addr, len);
return ret; return ret;
} }
unsigned paravirt_patch_insns(void *insnbuf, unsigned len, unsigned paravirt_patch_insns(void *insn_buff, unsigned len,
const char *start, const char *end) const char *start, const char *end)
{ {
unsigned insn_len = end - start; unsigned insn_len = end - start;
if (insn_len > len || start == NULL) /* Alternative instruction is too large for the patch site and we cannot continue: */
insn_len = len; BUG_ON(insn_len > len || start == NULL);
else
memcpy(insnbuf, start, insn_len); memcpy(insn_buff, start, insn_len);
return insn_len; return insn_len;
} }

126
arch/x86/kernel/paravirt_patch.c Normal file
View File

@ -0,0 +1,126 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/stringify.h>
#include <asm/paravirt.h>
#include <asm/asm-offsets.h>
#define PSTART(d, m) \
patch_data_##d.m
#define PEND(d, m) \
(PSTART(d, m) + sizeof(patch_data_##d.m))
#define PATCH(d, m, insn_buff, len) \
paravirt_patch_insns(insn_buff, len, PSTART(d, m), PEND(d, m))
#define PATCH_CASE(ops, m, data, insn_buff, len) \
case PARAVIRT_PATCH(ops.m): \
return PATCH(data, ops##_##m, insn_buff, len)
#ifdef CONFIG_PARAVIRT_XXL
struct patch_xxl {
const unsigned char irq_irq_disable[1];
const unsigned char irq_irq_enable[1];
const unsigned char irq_save_fl[2];
const unsigned char mmu_read_cr2[3];
const unsigned char mmu_read_cr3[3];
const unsigned char mmu_write_cr3[3];
const unsigned char irq_restore_fl[2];
# ifdef CONFIG_X86_64
const unsigned char cpu_wbinvd[2];
const unsigned char cpu_usergs_sysret64[6];
const unsigned char cpu_swapgs[3];
const unsigned char mov64[3];
# else
const unsigned char cpu_iret[1];
# endif
};
static const struct patch_xxl patch_data_xxl = {
.irq_irq_disable = { 0xfa }, // cli
.irq_irq_enable = { 0xfb }, // sti
.irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax
.mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax
.mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax
# ifdef CONFIG_X86_64
.mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3
.irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq
.cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd
.cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8,
0x48, 0x0f, 0x07 }, // swapgs; sysretq
.cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs
.mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax
# else
.mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3
.irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf
.cpu_iret = { 0xcf }, // iret
# endif
};
unsigned int paravirt_patch_ident_64(void *insn_buff, unsigned int len)
{
#ifdef CONFIG_X86_64
return PATCH(xxl, mov64, insn_buff, len);
#endif
return 0;
}
# endif /* CONFIG_PARAVIRT_XXL */
#ifdef CONFIG_PARAVIRT_SPINLOCKS
struct patch_lock {
unsigned char queued_spin_unlock[3];
unsigned char vcpu_is_preempted[2];
};
static const struct patch_lock patch_data_lock = {
.vcpu_is_preempted = { 0x31, 0xc0 }, // xor %eax, %eax
# ifdef CONFIG_X86_64
.queued_spin_unlock = { 0xc6, 0x07, 0x00 }, // movb $0, (%rdi)
# else
.queued_spin_unlock = { 0xc6, 0x00, 0x00 }, // movb $0, (%eax)
# endif
};
#endif /* CONFIG_PARAVIRT_SPINLOCKS */
unsigned int native_patch(u8 type, void *insn_buff, unsigned long addr,
unsigned int len)
{
switch (type) {
#ifdef CONFIG_PARAVIRT_XXL
PATCH_CASE(irq, restore_fl, xxl, insn_buff, len);
PATCH_CASE(irq, save_fl, xxl, insn_buff, len);
PATCH_CASE(irq, irq_enable, xxl, insn_buff, len);
PATCH_CASE(irq, irq_disable, xxl, insn_buff, len);
PATCH_CASE(mmu, read_cr2, xxl, insn_buff, len);
PATCH_CASE(mmu, read_cr3, xxl, insn_buff, len);
PATCH_CASE(mmu, write_cr3, xxl, insn_buff, len);
# ifdef CONFIG_X86_64
PATCH_CASE(cpu, usergs_sysret64, xxl, insn_buff, len);
PATCH_CASE(cpu, swapgs, xxl, insn_buff, len);
PATCH_CASE(cpu, wbinvd, xxl, insn_buff, len);
# else
PATCH_CASE(cpu, iret, xxl, insn_buff, len);
# endif
#endif
#ifdef CONFIG_PARAVIRT_SPINLOCKS
case PARAVIRT_PATCH(lock.queued_spin_unlock):
if (pv_is_native_spin_unlock())
return PATCH(lock, queued_spin_unlock, insn_buff, len);
break;
case PARAVIRT_PATCH(lock.vcpu_is_preempted):
if (pv_is_native_vcpu_is_preempted())
return PATCH(lock, vcpu_is_preempted, insn_buff, len);
break;
#endif
default:
break;
}
return paravirt_patch_default(type, insn_buff, addr, len);
}

67
arch/x86/kernel/paravirt_patch_32.c
View File

@ -1,67 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
#include <asm/paravirt.h>
#ifdef CONFIG_PARAVIRT_XXL
DEF_NATIVE(irq, irq_disable, "cli");
DEF_NATIVE(irq, irq_enable, "sti");
DEF_NATIVE(irq, restore_fl, "push %eax; popf");
DEF_NATIVE(irq, save_fl, "pushf; pop %eax");
DEF_NATIVE(cpu, iret, "iret");
DEF_NATIVE(mmu, read_cr2, "mov %cr2, %eax");
DEF_NATIVE(mmu, write_cr3, "mov %eax, %cr3");
DEF_NATIVE(mmu, read_cr3, "mov %cr3, %eax");
unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
{
/* arg in %edx:%eax, return in %edx:%eax */
return 0;
}
#endif
#if defined(CONFIG_PARAVIRT_SPINLOCKS)
DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%eax)");
DEF_NATIVE(lock, vcpu_is_preempted, "xor %eax, %eax");
#endif
extern bool pv_is_native_spin_unlock(void);
extern bool pv_is_native_vcpu_is_preempted(void);
unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len)
{
#define PATCH_SITE(ops, x) \
case PARAVIRT_PATCH(ops.x): \
return paravirt_patch_insns(ibuf, len, start_##ops##_##x, end_##ops##_##x)
switch (type) {
#ifdef CONFIG_PARAVIRT_XXL
PATCH_SITE(irq, irq_disable);
PATCH_SITE(irq, irq_enable);
PATCH_SITE(irq, restore_fl);
PATCH_SITE(irq, save_fl);
PATCH_SITE(cpu, iret);
PATCH_SITE(mmu, read_cr2);
PATCH_SITE(mmu, read_cr3);
PATCH_SITE(mmu, write_cr3);
#endif
#if defined(CONFIG_PARAVIRT_SPINLOCKS)
case PARAVIRT_PATCH(lock.queued_spin_unlock):
if (pv_is_native_spin_unlock())
return paravirt_patch_insns(ibuf, len,
start_lock_queued_spin_unlock,
end_lock_queued_spin_unlock);
break;
case PARAVIRT_PATCH(lock.vcpu_is_preempted):
if (pv_is_native_vcpu_is_preempted())
return paravirt_patch_insns(ibuf, len,
start_lock_vcpu_is_preempted,
end_lock_vcpu_is_preempted);
break;
#endif
default:
break;
}
#undef PATCH_SITE
return paravirt_patch_default(type, ibuf, addr, len);
}

75
arch/x86/kernel/paravirt_patch_64.c
View File

@ -1,75 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
#include <asm/paravirt.h>
#include <asm/asm-offsets.h>
#include <linux/stringify.h>
#ifdef CONFIG_PARAVIRT_XXL
DEF_NATIVE(irq, irq_disable, "cli");
DEF_NATIVE(irq, irq_enable, "sti");
DEF_NATIVE(irq, restore_fl, "pushq %rdi; popfq");
DEF_NATIVE(irq, save_fl, "pushfq; popq %rax");
DEF_NATIVE(mmu, read_cr2, "movq %cr2, %rax");
DEF_NATIVE(mmu, read_cr3, "movq %cr3, %rax");
DEF_NATIVE(mmu, write_cr3, "movq %rdi, %cr3");
DEF_NATIVE(cpu, wbinvd, "wbinvd");
DEF_NATIVE(cpu, usergs_sysret64, "swapgs; sysretq");
DEF_NATIVE(cpu, swapgs, "swapgs");
DEF_NATIVE(, mov64, "mov %rdi, %rax");
unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
{
return paravirt_patch_insns(insnbuf, len,
start__mov64, end__mov64);
}
#endif
#if defined(CONFIG_PARAVIRT_SPINLOCKS)
DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%rdi)");
DEF_NATIVE(lock, vcpu_is_preempted, "xor %eax, %eax");
#endif
extern bool pv_is_native_spin_unlock(void);
extern bool pv_is_native_vcpu_is_preempted(void);
unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len)
{
#define PATCH_SITE(ops, x) \
case PARAVIRT_PATCH(ops.x): \
return paravirt_patch_insns(ibuf, len, start_##ops##_##x, end_##ops##_##x)
switch (type) {
#ifdef CONFIG_PARAVIRT_XXL
PATCH_SITE(irq, restore_fl);
PATCH_SITE(irq, save_fl);
PATCH_SITE(irq, irq_enable);
PATCH_SITE(irq, irq_disable);
PATCH_SITE(cpu, usergs_sysret64);
PATCH_SITE(cpu, swapgs);
PATCH_SITE(cpu, wbinvd);
PATCH_SITE(mmu, read_cr2);
PATCH_SITE(mmu, read_cr3);
PATCH_SITE(mmu, write_cr3);
#endif
#if defined(CONFIG_PARAVIRT_SPINLOCKS)
case PARAVIRT_PATCH(lock.queued_spin_unlock):
if (pv_is_native_spin_unlock())
return paravirt_patch_insns(ibuf, len,
start_lock_queued_spin_unlock,
end_lock_queued_spin_unlock);
break;
case PARAVIRT_PATCH(lock.vcpu_is_preempted):
if (pv_is_native_vcpu_is_preempted())
return paravirt_patch_insns(ibuf, len,
start_lock_vcpu_is_preempted,
end_lock_vcpu_is_preempted);
break;
#endif
default:
break;
}
#undef PATCH_SITE
return paravirt_patch_default(type, ibuf, addr, len);
}

8
arch/x86/tools/insn_decoder_test.c
View File

@ -111,7 +111,7 @@ static void parse_args(int argc, char **argv)
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
char line[BUFSIZE], sym[BUFSIZE] = "<unknown>"; char line[BUFSIZE], sym[BUFSIZE] = "<unknown>";
unsigned char insn_buf[16]; unsigned char insn_buff[16];
struct insn insn; struct insn insn;
int insns = 0; int insns = 0;
int warnings = 0; int warnings = 0;
@ -130,7 +130,7 @@ int main(int argc, char **argv)
} }
insns++; insns++;
memset(insn_buf, 0, 16); memset(insn_buff, 0, 16);
strcpy(copy, line); strcpy(copy, line);
tab1 = strchr(copy, '\t'); tab1 = strchr(copy, '\t');
if (!tab1) if (!tab1)
@ -143,13 +143,13 @@ int main(int argc, char **argv)
*tab2 = '\0'; /* Characters beyond tab2 aren't examined */ *tab2 = '\0'; /* Characters beyond tab2 aren't examined */
while (s < tab2) { while (s < tab2) {
if (sscanf(s, "%x", &b) == 1) { if (sscanf(s, "%x", &b) == 1) {
insn_buf[nb++] = (unsigned char) b; insn_buff[nb++] = (unsigned char) b;
s += 3; s += 3;
} else } else
break; break;
} }
/* Decode an instruction */ /* Decode an instruction */
insn_init(&insn, insn_buf, sizeof(insn_buf), x86_64); insn_init(&insn, insn_buff, sizeof(insn_buff), x86_64);
insn_get_length(&insn); insn_get_length(&insn);
if (insn.length != nb) { if (insn.length != nb) {
warnings++; warnings++;

28
arch/x86/tools/insn_sanity.c
View File

@ -83,7 +83,7 @@ static void dump_insn(FILE *fp, struct insn *insn)
} }
static void dump_stream(FILE *fp, const char *msg, unsigned long nr_iter, static void dump_stream(FILE *fp, const char *msg, unsigned long nr_iter,
unsigned char *insn_buf, struct insn *insn) unsigned char *insn_buff, struct insn *insn)
{ {
int i; int i;
@ -96,7 +96,7 @@ static void dump_stream(FILE *fp, const char *msg, unsigned long nr_iter,
/* Input a decoded instruction sequence directly */ /* Input a decoded instruction sequence directly */
fprintf(fp, " $ echo "); fprintf(fp, " $ echo ");
for (i = 0; i < MAX_INSN_SIZE; i++) for (i = 0; i < MAX_INSN_SIZE; i++)
fprintf(fp, " %02x", insn_buf[i]); fprintf(fp, " %02x", insn_buff[i]);
fprintf(fp, " | %s -i -\n", prog); fprintf(fp, " | %s -i -\n", prog);
if (!input_file) { if (!input_file) {
@ -124,7 +124,7 @@ fail:
} }
/* Read given instruction sequence from the input file */ /* Read given instruction sequence from the input file */
static int read_next_insn(unsigned char *insn_buf) static int read_next_insn(unsigned char *insn_buff)
{ {
char buf[256] = "", *tmp; char buf[256] = "", *tmp;
int i; int i;
@ -134,7 +134,7 @@ static int read_next_insn(unsigned char *insn_buf)
return 0; return 0;
for (i = 0; i < MAX_INSN_SIZE; i++) { for (i = 0; i < MAX_INSN_SIZE; i++) {
insn_buf[i] = (unsigned char)strtoul(tmp, &tmp, 16); insn_buff[i] = (unsigned char)strtoul(tmp, &tmp, 16);
if (*tmp != ' ') if (*tmp != ' ')
break; break;
} }
@ -142,19 +142,19 @@ static int read_next_insn(unsigned char *insn_buf)
return i; return i;
} }
static int generate_insn(unsigned char *insn_buf) static int generate_insn(unsigned char *insn_buff)
{ {
int i; int i;
if (input_file) if (input_file)
return read_next_insn(insn_buf); return read_next_insn(insn_buff);
/* Fills buffer with random binary up to MAX_INSN_SIZE */ /* Fills buffer with random binary up to MAX_INSN_SIZE */
for (i = 0; i < MAX_INSN_SIZE - 1; i += 2) for (i = 0; i < MAX_INSN_SIZE - 1; i += 2)
*(unsigned short *)(&insn_buf[i]) = random() & 0xffff; *(unsigned short *)(&insn_buff[i]) = random() & 0xffff;
while (i < MAX_INSN_SIZE) while (i < MAX_INSN_SIZE)
insn_buf[i++] = random() & 0xff; insn_buff[i++] = random() & 0xff;
return i; return i;
} }
@ -226,31 +226,31 @@ int main(int argc, char **argv)
int insns = 0; int insns = 0;
int errors = 0; int errors = 0;
unsigned long i; unsigned long i;
unsigned char insn_buf[MAX_INSN_SIZE * 2]; unsigned char insn_buff[MAX_INSN_SIZE * 2];
parse_args(argc, argv); parse_args(argc, argv);
/* Prepare stop bytes with NOPs */ /* Prepare stop bytes with NOPs */
memset(insn_buf + MAX_INSN_SIZE, INSN_NOP, MAX_INSN_SIZE); memset(insn_buff + MAX_INSN_SIZE, INSN_NOP, MAX_INSN_SIZE);
for (i = 0; i < iter_end; i++) { for (i = 0; i < iter_end; i++) {
if (generate_insn(insn_buf) <= 0) if (generate_insn(insn_buff) <= 0)
break; break;
if (i < iter_start) /* Skip to given iteration number */ if (i < iter_start) /* Skip to given iteration number */
continue; continue;
/* Decode an instruction */ /* Decode an instruction */
insn_init(&insn, insn_buf, sizeof(insn_buf), x86_64); insn_init(&insn, insn_buff, sizeof(insn_buff), x86_64);
insn_get_length(&insn); insn_get_length(&insn);
if (insn.next_byte <= insn.kaddr || if (insn.next_byte <= insn.kaddr ||
insn.kaddr + MAX_INSN_SIZE < insn.next_byte) { insn.kaddr + MAX_INSN_SIZE < insn.next_byte) {
/* Access out-of-range memory */ /* Access out-of-range memory */
dump_stream(stderr, "Error: Found an access violation", i, insn_buf, &insn); dump_stream(stderr, "Error: Found an access violation", i, insn_buff, &insn);
errors++; errors++;
} else if (verbose && !insn_complete(&insn)) } else if (verbose && !insn_complete(&insn))
dump_stream(stdout, "Info: Found an undecodable input", i, insn_buf, &insn); dump_stream(stdout, "Info: Found an undecodable input", i, insn_buff, &insn);
else if (verbose >= 2) else if (verbose >= 2)
dump_insn(stdout, &insn); dump_insn(stdout, &insn);
insns++; insns++;