2012-02-18 20:03:15 +08:00
|
|
|
//===-- X86InstrControl.td - Control Flow Instructions -----*- tablegen -*-===//
|
2011-01-26 10:03:37 +08:00
|
|
|
//
|
2010-10-05 14:04:14 +08:00
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
|
//
|
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
|
// License. See LICENSE.TXT for details.
|
2011-01-26 10:03:37 +08:00
|
|
|
//
|
2010-10-05 14:04:14 +08:00
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
//
|
|
|
|
|
// This file describes the X86 jump, return, call, and related instructions.
|
|
|
|
|
//
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// Control Flow Instructions.
|
|
|
|
|
//
|
|
|
|
|
|
|
|
|
|
// Return instructions.
|
2012-08-25 04:52:44 +08:00
|
|
|
//
|
|
|
|
|
// The X86retflag return instructions are variadic because we may add ST0 and
|
|
|
|
|
// ST1 arguments when returning values on the x87 stack.
|
2010-10-05 14:04:14 +08:00
|
|
|
let isTerminator = 1, isReturn = 1, isBarrier = 1,
|
2013-03-27 02:24:17 +08:00
|
|
|
hasCtrlDep = 1, FPForm = SpecialFP, SchedRW = [WriteJumpLd] in {
|
2014-01-08 20:58:07 +08:00
|
|
|
def RETL : I <0xC3, RawFrm, (outs), (ins variable_ops),
|
2016-03-05 06:56:17 +08:00
|
|
|
"ret{l}", [], IIC_RET>, OpSize32,
|
2014-02-18 16:18:29 +08:00
|
|
|
Requires<[Not64BitMode]>;
|
2014-01-08 20:58:07 +08:00
|
|
|
def RETQ : I <0xC3, RawFrm, (outs), (ins variable_ops),
|
2016-03-05 06:56:17 +08:00
|
|
|
"ret{q}", [], IIC_RET>, OpSize32,
|
2014-02-18 16:18:29 +08:00
|
|
|
Requires<[In64BitMode]>;
|
2012-07-05 07:53:27 +08:00
|
|
|
def RETW : I <0xC3, RawFrm, (outs), (ins),
|
2012-04-11 09:10:53 +08:00
|
|
|
"ret{w}",
|
2014-02-02 17:25:09 +08:00
|
|
|
[], IIC_RET>, OpSize16;
|
2014-01-13 22:05:59 +08:00
|
|
|
def RETIL : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt, variable_ops),
|
2014-01-08 20:58:07 +08:00
|
|
|
"ret{l}\t$amt",
|
2016-03-05 06:56:17 +08:00
|
|
|
[], IIC_RET_IMM>, OpSize32,
|
2014-01-13 22:05:59 +08:00
|
|
|
Requires<[Not64BitMode]>;
|
|
|
|
|
def RETIQ : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt, variable_ops),
|
|
|
|
|
"ret{q}\t$amt",
|
2016-03-05 06:56:17 +08:00
|
|
|
[], IIC_RET_IMM>, OpSize32,
|
2014-01-13 22:05:59 +08:00
|
|
|
Requires<[In64BitMode]>;
|
2012-07-05 07:53:27 +08:00
|
|
|
def RETIW : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt),
|
2012-04-11 09:10:53 +08:00
|
|
|
"ret{w}\t$amt",
|
2014-02-02 17:25:09 +08:00
|
|
|
[], IIC_RET_IMM>, OpSize16;
|
2010-11-13 02:54:56 +08:00
|
|
|
def LRETL : I <0xCB, RawFrm, (outs), (ins),
|
2014-02-02 17:25:09 +08:00
|
|
|
"{l}ret{l|f}", [], IIC_RET>, OpSize32;
|
2014-01-13 22:05:59 +08:00
|
|
|
def LRETQ : RI <0xCB, RawFrm, (outs), (ins),
|
|
|
|
|
"{l}ret{|f}q", [], IIC_RET>, Requires<[In64BitMode]>;
|
2012-04-11 09:10:53 +08:00
|
|
|
def LRETW : I <0xCB, RawFrm, (outs), (ins),
|
2014-02-02 17:25:09 +08:00
|
|
|
"{l}ret{w|f}", [], IIC_RET>, OpSize16;
|
2014-01-13 22:05:59 +08:00
|
|
|
def LRETIL : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt),
|
2014-02-02 17:25:09 +08:00
|
|
|
"{l}ret{l|f}\t$amt", [], IIC_RET>, OpSize32;
|
2014-01-13 22:05:59 +08:00
|
|
|
def LRETIQ : RIi16<0xCA, RawFrm, (outs), (ins i16imm:$amt),
|
|
|
|
|
"{l}ret{|f}q\t$amt", [], IIC_RET>, Requires<[In64BitMode]>;
|
2010-10-19 01:04:36 +08:00
|
|
|
def LRETIW : Ii16<0xCA, RawFrm, (outs), (ins i16imm:$amt),
|
2014-02-02 17:25:09 +08:00
|
|
|
"{l}ret{w|f}\t$amt", [], IIC_RET>, OpSize16;
|
2015-12-21 22:07:14 +08:00
|
|
|
|
|
|
|
|
// The machine return from interrupt instruction, but sometimes we need to
|
|
|
|
|
// perform a post-epilogue stack adjustment. Codegen emits the pseudo form
|
|
|
|
|
// which expands to include an SP adjustment if necessary.
|
|
|
|
|
def IRET16 : I <0xcf, RawFrm, (outs), (ins), "iret{w}", [], IIC_IRET>,
|
|
|
|
|
OpSize16;
|
|
|
|
|
def IRET32 : I <0xcf, RawFrm, (outs), (ins), "iret{l|d}", [],
|
|
|
|
|
IIC_IRET>, OpSize32;
|
|
|
|
|
def IRET64 : RI <0xcf, RawFrm, (outs), (ins), "iretq", [],
|
|
|
|
|
IIC_IRET>, Requires<[In64BitMode]>;
|
|
|
|
|
let isCodeGenOnly = 1 in
|
2016-03-05 06:56:17 +08:00
|
|
|
def IRET : PseudoI<(outs), (ins i32imm:$adj), [(X86iret timm:$adj)]>;
|
|
|
|
|
def RET : PseudoI<(outs), (ins i32imm:$adj, variable_ops), [(X86retflag timm:$adj)]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Unconditional branches.
|
2013-03-27 02:24:17 +08:00
|
|
|
let isBarrier = 1, isBranch = 1, isTerminator = 1, SchedRW = [WriteJump] in {
|
2010-10-05 14:04:14 +08:00
|
|
|
def JMP_1 : Ii8PCRel<0xEB, RawFrm, (outs), (ins brtarget8:$dst),
|
2015-01-06 12:23:53 +08:00
|
|
|
"jmp\t$dst", [(br bb:$dst)], IIC_JMP_REL>;
|
2015-01-06 12:23:57 +08:00
|
|
|
let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
|
2015-01-06 16:59:30 +08:00
|
|
|
def JMP_2 : Ii16PCRel<0xE9, RawFrm, (outs), (ins brtarget16:$dst),
|
2015-01-06 12:23:53 +08:00
|
|
|
"jmp\t$dst", [], IIC_JMP_REL>, OpSize16;
|
2015-01-06 16:59:30 +08:00
|
|
|
def JMP_4 : Ii32PCRel<0xE9, RawFrm, (outs), (ins brtarget32:$dst),
|
2015-01-06 12:23:53 +08:00
|
|
|
"jmp\t$dst", [], IIC_JMP_REL>, OpSize32;
|
|
|
|
|
}
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Conditional Branches.
|
2013-03-27 02:24:17 +08:00
|
|
|
let isBranch = 1, isTerminator = 1, Uses = [EFLAGS], SchedRW = [WriteJump] in {
|
2010-10-05 14:04:14 +08:00
|
|
|
multiclass ICBr<bits<8> opc1, bits<8> opc4, string asm, PatFrag Cond> {
|
2015-01-06 12:23:53 +08:00
|
|
|
def _1 : Ii8PCRel <opc1, RawFrm, (outs), (ins brtarget8:$dst), asm,
|
|
|
|
|
[(X86brcond bb:$dst, Cond, EFLAGS)], IIC_Jcc>;
|
2015-01-06 12:23:57 +08:00
|
|
|
let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
|
2015-01-06 16:59:30 +08:00
|
|
|
def _2 : Ii16PCRel<opc4, RawFrm, (outs), (ins brtarget16:$dst), asm,
|
2015-01-06 12:23:53 +08:00
|
|
|
[], IIC_Jcc>, OpSize16, TB;
|
2015-01-06 16:59:30 +08:00
|
|
|
def _4 : Ii32PCRel<opc4, RawFrm, (outs), (ins brtarget32:$dst), asm,
|
2015-01-06 12:23:53 +08:00
|
|
|
[], IIC_Jcc>, TB, OpSize32;
|
|
|
|
|
}
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
defm JO : ICBr<0x70, 0x80, "jo\t$dst" , X86_COND_O>;
|
2015-01-06 12:23:53 +08:00
|
|
|
defm JNO : ICBr<0x71, 0x81, "jno\t$dst", X86_COND_NO>;
|
2010-10-05 14:04:14 +08:00
|
|
|
defm JB : ICBr<0x72, 0x82, "jb\t$dst" , X86_COND_B>;
|
|
|
|
|
defm JAE : ICBr<0x73, 0x83, "jae\t$dst", X86_COND_AE>;
|
|
|
|
|
defm JE : ICBr<0x74, 0x84, "je\t$dst" , X86_COND_E>;
|
|
|
|
|
defm JNE : ICBr<0x75, 0x85, "jne\t$dst", X86_COND_NE>;
|
|
|
|
|
defm JBE : ICBr<0x76, 0x86, "jbe\t$dst", X86_COND_BE>;
|
|
|
|
|
defm JA : ICBr<0x77, 0x87, "ja\t$dst" , X86_COND_A>;
|
|
|
|
|
defm JS : ICBr<0x78, 0x88, "js\t$dst" , X86_COND_S>;
|
|
|
|
|
defm JNS : ICBr<0x79, 0x89, "jns\t$dst", X86_COND_NS>;
|
|
|
|
|
defm JP : ICBr<0x7A, 0x8A, "jp\t$dst" , X86_COND_P>;
|
|
|
|
|
defm JNP : ICBr<0x7B, 0x8B, "jnp\t$dst", X86_COND_NP>;
|
|
|
|
|
defm JL : ICBr<0x7C, 0x8C, "jl\t$dst" , X86_COND_L>;
|
|
|
|
|
defm JGE : ICBr<0x7D, 0x8D, "jge\t$dst", X86_COND_GE>;
|
|
|
|
|
defm JLE : ICBr<0x7E, 0x8E, "jle\t$dst", X86_COND_LE>;
|
|
|
|
|
defm JG : ICBr<0x7F, 0x8F, "jg\t$dst" , X86_COND_G>;
|
|
|
|
|
|
|
|
|
|
// jcx/jecx/jrcx instructions.
|
2013-09-03 11:56:17 +08:00
|
|
|
let isBranch = 1, isTerminator = 1, hasSideEffects = 0, SchedRW = [WriteJump] in {
|
2010-10-05 14:04:14 +08:00
|
|
|
// These are the 32-bit versions of this instruction for the asmparser. In
|
|
|
|
|
// 32-bit mode, the address size prefix is jcxz and the unprefixed version is
|
|
|
|
|
// jecxz.
|
|
|
|
|
let Uses = [CX] in
|
|
|
|
|
def JCXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst),
|
2015-07-04 08:01:07 +08:00
|
|
|
"jcxz\t$dst", [], IIC_JCXZ>, AdSize16,
|
|
|
|
|
Requires<[Not64BitMode]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
let Uses = [ECX] in
|
2015-01-02 15:02:25 +08:00
|
|
|
def JECXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst),
|
|
|
|
|
"jecxz\t$dst", [], IIC_JCXZ>, AdSize32;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
let Uses = [RCX] in
|
|
|
|
|
def JRCXZ : Ii8PCRel<0xE3, RawFrm, (outs), (ins brtarget8:$dst),
|
2015-07-04 08:01:07 +08:00
|
|
|
"jrcxz\t$dst", [], IIC_JCXZ>, AdSize64,
|
|
|
|
|
Requires<[In64BitMode]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Indirect branches
|
|
|
|
|
let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
|
2014-01-08 20:57:49 +08:00
|
|
|
def JMP16r : I<0xFF, MRM4r, (outs), (ins GR16:$dst), "jmp{w}\t{*}$dst",
|
|
|
|
|
[(brind GR16:$dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>,
|
2014-02-02 17:25:09 +08:00
|
|
|
OpSize16, Sched<[WriteJump]>;
|
2014-01-08 20:57:49 +08:00
|
|
|
def JMP16m : I<0xFF, MRM4m, (outs), (ins i16mem:$dst), "jmp{w}\t{*}$dst",
|
|
|
|
|
[(brind (loadi16 addr:$dst))], IIC_JMP_MEM>,
|
2014-02-02 17:25:09 +08:00
|
|
|
Requires<[Not64BitMode]>, OpSize16, Sched<[WriteJumpLd]>;
|
2014-01-08 20:57:49 +08:00
|
|
|
|
2010-10-05 14:04:14 +08:00
|
|
|
def JMP32r : I<0xFF, MRM4r, (outs), (ins GR32:$dst), "jmp{l}\t{*}$dst",
|
2013-12-20 10:04:49 +08:00
|
|
|
[(brind GR32:$dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>,
|
2014-02-02 17:25:09 +08:00
|
|
|
OpSize32, Sched<[WriteJump]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
def JMP32m : I<0xFF, MRM4m, (outs), (ins i32mem:$dst), "jmp{l}\t{*}$dst",
|
2013-03-27 02:24:17 +08:00
|
|
|
[(brind (loadi32 addr:$dst))], IIC_JMP_MEM>,
|
2014-02-02 17:25:09 +08:00
|
|
|
Requires<[Not64BitMode]>, OpSize32, Sched<[WriteJumpLd]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
def JMP64r : I<0xFF, MRM4r, (outs), (ins GR64:$dst), "jmp{q}\t{*}$dst",
|
2013-03-27 02:24:17 +08:00
|
|
|
[(brind GR64:$dst)], IIC_JMP_REG>, Requires<[In64BitMode]>,
|
|
|
|
|
Sched<[WriteJump]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
def JMP64m : I<0xFF, MRM4m, (outs), (ins i64mem:$dst), "jmp{q}\t{*}$dst",
|
2013-03-27 02:24:17 +08:00
|
|
|
[(brind (loadi64 addr:$dst))], IIC_JMP_MEM>,
|
|
|
|
|
Requires<[In64BitMode]>, Sched<[WriteJumpLd]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
2014-12-20 15:43:27 +08:00
|
|
|
let Predicates = [Not64BitMode] in {
|
|
|
|
|
def FARJMP16i : Iseg16<0xEA, RawFrmImm16, (outs),
|
|
|
|
|
(ins i16imm:$off, i16imm:$seg),
|
2014-12-21 07:05:52 +08:00
|
|
|
"ljmp{w}\t$seg, $off", [],
|
2014-12-20 15:43:27 +08:00
|
|
|
IIC_JMP_FAR_PTR>, OpSize16, Sched<[WriteJump]>;
|
|
|
|
|
def FARJMP32i : Iseg32<0xEA, RawFrmImm16, (outs),
|
|
|
|
|
(ins i32imm:$off, i16imm:$seg),
|
2014-12-21 07:05:52 +08:00
|
|
|
"ljmp{l}\t$seg, $off", [],
|
2014-12-20 15:43:27 +08:00
|
|
|
IIC_JMP_FAR_PTR>, OpSize32, Sched<[WriteJump]>;
|
|
|
|
|
}
|
2010-10-05 14:04:14 +08:00
|
|
|
def FARJMP64 : RI<0xFF, MRM5m, (outs), (ins opaque80mem:$dst),
|
2013-03-27 02:24:17 +08:00
|
|
|
"ljmp{q}\t{*}$dst", [], IIC_JMP_FAR_MEM>,
|
|
|
|
|
Sched<[WriteJump]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
2011-01-26 10:03:37 +08:00
|
|
|
def FARJMP16m : I<0xFF, MRM5m, (outs), (ins opaque32mem:$dst),
|
2014-02-02 17:25:09 +08:00
|
|
|
"ljmp{w}\t{*}$dst", [], IIC_JMP_FAR_MEM>, OpSize16,
|
2013-03-27 02:24:17 +08:00
|
|
|
Sched<[WriteJumpLd]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
def FARJMP32m : I<0xFF, MRM5m, (outs), (ins opaque48mem:$dst),
|
2017-08-09 23:34:55 +08:00
|
|
|
"{l}jmp{l}\t{*}$dst", [], IIC_JMP_FAR_MEM>, OpSize32,
|
2013-03-27 02:24:17 +08:00
|
|
|
Sched<[WriteJumpLd]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Loop instructions
|
2013-03-27 02:24:17 +08:00
|
|
|
let SchedRW = [WriteJump] in {
|
2012-02-02 07:20:51 +08:00
|
|
|
def LOOP : Ii8PCRel<0xE2, RawFrm, (outs), (ins brtarget8:$dst), "loop\t$dst", [], IIC_LOOP>;
|
|
|
|
|
def LOOPE : Ii8PCRel<0xE1, RawFrm, (outs), (ins brtarget8:$dst), "loope\t$dst", [], IIC_LOOPE>;
|
|
|
|
|
def LOOPNE : Ii8PCRel<0xE0, RawFrm, (outs), (ins brtarget8:$dst), "loopne\t$dst", [], IIC_LOOPNE>;
|
2013-03-27 02:24:17 +08:00
|
|
|
}
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// Call Instructions...
|
|
|
|
|
//
|
|
|
|
|
let isCall = 1 in
|
|
|
|
|
// All calls clobber the non-callee saved registers. ESP is marked as
|
|
|
|
|
// a use to prevent stack-pointer assignments that appear immediately
|
|
|
|
|
// before calls from potentially appearing dead. Uses for argument
|
|
|
|
|
// registers are added manually.
|
2017-11-26 21:02:45 +08:00
|
|
|
let Uses = [ESP, SSP] in {
|
2010-10-05 14:04:14 +08:00
|
|
|
def CALLpcrel32 : Ii32PCRel<0xE8, RawFrm,
|
2012-07-05 07:53:27 +08:00
|
|
|
(outs), (ins i32imm_pcrel:$dst),
|
2014-02-02 17:25:09 +08:00
|
|
|
"call{l}\t$dst", [], IIC_CALL_RI>, OpSize32,
|
2013-12-20 10:04:49 +08:00
|
|
|
Requires<[Not64BitMode]>, Sched<[WriteJump]>;
|
2014-12-22 04:05:06 +08:00
|
|
|
let hasSideEffects = 0 in
|
|
|
|
|
def CALLpcrel16 : Ii16PCRel<0xE8, RawFrm,
|
|
|
|
|
(outs), (ins i16imm_pcrel:$dst),
|
|
|
|
|
"call{w}\t$dst", [], IIC_CALL_RI>, OpSize16,
|
|
|
|
|
Sched<[WriteJump]>;
|
2014-01-08 20:57:49 +08:00
|
|
|
def CALL16r : I<0xFF, MRM2r, (outs), (ins GR16:$dst),
|
|
|
|
|
"call{w}\t{*}$dst", [(X86call GR16:$dst)], IIC_CALL_RI>,
|
2014-02-02 17:25:09 +08:00
|
|
|
OpSize16, Requires<[Not64BitMode]>, Sched<[WriteJump]>;
|
2014-01-08 20:57:49 +08:00
|
|
|
def CALL16m : I<0xFF, MRM2m, (outs), (ins i16mem:$dst),
|
|
|
|
|
"call{w}\t{*}$dst", [(X86call (loadi16 addr:$dst))],
|
2014-02-02 17:25:09 +08:00
|
|
|
IIC_CALL_MEM>, OpSize16,
|
2014-01-08 20:57:49 +08:00
|
|
|
Requires<[Not64BitMode,FavorMemIndirectCall]>,
|
|
|
|
|
Sched<[WriteJumpLd]>;
|
2012-07-05 07:53:27 +08:00
|
|
|
def CALL32r : I<0xFF, MRM2r, (outs), (ins GR32:$dst),
|
2012-02-02 07:20:51 +08:00
|
|
|
"call{l}\t{*}$dst", [(X86call GR32:$dst)], IIC_CALL_RI>,
|
Introduce the "retpoline" x86 mitigation technique for variant #2 of the speculative execution vulnerabilities disclosed today, specifically identified by CVE-2017-5715, "Branch Target Injection", and is one of the two halves to Spectre..
Summary:
First, we need to explain the core of the vulnerability. Note that this
is a very incomplete description, please see the Project Zero blog post
for details:
https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
The basis for branch target injection is to direct speculative execution
of the processor to some "gadget" of executable code by poisoning the
prediction of indirect branches with the address of that gadget. The
gadget in turn contains an operation that provides a side channel for
reading data. Most commonly, this will look like a load of secret data
followed by a branch on the loaded value and then a load of some
predictable cache line. The attacker then uses timing of the processors
cache to determine which direction the branch took *in the speculative
execution*, and in turn what one bit of the loaded value was. Due to the
nature of these timing side channels and the branch predictor on Intel
processors, this allows an attacker to leak data only accessible to
a privileged domain (like the kernel) back into an unprivileged domain.
The goal is simple: avoid generating code which contains an indirect
branch that could have its prediction poisoned by an attacker. In many
cases, the compiler can simply use directed conditional branches and
a small search tree. LLVM already has support for lowering switches in
this way and the first step of this patch is to disable jump-table
lowering of switches and introduce a pass to rewrite explicit indirectbr
sequences into a switch over integers.
However, there is no fully general alternative to indirect calls. We
introduce a new construct we call a "retpoline" to implement indirect
calls in a non-speculatable way. It can be thought of loosely as
a trampoline for indirect calls which uses the RET instruction on x86.
Further, we arrange for a specific call->ret sequence which ensures the
processor predicts the return to go to a controlled, known location. The
retpoline then "smashes" the return address pushed onto the stack by the
call with the desired target of the original indirect call. The result
is a predicted return to the next instruction after a call (which can be
used to trap speculative execution within an infinite loop) and an
actual indirect branch to an arbitrary address.
On 64-bit x86 ABIs, this is especially easily done in the compiler by
using a guaranteed scratch register to pass the target into this device.
For 32-bit ABIs there isn't a guaranteed scratch register and so several
different retpoline variants are introduced to use a scratch register if
one is available in the calling convention and to otherwise use direct
stack push/pop sequences to pass the target address.
This "retpoline" mitigation is fully described in the following blog
post: https://support.google.com/faqs/answer/7625886
We also support a target feature that disables emission of the retpoline
thunk by the compiler to allow for custom thunks if users want them.
These are particularly useful in environments like kernels that
routinely do hot-patching on boot and want to hot-patch their thunk to
different code sequences. They can write this custom thunk and use
`-mretpoline-external-thunk` *in addition* to `-mretpoline`. In this
case, on x86-64 thu thunk names must be:
```
__llvm_external_retpoline_r11
```
or on 32-bit:
```
__llvm_external_retpoline_eax
__llvm_external_retpoline_ecx
__llvm_external_retpoline_edx
__llvm_external_retpoline_push
```
And the target of the retpoline is passed in the named register, or in
the case of the `push` suffix on the top of the stack via a `pushl`
instruction.
There is one other important source of indirect branches in x86 ELF
binaries: the PLT. These patches also include support for LLD to
generate PLT entries that perform a retpoline-style indirection.
The only other indirect branches remaining that we are aware of are from
precompiled runtimes (such as crt0.o and similar). The ones we have
found are not really attackable, and so we have not focused on them
here, but eventually these runtimes should also be replicated for
retpoline-ed configurations for completeness.
For kernels or other freestanding or fully static executables, the
compiler switch `-mretpoline` is sufficient to fully mitigate this
particular attack. For dynamic executables, you must compile *all*
libraries with `-mretpoline` and additionally link the dynamic
executable and all shared libraries with LLD and pass `-z retpolineplt`
(or use similar functionality from some other linker). We strongly
recommend also using `-z now` as non-lazy binding allows the
retpoline-mitigated PLT to be substantially smaller.
When manually apply similar transformations to `-mretpoline` to the
Linux kernel we observed very small performance hits to applications
running typical workloads, and relatively minor hits (approximately 2%)
even for extremely syscall-heavy applications. This is largely due to
the small number of indirect branches that occur in performance
sensitive paths of the kernel.
When using these patches on statically linked applications, especially
C++ applications, you should expect to see a much more dramatic
performance hit. For microbenchmarks that are switch, indirect-, or
virtual-call heavy we have seen overheads ranging from 10% to 50%.
However, real-world workloads exhibit substantially lower performance
impact. Notably, techniques such as PGO and ThinLTO dramatically reduce
the impact of hot indirect calls (by speculatively promoting them to
direct calls) and allow optimized search trees to be used to lower
switches. If you need to deploy these techniques in C++ applications, we
*strongly* recommend that you ensure all hot call targets are statically
linked (avoiding PLT indirection) and use both PGO and ThinLTO. Well
tuned servers using all of these techniques saw 5% - 10% overhead from
the use of retpoline.
We will add detailed documentation covering these components in
subsequent patches, but wanted to make the core functionality available
as soon as possible. Happy for more code review, but we'd really like to
get these patches landed and backported ASAP for obvious reasons. We're
planning to backport this to both 6.0 and 5.0 release streams and get
a 5.0 release with just this cherry picked ASAP for distros and vendors.
This patch is the work of a number of people over the past month: Eric, Reid,
Rui, and myself. I'm mailing it out as a single commit due to the time
sensitive nature of landing this and the need to backport it. Huge thanks to
everyone who helped out here, and everyone at Intel who helped out in
discussions about how to craft this. Also, credit goes to Paul Turner (at
Google, but not an LLVM contributor) for much of the underlying retpoline
design.
Reviewers: echristo, rnk, ruiu, craig.topper, DavidKreitzer
Subscribers: sanjoy, emaste, mcrosier, mgorny, mehdi_amini, hiraditya, llvm-commits
Differential Revision: https://reviews.llvm.org/D41723
llvm-svn: 323155
2018-01-23 06:05:25 +08:00
|
|
|
OpSize32, Requires<[Not64BitMode,NotUseRetpoline]>,
|
|
|
|
|
Sched<[WriteJump]>;
|
2012-07-05 07:53:27 +08:00
|
|
|
def CALL32m : I<0xFF, MRM2m, (outs), (ins i32mem:$dst),
|
2013-03-27 02:24:17 +08:00
|
|
|
"call{l}\t{*}$dst", [(X86call (loadi32 addr:$dst))],
|
2014-02-02 17:25:09 +08:00
|
|
|
IIC_CALL_MEM>, OpSize32,
|
Introduce the "retpoline" x86 mitigation technique for variant #2 of the speculative execution vulnerabilities disclosed today, specifically identified by CVE-2017-5715, "Branch Target Injection", and is one of the two halves to Spectre..
Summary:
First, we need to explain the core of the vulnerability. Note that this
is a very incomplete description, please see the Project Zero blog post
for details:
https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
The basis for branch target injection is to direct speculative execution
of the processor to some "gadget" of executable code by poisoning the
prediction of indirect branches with the address of that gadget. The
gadget in turn contains an operation that provides a side channel for
reading data. Most commonly, this will look like a load of secret data
followed by a branch on the loaded value and then a load of some
predictable cache line. The attacker then uses timing of the processors
cache to determine which direction the branch took *in the speculative
execution*, and in turn what one bit of the loaded value was. Due to the
nature of these timing side channels and the branch predictor on Intel
processors, this allows an attacker to leak data only accessible to
a privileged domain (like the kernel) back into an unprivileged domain.
The goal is simple: avoid generating code which contains an indirect
branch that could have its prediction poisoned by an attacker. In many
cases, the compiler can simply use directed conditional branches and
a small search tree. LLVM already has support for lowering switches in
this way and the first step of this patch is to disable jump-table
lowering of switches and introduce a pass to rewrite explicit indirectbr
sequences into a switch over integers.
However, there is no fully general alternative to indirect calls. We
introduce a new construct we call a "retpoline" to implement indirect
calls in a non-speculatable way. It can be thought of loosely as
a trampoline for indirect calls which uses the RET instruction on x86.
Further, we arrange for a specific call->ret sequence which ensures the
processor predicts the return to go to a controlled, known location. The
retpoline then "smashes" the return address pushed onto the stack by the
call with the desired target of the original indirect call. The result
is a predicted return to the next instruction after a call (which can be
used to trap speculative execution within an infinite loop) and an
actual indirect branch to an arbitrary address.
On 64-bit x86 ABIs, this is especially easily done in the compiler by
using a guaranteed scratch register to pass the target into this device.
For 32-bit ABIs there isn't a guaranteed scratch register and so several
different retpoline variants are introduced to use a scratch register if
one is available in the calling convention and to otherwise use direct
stack push/pop sequences to pass the target address.
This "retpoline" mitigation is fully described in the following blog
post: https://support.google.com/faqs/answer/7625886
We also support a target feature that disables emission of the retpoline
thunk by the compiler to allow for custom thunks if users want them.
These are particularly useful in environments like kernels that
routinely do hot-patching on boot and want to hot-patch their thunk to
different code sequences. They can write this custom thunk and use
`-mretpoline-external-thunk` *in addition* to `-mretpoline`. In this
case, on x86-64 thu thunk names must be:
```
__llvm_external_retpoline_r11
```
or on 32-bit:
```
__llvm_external_retpoline_eax
__llvm_external_retpoline_ecx
__llvm_external_retpoline_edx
__llvm_external_retpoline_push
```
And the target of the retpoline is passed in the named register, or in
the case of the `push` suffix on the top of the stack via a `pushl`
instruction.
There is one other important source of indirect branches in x86 ELF
binaries: the PLT. These patches also include support for LLD to
generate PLT entries that perform a retpoline-style indirection.
The only other indirect branches remaining that we are aware of are from
precompiled runtimes (such as crt0.o and similar). The ones we have
found are not really attackable, and so we have not focused on them
here, but eventually these runtimes should also be replicated for
retpoline-ed configurations for completeness.
For kernels or other freestanding or fully static executables, the
compiler switch `-mretpoline` is sufficient to fully mitigate this
particular attack. For dynamic executables, you must compile *all*
libraries with `-mretpoline` and additionally link the dynamic
executable and all shared libraries with LLD and pass `-z retpolineplt`
(or use similar functionality from some other linker). We strongly
recommend also using `-z now` as non-lazy binding allows the
retpoline-mitigated PLT to be substantially smaller.
When manually apply similar transformations to `-mretpoline` to the
Linux kernel we observed very small performance hits to applications
running typical workloads, and relatively minor hits (approximately 2%)
even for extremely syscall-heavy applications. This is largely due to
the small number of indirect branches that occur in performance
sensitive paths of the kernel.
When using these patches on statically linked applications, especially
C++ applications, you should expect to see a much more dramatic
performance hit. For microbenchmarks that are switch, indirect-, or
virtual-call heavy we have seen overheads ranging from 10% to 50%.
However, real-world workloads exhibit substantially lower performance
impact. Notably, techniques such as PGO and ThinLTO dramatically reduce
the impact of hot indirect calls (by speculatively promoting them to
direct calls) and allow optimized search trees to be used to lower
switches. If you need to deploy these techniques in C++ applications, we
*strongly* recommend that you ensure all hot call targets are statically
linked (avoiding PLT indirection) and use both PGO and ThinLTO. Well
tuned servers using all of these techniques saw 5% - 10% overhead from
the use of retpoline.
We will add detailed documentation covering these components in
subsequent patches, but wanted to make the core functionality available
as soon as possible. Happy for more code review, but we'd really like to
get these patches landed and backported ASAP for obvious reasons. We're
planning to backport this to both 6.0 and 5.0 release streams and get
a 5.0 release with just this cherry picked ASAP for distros and vendors.
This patch is the work of a number of people over the past month: Eric, Reid,
Rui, and myself. I'm mailing it out as a single commit due to the time
sensitive nature of landing this and the need to backport it. Huge thanks to
everyone who helped out here, and everyone at Intel who helped out in
discussions about how to craft this. Also, credit goes to Paul Turner (at
Google, but not an LLVM contributor) for much of the underlying retpoline
design.
Reviewers: echristo, rnk, ruiu, craig.topper, DavidKreitzer
Subscribers: sanjoy, emaste, mcrosier, mgorny, mehdi_amini, hiraditya, llvm-commits
Differential Revision: https://reviews.llvm.org/D41723
llvm-svn: 323155
2018-01-23 06:05:25 +08:00
|
|
|
Requires<[Not64BitMode,FavorMemIndirectCall,NotUseRetpoline]>,
|
2013-03-29 07:13:21 +08:00
|
|
|
Sched<[WriteJumpLd]>;
|
2011-01-26 10:03:37 +08:00
|
|
|
|
2014-12-20 15:43:27 +08:00
|
|
|
let Predicates = [Not64BitMode] in {
|
|
|
|
|
def FARCALL16i : Iseg16<0x9A, RawFrmImm16, (outs),
|
|
|
|
|
(ins i16imm:$off, i16imm:$seg),
|
2014-12-21 07:05:52 +08:00
|
|
|
"lcall{w}\t$seg, $off", [],
|
2014-12-20 15:43:27 +08:00
|
|
|
IIC_CALL_FAR_PTR>, OpSize16, Sched<[WriteJump]>;
|
|
|
|
|
def FARCALL32i : Iseg32<0x9A, RawFrmImm16, (outs),
|
|
|
|
|
(ins i32imm:$off, i16imm:$seg),
|
2014-12-21 07:05:52 +08:00
|
|
|
"lcall{l}\t$seg, $off", [],
|
2014-12-20 15:43:27 +08:00
|
|
|
IIC_CALL_FAR_PTR>, OpSize32, Sched<[WriteJump]>;
|
|
|
|
|
}
|
2011-01-26 10:03:37 +08:00
|
|
|
|
2010-10-05 14:04:14 +08:00
|
|
|
def FARCALL16m : I<0xFF, MRM3m, (outs), (ins opaque32mem:$dst),
|
2014-02-02 17:25:09 +08:00
|
|
|
"lcall{w}\t{*}$dst", [], IIC_CALL_FAR_MEM>, OpSize16,
|
2013-03-27 02:24:17 +08:00
|
|
|
Sched<[WriteJumpLd]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
def FARCALL32m : I<0xFF, MRM3m, (outs), (ins opaque48mem:$dst),
|
2017-08-09 23:34:55 +08:00
|
|
|
"{l}call{l}\t{*}$dst", [], IIC_CALL_FAR_MEM>, OpSize32,
|
2013-03-27 02:24:17 +08:00
|
|
|
Sched<[WriteJumpLd]>;
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Tail call stuff.
|
|
|
|
|
let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
|
2013-03-27 02:24:17 +08:00
|
|
|
isCodeGenOnly = 1, SchedRW = [WriteJumpLd] in
|
2017-11-26 21:02:45 +08:00
|
|
|
let Uses = [ESP, SSP] in {
|
2011-01-26 10:03:37 +08:00
|
|
|
def TCRETURNdi : PseudoI<(outs),
|
2017-10-08 16:32:56 +08:00
|
|
|
(ins i32imm_pcrel:$dst, i32imm:$offset), []>, NotMemoryFoldable;
|
2011-01-26 10:03:37 +08:00
|
|
|
def TCRETURNri : PseudoI<(outs),
|
2017-10-08 16:32:56 +08:00
|
|
|
(ins ptr_rc_tailcall:$dst, i32imm:$offset), []>, NotMemoryFoldable;
|
2010-10-05 14:04:14 +08:00
|
|
|
let mayLoad = 1 in
|
2011-01-26 10:03:37 +08:00
|
|
|
def TCRETURNmi : PseudoI<(outs),
|
2012-07-05 07:53:27 +08:00
|
|
|
(ins i32mem_TC:$dst, i32imm:$offset), []>;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
// FIXME: The should be pseudo instructions that are lowered when going to
|
|
|
|
|
// mcinst.
|
|
|
|
|
def TAILJMPd : Ii32PCRel<0xE9, RawFrm, (outs),
|
2012-07-05 07:53:27 +08:00
|
|
|
(ins i32imm_pcrel:$dst),
|
2015-01-31 05:03:31 +08:00
|
|
|
"jmp\t$dst",
|
2012-02-02 07:20:51 +08:00
|
|
|
[], IIC_JMP_REL>;
|
X86: Fold tail calls into conditional branches where possible (PR26302)
When branching to a block that immediately tail calls, it is possible to fold
the call directly into the branch if the call is direct and there is no stack
adjustment, saving one byte.
Example:
define void @f(i32 %x, i32 %y) {
entry:
%p = icmp eq i32 %x, %y
br i1 %p, label %bb1, label %bb2
bb1:
tail call void @foo()
ret void
bb2:
tail call void @bar()
ret void
}
before:
f:
movl 4(%esp), %eax
cmpl 8(%esp), %eax
jne .LBB0_2
jmp foo
.LBB0_2:
jmp bar
after:
f:
movl 4(%esp), %eax
cmpl 8(%esp), %eax
jne bar
.LBB0_1:
jmp foo
I don't expect any significant size savings from this (on a Clang bootstrap I
saw 288 bytes), but it does make the code a little tighter.
This patch only does 32-bit, but 64-bit would work similarly.
Differential Revision: https://reviews.llvm.org/D24108
llvm-svn: 280832
2016-09-08 01:52:14 +08:00
|
|
|
|
2012-07-05 07:53:27 +08:00
|
|
|
def TAILJMPr : I<0xFF, MRM4r, (outs), (ins ptr_rc_tailcall:$dst),
|
2012-02-02 07:20:51 +08:00
|
|
|
"", [], IIC_JMP_REG>; // FIXME: Remove encoding when JIT is dead.
|
2010-10-05 14:04:14 +08:00
|
|
|
let mayLoad = 1 in
|
2012-07-05 07:53:27 +08:00
|
|
|
def TAILJMPm : I<0xFF, MRM4m, (outs), (ins i32mem_TC:$dst),
|
2015-01-31 05:03:31 +08:00
|
|
|
"jmp{l}\t{*}$dst", [], IIC_JMP_MEM>;
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
|
|
|
|
|
2017-02-16 08:04:05 +08:00
|
|
|
// Conditional tail calls are similar to the above, but they are branches
|
|
|
|
|
// rather than barriers, and they use EFLAGS.
|
|
|
|
|
let isCall = 1, isTerminator = 1, isReturn = 1, isBranch = 1,
|
|
|
|
|
isCodeGenOnly = 1, SchedRW = [WriteJumpLd] in
|
2017-11-26 21:02:45 +08:00
|
|
|
let Uses = [ESP, EFLAGS, SSP] in {
|
2017-02-16 08:04:05 +08:00
|
|
|
def TCRETURNdicc : PseudoI<(outs),
|
|
|
|
|
(ins i32imm_pcrel:$dst, i32imm:$offset, i32imm:$cond), []>;
|
|
|
|
|
|
|
|
|
|
// This gets substituted to a conditional jump instruction in MC lowering.
|
|
|
|
|
def TAILJMPd_CC : Ii32PCRel<0x80, RawFrm, (outs),
|
|
|
|
|
(ins i32imm_pcrel:$dst, i32imm:$cond),
|
|
|
|
|
"",
|
|
|
|
|
[], IIC_JMP_REL>;
|
|
|
|
|
}
|
|
|
|
|
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
// Call Instructions...
|
|
|
|
|
//
|
|
|
|
|
|
2012-02-17 01:56:02 +08:00
|
|
|
// RSP is marked as a use to prevent stack-pointer assignments that appear
|
|
|
|
|
// immediately before calls from potentially appearing dead. Uses for argument
|
|
|
|
|
// registers are added manually.
|
2017-11-26 21:02:45 +08:00
|
|
|
let isCall = 1, Uses = [RSP, SSP], SchedRW = [WriteJump] in {
|
2012-02-17 01:56:02 +08:00
|
|
|
// NOTE: this pattern doesn't match "X86call imm", because we do not know
|
|
|
|
|
// that the offset between an arbitrary immediate and the call will fit in
|
|
|
|
|
// the 32-bit pcrel field that we have.
|
|
|
|
|
def CALL64pcrel32 : Ii32PCRel<0xE8, RawFrm,
|
2012-07-05 07:53:27 +08:00
|
|
|
(outs), (ins i64i32imm_pcrel:$dst),
|
2014-02-18 16:18:29 +08:00
|
|
|
"call{q}\t$dst", [], IIC_CALL_RI>, OpSize32,
|
2012-02-17 01:56:02 +08:00
|
|
|
Requires<[In64BitMode]>;
|
2012-07-05 07:53:27 +08:00
|
|
|
def CALL64r : I<0xFF, MRM2r, (outs), (ins GR64:$dst),
|
2012-02-17 01:56:02 +08:00
|
|
|
"call{q}\t{*}$dst", [(X86call GR64:$dst)],
|
|
|
|
|
IIC_CALL_RI>,
|
Introduce the "retpoline" x86 mitigation technique for variant #2 of the speculative execution vulnerabilities disclosed today, specifically identified by CVE-2017-5715, "Branch Target Injection", and is one of the two halves to Spectre..
Summary:
First, we need to explain the core of the vulnerability. Note that this
is a very incomplete description, please see the Project Zero blog post
for details:
https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
The basis for branch target injection is to direct speculative execution
of the processor to some "gadget" of executable code by poisoning the
prediction of indirect branches with the address of that gadget. The
gadget in turn contains an operation that provides a side channel for
reading data. Most commonly, this will look like a load of secret data
followed by a branch on the loaded value and then a load of some
predictable cache line. The attacker then uses timing of the processors
cache to determine which direction the branch took *in the speculative
execution*, and in turn what one bit of the loaded value was. Due to the
nature of these timing side channels and the branch predictor on Intel
processors, this allows an attacker to leak data only accessible to
a privileged domain (like the kernel) back into an unprivileged domain.
The goal is simple: avoid generating code which contains an indirect
branch that could have its prediction poisoned by an attacker. In many
cases, the compiler can simply use directed conditional branches and
a small search tree. LLVM already has support for lowering switches in
this way and the first step of this patch is to disable jump-table
lowering of switches and introduce a pass to rewrite explicit indirectbr
sequences into a switch over integers.
However, there is no fully general alternative to indirect calls. We
introduce a new construct we call a "retpoline" to implement indirect
calls in a non-speculatable way. It can be thought of loosely as
a trampoline for indirect calls which uses the RET instruction on x86.
Further, we arrange for a specific call->ret sequence which ensures the
processor predicts the return to go to a controlled, known location. The
retpoline then "smashes" the return address pushed onto the stack by the
call with the desired target of the original indirect call. The result
is a predicted return to the next instruction after a call (which can be
used to trap speculative execution within an infinite loop) and an
actual indirect branch to an arbitrary address.
On 64-bit x86 ABIs, this is especially easily done in the compiler by
using a guaranteed scratch register to pass the target into this device.
For 32-bit ABIs there isn't a guaranteed scratch register and so several
different retpoline variants are introduced to use a scratch register if
one is available in the calling convention and to otherwise use direct
stack push/pop sequences to pass the target address.
This "retpoline" mitigation is fully described in the following blog
post: https://support.google.com/faqs/answer/7625886
We also support a target feature that disables emission of the retpoline
thunk by the compiler to allow for custom thunks if users want them.
These are particularly useful in environments like kernels that
routinely do hot-patching on boot and want to hot-patch their thunk to
different code sequences. They can write this custom thunk and use
`-mretpoline-external-thunk` *in addition* to `-mretpoline`. In this
case, on x86-64 thu thunk names must be:
```
__llvm_external_retpoline_r11
```
or on 32-bit:
```
__llvm_external_retpoline_eax
__llvm_external_retpoline_ecx
__llvm_external_retpoline_edx
__llvm_external_retpoline_push
```
And the target of the retpoline is passed in the named register, or in
the case of the `push` suffix on the top of the stack via a `pushl`
instruction.
There is one other important source of indirect branches in x86 ELF
binaries: the PLT. These patches also include support for LLD to
generate PLT entries that perform a retpoline-style indirection.
The only other indirect branches remaining that we are aware of are from
precompiled runtimes (such as crt0.o and similar). The ones we have
found are not really attackable, and so we have not focused on them
here, but eventually these runtimes should also be replicated for
retpoline-ed configurations for completeness.
For kernels or other freestanding or fully static executables, the
compiler switch `-mretpoline` is sufficient to fully mitigate this
particular attack. For dynamic executables, you must compile *all*
libraries with `-mretpoline` and additionally link the dynamic
executable and all shared libraries with LLD and pass `-z retpolineplt`
(or use similar functionality from some other linker). We strongly
recommend also using `-z now` as non-lazy binding allows the
retpoline-mitigated PLT to be substantially smaller.
When manually apply similar transformations to `-mretpoline` to the
Linux kernel we observed very small performance hits to applications
running typical workloads, and relatively minor hits (approximately 2%)
even for extremely syscall-heavy applications. This is largely due to
the small number of indirect branches that occur in performance
sensitive paths of the kernel.
When using these patches on statically linked applications, especially
C++ applications, you should expect to see a much more dramatic
performance hit. For microbenchmarks that are switch, indirect-, or
virtual-call heavy we have seen overheads ranging from 10% to 50%.
However, real-world workloads exhibit substantially lower performance
impact. Notably, techniques such as PGO and ThinLTO dramatically reduce
the impact of hot indirect calls (by speculatively promoting them to
direct calls) and allow optimized search trees to be used to lower
switches. If you need to deploy these techniques in C++ applications, we
*strongly* recommend that you ensure all hot call targets are statically
linked (avoiding PLT indirection) and use both PGO and ThinLTO. Well
tuned servers using all of these techniques saw 5% - 10% overhead from
the use of retpoline.
We will add detailed documentation covering these components in
subsequent patches, but wanted to make the core functionality available
as soon as possible. Happy for more code review, but we'd really like to
get these patches landed and backported ASAP for obvious reasons. We're
planning to backport this to both 6.0 and 5.0 release streams and get
a 5.0 release with just this cherry picked ASAP for distros and vendors.
This patch is the work of a number of people over the past month: Eric, Reid,
Rui, and myself. I'm mailing it out as a single commit due to the time
sensitive nature of landing this and the need to backport it. Huge thanks to
everyone who helped out here, and everyone at Intel who helped out in
discussions about how to craft this. Also, credit goes to Paul Turner (at
Google, but not an LLVM contributor) for much of the underlying retpoline
design.
Reviewers: echristo, rnk, ruiu, craig.topper, DavidKreitzer
Subscribers: sanjoy, emaste, mcrosier, mgorny, mehdi_amini, hiraditya, llvm-commits
Differential Revision: https://reviews.llvm.org/D41723
llvm-svn: 323155
2018-01-23 06:05:25 +08:00
|
|
|
Requires<[In64BitMode,NotUseRetpoline]>;
|
2012-07-05 07:53:27 +08:00
|
|
|
def CALL64m : I<0xFF, MRM2m, (outs), (ins i64mem:$dst),
|
2012-02-17 01:56:02 +08:00
|
|
|
"call{q}\t{*}$dst", [(X86call (loadi64 addr:$dst))],
|
|
|
|
|
IIC_CALL_MEM>,
|
Introduce the "retpoline" x86 mitigation technique for variant #2 of the speculative execution vulnerabilities disclosed today, specifically identified by CVE-2017-5715, "Branch Target Injection", and is one of the two halves to Spectre..
Summary:
First, we need to explain the core of the vulnerability. Note that this
is a very incomplete description, please see the Project Zero blog post
for details:
https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
The basis for branch target injection is to direct speculative execution
of the processor to some "gadget" of executable code by poisoning the
prediction of indirect branches with the address of that gadget. The
gadget in turn contains an operation that provides a side channel for
reading data. Most commonly, this will look like a load of secret data
followed by a branch on the loaded value and then a load of some
predictable cache line. The attacker then uses timing of the processors
cache to determine which direction the branch took *in the speculative
execution*, and in turn what one bit of the loaded value was. Due to the
nature of these timing side channels and the branch predictor on Intel
processors, this allows an attacker to leak data only accessible to
a privileged domain (like the kernel) back into an unprivileged domain.
The goal is simple: avoid generating code which contains an indirect
branch that could have its prediction poisoned by an attacker. In many
cases, the compiler can simply use directed conditional branches and
a small search tree. LLVM already has support for lowering switches in
this way and the first step of this patch is to disable jump-table
lowering of switches and introduce a pass to rewrite explicit indirectbr
sequences into a switch over integers.
However, there is no fully general alternative to indirect calls. We
introduce a new construct we call a "retpoline" to implement indirect
calls in a non-speculatable way. It can be thought of loosely as
a trampoline for indirect calls which uses the RET instruction on x86.
Further, we arrange for a specific call->ret sequence which ensures the
processor predicts the return to go to a controlled, known location. The
retpoline then "smashes" the return address pushed onto the stack by the
call with the desired target of the original indirect call. The result
is a predicted return to the next instruction after a call (which can be
used to trap speculative execution within an infinite loop) and an
actual indirect branch to an arbitrary address.
On 64-bit x86 ABIs, this is especially easily done in the compiler by
using a guaranteed scratch register to pass the target into this device.
For 32-bit ABIs there isn't a guaranteed scratch register and so several
different retpoline variants are introduced to use a scratch register if
one is available in the calling convention and to otherwise use direct
stack push/pop sequences to pass the target address.
This "retpoline" mitigation is fully described in the following blog
post: https://support.google.com/faqs/answer/7625886
We also support a target feature that disables emission of the retpoline
thunk by the compiler to allow for custom thunks if users want them.
These are particularly useful in environments like kernels that
routinely do hot-patching on boot and want to hot-patch their thunk to
different code sequences. They can write this custom thunk and use
`-mretpoline-external-thunk` *in addition* to `-mretpoline`. In this
case, on x86-64 thu thunk names must be:
```
__llvm_external_retpoline_r11
```
or on 32-bit:
```
__llvm_external_retpoline_eax
__llvm_external_retpoline_ecx
__llvm_external_retpoline_edx
__llvm_external_retpoline_push
```
And the target of the retpoline is passed in the named register, or in
the case of the `push` suffix on the top of the stack via a `pushl`
instruction.
There is one other important source of indirect branches in x86 ELF
binaries: the PLT. These patches also include support for LLD to
generate PLT entries that perform a retpoline-style indirection.
The only other indirect branches remaining that we are aware of are from
precompiled runtimes (such as crt0.o and similar). The ones we have
found are not really attackable, and so we have not focused on them
here, but eventually these runtimes should also be replicated for
retpoline-ed configurations for completeness.
For kernels or other freestanding or fully static executables, the
compiler switch `-mretpoline` is sufficient to fully mitigate this
particular attack. For dynamic executables, you must compile *all*
libraries with `-mretpoline` and additionally link the dynamic
executable and all shared libraries with LLD and pass `-z retpolineplt`
(or use similar functionality from some other linker). We strongly
recommend also using `-z now` as non-lazy binding allows the
retpoline-mitigated PLT to be substantially smaller.
When manually apply similar transformations to `-mretpoline` to the
Linux kernel we observed very small performance hits to applications
running typical workloads, and relatively minor hits (approximately 2%)
even for extremely syscall-heavy applications. This is largely due to
the small number of indirect branches that occur in performance
sensitive paths of the kernel.
When using these patches on statically linked applications, especially
C++ applications, you should expect to see a much more dramatic
performance hit. For microbenchmarks that are switch, indirect-, or
virtual-call heavy we have seen overheads ranging from 10% to 50%.
However, real-world workloads exhibit substantially lower performance
impact. Notably, techniques such as PGO and ThinLTO dramatically reduce
the impact of hot indirect calls (by speculatively promoting them to
direct calls) and allow optimized search trees to be used to lower
switches. If you need to deploy these techniques in C++ applications, we
*strongly* recommend that you ensure all hot call targets are statically
linked (avoiding PLT indirection) and use both PGO and ThinLTO. Well
tuned servers using all of these techniques saw 5% - 10% overhead from
the use of retpoline.
We will add detailed documentation covering these components in
subsequent patches, but wanted to make the core functionality available
as soon as possible. Happy for more code review, but we'd really like to
get these patches landed and backported ASAP for obvious reasons. We're
planning to backport this to both 6.0 and 5.0 release streams and get
a 5.0 release with just this cherry picked ASAP for distros and vendors.
This patch is the work of a number of people over the past month: Eric, Reid,
Rui, and myself. I'm mailing it out as a single commit due to the time
sensitive nature of landing this and the need to backport it. Huge thanks to
everyone who helped out here, and everyone at Intel who helped out in
discussions about how to craft this. Also, credit goes to Paul Turner (at
Google, but not an LLVM contributor) for much of the underlying retpoline
design.
Reviewers: echristo, rnk, ruiu, craig.topper, DavidKreitzer
Subscribers: sanjoy, emaste, mcrosier, mgorny, mehdi_amini, hiraditya, llvm-commits
Differential Revision: https://reviews.llvm.org/D41723
llvm-svn: 323155
2018-01-23 06:05:25 +08:00
|
|
|
Requires<[In64BitMode,FavorMemIndirectCall,
|
|
|
|
|
NotUseRetpoline]>;
|
2012-02-17 01:56:02 +08:00
|
|
|
|
|
|
|
|
def FARCALL64 : RI<0xFF, MRM3m, (outs), (ins opaque80mem:$dst),
|
|
|
|
|
"lcall{q}\t{*}$dst", [], IIC_CALL_FAR_MEM>;
|
|
|
|
|
}
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
|
2018-01-04 02:20:36 +08:00
|
|
|
isCodeGenOnly = 1, Uses = [RSP, SSP], SchedRW = [WriteJump] in {
|
2016-09-10 06:37:27 +08:00
|
|
|
def TCRETURNdi64 : PseudoI<(outs),
|
|
|
|
|
(ins i64i32imm_pcrel:$dst, i32imm:$offset),
|
|
|
|
|
[]>;
|
|
|
|
|
def TCRETURNri64 : PseudoI<(outs),
|
2017-10-08 16:32:56 +08:00
|
|
|
(ins ptr_rc_tailcall:$dst, i32imm:$offset), []>, NotMemoryFoldable;
|
2010-10-05 14:04:14 +08:00
|
|
|
let mayLoad = 1 in
|
2016-09-10 06:37:27 +08:00
|
|
|
def TCRETURNmi64 : PseudoI<(outs),
|
2017-10-08 16:32:56 +08:00
|
|
|
(ins i64mem_TC:$dst, i32imm:$offset), []>, NotMemoryFoldable;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
2015-01-31 05:03:31 +08:00
|
|
|
def TAILJMPd64 : Ii32PCRel<0xE9, RawFrm, (outs), (ins i64i32imm_pcrel:$dst),
|
|
|
|
|
"jmp\t$dst", [], IIC_JMP_REL>;
|
2016-09-10 06:37:27 +08:00
|
|
|
|
2012-07-05 07:53:27 +08:00
|
|
|
def TAILJMPr64 : I<0xFF, MRM4r, (outs), (ins ptr_rc_tailcall:$dst),
|
2015-01-31 05:03:31 +08:00
|
|
|
"jmp{q}\t{*}$dst", [], IIC_JMP_MEM>;
|
2010-10-05 14:04:14 +08:00
|
|
|
|
|
|
|
|
let mayLoad = 1 in
|
2012-07-05 07:53:27 +08:00
|
|
|
def TAILJMPm64 : I<0xFF, MRM4m, (outs), (ins i64mem_TC:$dst),
|
2015-01-31 05:03:31 +08:00
|
|
|
"jmp{q}\t{*}$dst", [], IIC_JMP_MEM>;
|
|
|
|
|
|
2016-09-09 07:35:10 +08:00
|
|
|
// Win64 wants indirect jumps leaving the function to have a REX_W prefix.
|
2015-01-31 05:03:31 +08:00
|
|
|
let hasREX_WPrefix = 1 in {
|
|
|
|
|
def TAILJMPr64_REX : I<0xFF, MRM4r, (outs), (ins ptr_rc_tailcall:$dst),
|
|
|
|
|
"rex64 jmp{q}\t{*}$dst", [], IIC_JMP_MEM>;
|
|
|
|
|
|
|
|
|
|
let mayLoad = 1 in
|
|
|
|
|
def TAILJMPm64_REX : I<0xFF, MRM4m, (outs), (ins i64mem_TC:$dst),
|
|
|
|
|
"rex64 jmp{q}\t{*}$dst", [], IIC_JMP_MEM>;
|
|
|
|
|
}
|
2010-10-05 14:04:14 +08:00
|
|
|
}
|
2017-02-16 08:04:05 +08:00
|
|
|
|
Introduce the "retpoline" x86 mitigation technique for variant #2 of the speculative execution vulnerabilities disclosed today, specifically identified by CVE-2017-5715, "Branch Target Injection", and is one of the two halves to Spectre..
Summary:
First, we need to explain the core of the vulnerability. Note that this
is a very incomplete description, please see the Project Zero blog post
for details:
https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
The basis for branch target injection is to direct speculative execution
of the processor to some "gadget" of executable code by poisoning the
prediction of indirect branches with the address of that gadget. The
gadget in turn contains an operation that provides a side channel for
reading data. Most commonly, this will look like a load of secret data
followed by a branch on the loaded value and then a load of some
predictable cache line. The attacker then uses timing of the processors
cache to determine which direction the branch took *in the speculative
execution*, and in turn what one bit of the loaded value was. Due to the
nature of these timing side channels and the branch predictor on Intel
processors, this allows an attacker to leak data only accessible to
a privileged domain (like the kernel) back into an unprivileged domain.
The goal is simple: avoid generating code which contains an indirect
branch that could have its prediction poisoned by an attacker. In many
cases, the compiler can simply use directed conditional branches and
a small search tree. LLVM already has support for lowering switches in
this way and the first step of this patch is to disable jump-table
lowering of switches and introduce a pass to rewrite explicit indirectbr
sequences into a switch over integers.
However, there is no fully general alternative to indirect calls. We
introduce a new construct we call a "retpoline" to implement indirect
calls in a non-speculatable way. It can be thought of loosely as
a trampoline for indirect calls which uses the RET instruction on x86.
Further, we arrange for a specific call->ret sequence which ensures the
processor predicts the return to go to a controlled, known location. The
retpoline then "smashes" the return address pushed onto the stack by the
call with the desired target of the original indirect call. The result
is a predicted return to the next instruction after a call (which can be
used to trap speculative execution within an infinite loop) and an
actual indirect branch to an arbitrary address.
On 64-bit x86 ABIs, this is especially easily done in the compiler by
using a guaranteed scratch register to pass the target into this device.
For 32-bit ABIs there isn't a guaranteed scratch register and so several
different retpoline variants are introduced to use a scratch register if
one is available in the calling convention and to otherwise use direct
stack push/pop sequences to pass the target address.
This "retpoline" mitigation is fully described in the following blog
post: https://support.google.com/faqs/answer/7625886
We also support a target feature that disables emission of the retpoline
thunk by the compiler to allow for custom thunks if users want them.
These are particularly useful in environments like kernels that
routinely do hot-patching on boot and want to hot-patch their thunk to
different code sequences. They can write this custom thunk and use
`-mretpoline-external-thunk` *in addition* to `-mretpoline`. In this
case, on x86-64 thu thunk names must be:
```
__llvm_external_retpoline_r11
```
or on 32-bit:
```
__llvm_external_retpoline_eax
__llvm_external_retpoline_ecx
__llvm_external_retpoline_edx
__llvm_external_retpoline_push
```
And the target of the retpoline is passed in the named register, or in
the case of the `push` suffix on the top of the stack via a `pushl`
instruction.
There is one other important source of indirect branches in x86 ELF
binaries: the PLT. These patches also include support for LLD to
generate PLT entries that perform a retpoline-style indirection.
The only other indirect branches remaining that we are aware of are from
precompiled runtimes (such as crt0.o and similar). The ones we have
found are not really attackable, and so we have not focused on them
here, but eventually these runtimes should also be replicated for
retpoline-ed configurations for completeness.
For kernels or other freestanding or fully static executables, the
compiler switch `-mretpoline` is sufficient to fully mitigate this
particular attack. For dynamic executables, you must compile *all*
libraries with `-mretpoline` and additionally link the dynamic
executable and all shared libraries with LLD and pass `-z retpolineplt`
(or use similar functionality from some other linker). We strongly
recommend also using `-z now` as non-lazy binding allows the
retpoline-mitigated PLT to be substantially smaller.
When manually apply similar transformations to `-mretpoline` to the
Linux kernel we observed very small performance hits to applications
running typical workloads, and relatively minor hits (approximately 2%)
even for extremely syscall-heavy applications. This is largely due to
the small number of indirect branches that occur in performance
sensitive paths of the kernel.
When using these patches on statically linked applications, especially
C++ applications, you should expect to see a much more dramatic
performance hit. For microbenchmarks that are switch, indirect-, or
virtual-call heavy we have seen overheads ranging from 10% to 50%.
However, real-world workloads exhibit substantially lower performance
impact. Notably, techniques such as PGO and ThinLTO dramatically reduce
the impact of hot indirect calls (by speculatively promoting them to
direct calls) and allow optimized search trees to be used to lower
switches. If you need to deploy these techniques in C++ applications, we
*strongly* recommend that you ensure all hot call targets are statically
linked (avoiding PLT indirection) and use both PGO and ThinLTO. Well
tuned servers using all of these techniques saw 5% - 10% overhead from
the use of retpoline.
We will add detailed documentation covering these components in
subsequent patches, but wanted to make the core functionality available
as soon as possible. Happy for more code review, but we'd really like to
get these patches landed and backported ASAP for obvious reasons. We're
planning to backport this to both 6.0 and 5.0 release streams and get
a 5.0 release with just this cherry picked ASAP for distros and vendors.
This patch is the work of a number of people over the past month: Eric, Reid,
Rui, and myself. I'm mailing it out as a single commit due to the time
sensitive nature of landing this and the need to backport it. Huge thanks to
everyone who helped out here, and everyone at Intel who helped out in
discussions about how to craft this. Also, credit goes to Paul Turner (at
Google, but not an LLVM contributor) for much of the underlying retpoline
design.
Reviewers: echristo, rnk, ruiu, craig.topper, DavidKreitzer
Subscribers: sanjoy, emaste, mcrosier, mgorny, mehdi_amini, hiraditya, llvm-commits
Differential Revision: https://reviews.llvm.org/D41723
llvm-svn: 323155
2018-01-23 06:05:25 +08:00
|
|
|
let isPseudo = 1, isCall = 1, isCodeGenOnly = 1,
|
|
|
|
|
Uses = [RSP, SSP],
|
|
|
|
|
usesCustomInserter = 1,
|
|
|
|
|
SchedRW = [WriteJump] in {
|
|
|
|
|
def RETPOLINE_CALL32 :
|
|
|
|
|
PseudoI<(outs), (ins GR32:$dst), [(X86call GR32:$dst)]>,
|
|
|
|
|
Requires<[Not64BitMode,UseRetpoline]>;
|
|
|
|
|
|
|
|
|
|
def RETPOLINE_CALL64 :
|
|
|
|
|
PseudoI<(outs), (ins GR64:$dst), [(X86call GR64:$dst)]>,
|
|
|
|
|
Requires<[In64BitMode,UseRetpoline]>;
|
|
|
|
|
|
|
|
|
|
// Retpoline variant of indirect tail calls.
|
|
|
|
|
let isTerminator = 1, isReturn = 1, isBarrier = 1 in {
|
|
|
|
|
def RETPOLINE_TCRETURN64 :
|
|
|
|
|
PseudoI<(outs), (ins GR64:$dst, i32imm:$offset), []>;
|
|
|
|
|
def RETPOLINE_TCRETURN32 :
|
|
|
|
|
PseudoI<(outs), (ins GR32:$dst, i32imm:$offset), []>;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2017-02-16 08:04:05 +08:00
|
|
|
// Conditional tail calls are similar to the above, but they are branches
|
|
|
|
|
// rather than barriers, and they use EFLAGS.
|
|
|
|
|
let isCall = 1, isTerminator = 1, isReturn = 1, isBranch = 1,
|
|
|
|
|
isCodeGenOnly = 1, SchedRW = [WriteJumpLd] in
|
2017-11-26 21:02:45 +08:00
|
|
|
let Uses = [RSP, EFLAGS, SSP] in {
|
2017-02-16 08:04:05 +08:00
|
|
|
def TCRETURNdi64cc : PseudoI<(outs),
|
|
|
|
|
(ins i64i32imm_pcrel:$dst, i32imm:$offset,
|
|
|
|
|
i32imm:$cond), []>;
|
|
|
|
|
|
|
|
|
|
// This gets substituted to a conditional jump instruction in MC lowering.
|
|
|
|
|
def TAILJMPd64_CC : Ii32PCRel<0x80, RawFrm, (outs),
|
|
|
|
|
(ins i64i32imm_pcrel:$dst, i32imm:$cond),
|
|
|
|
|
"",
|
|
|
|
|
[], IIC_JMP_REL>;
|
|
|
|
|
}
|