Visibly fail if attempting to encode register AH,BH,CH,DH in a REX-prefixed instruction.

Differential Revision: http://reviews.llvm.org/D13316
Fixes PR25003

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@252743 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index ff7597b29677b73ccef17dbc9a1d79103d449f2a..dfab6ec10775c0417302fdd574f05bebc2cbd695 100644 (file)
--- a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -988,6 +988,8 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
 static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
                                    const MCInstrDesc &Desc) {
   unsigned REX = 0;
+  bool UsesHighByteReg = false;
+
   if (TSFlags & X86II::REX_W)
     REX |= 1 << 3; // set REX.W
 
@@ -1004,6 +1006,8 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
     const MCOperand &MO = MI.getOperand(i);
     if (!MO.isReg()) continue;
     unsigned Reg = MO.getReg();
+    if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
+      UsesHighByteReg = true;
     if (!X86II::isX86_64NonExtLowByteReg(Reg)) continue;
     // FIXME: The caller of DetermineREXPrefix slaps this prefix onto anything
     // that returns non-zero.
@@ -1073,6 +1077,9 @@ static unsigned DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
     }
     break;
   }
+  if (REX && UsesHighByteReg)
+    report_fatal_error("Cannot encode high byte register in REX-prefixed instruction");
+
   return REX;
 }
 

diff --git a/test/MC/X86/encoder-fail.s b/test/MC/X86/encoder-fail.s
new file mode 100644 (file)
index 0000000..3e845fe
--- /dev/null
+++ b/test/MC/X86/encoder-fail.s
@@ -0,0 +1,3 @@
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2>&1 | FileCheck %s
+// CHECK: LLVM ERROR: Cannot encode high byte register in REX-prefixed instruction
+ movzx %dh, %rsi