AArch64/ARM64: add support for :abs_gN_s: MOVZ modifiers
[oota-llvm.git] / lib / Target / ARM64 / AsmParser / ARM64AsmParser.cpp
index 705d192356e029ef36991219c6a1fbf80209ef41..f9d3c8b898acbb02eba08da7c9fd87abeeeddcff 100644 (file)
@@ -8,8 +8,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/ARM64AddressingModes.h"
-#include "MCTargetDesc/ARM64BaseInfo.h"
 #include "MCTargetDesc/ARM64MCExpr.h"
+#include "Utils/ARM64BaseInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
@@ -55,7 +55,7 @@ private:
   unsigned parseCondCodeString(StringRef Cond);
   bool parseCondCode(OperandVector &Operands, bool invertCondCode);
   int tryParseRegister();
-  int tryMatchVectorRegister(StringRef &Kind);
+  int tryMatchVectorRegister(StringRef &Kind, bool expected);
   bool parseOptionalShift(OperandVector &Operands);
   bool parseOptionalExtend(OperandVector &Operands);
   bool parseRegister(OperandVector &Operands);
@@ -88,8 +88,8 @@ private:
 
   OperandMatchResultTy tryParseNoIndexMemory(OperandVector &Operands);
   OperandMatchResultTy tryParseBarrierOperand(OperandVector &Operands);
-  OperandMatchResultTy tryParseSystemRegister(OperandVector &Operands);
-  OperandMatchResultTy tryParseCPSRField(OperandVector &Operands);
+  OperandMatchResultTy tryParseMRSSystemRegister(OperandVector &Operands);
+  OperandMatchResultTy tryParseSysReg(OperandVector &Operands);
   OperandMatchResultTy tryParseSysCROperand(OperandVector &Operands);
   OperandMatchResultTy tryParsePrefetch(OperandVector &Operands);
   OperandMatchResultTy tryParseAdrpLabel(OperandVector &Operands);
@@ -104,9 +104,13 @@ public:
 #include "ARM64GenAsmMatcher.inc"
   };
   ARM64AsmParser(MCSubtargetInfo &_STI, MCAsmParser &_Parser,
-                 const MCInstrInfo &MII)
+                 const MCInstrInfo &MII,
+                 const MCTargetOptions &Options)
       : MCTargetAsmParser(), STI(_STI), Parser(_Parser) {
     MCAsmParserExtension::Initialize(_Parser);
+
+    // Initialize the set of available features.
+    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
 
   virtual bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
@@ -141,14 +145,13 @@ private:
     k_VectorList,
     k_VectorIndex,
     k_Token,
+    k_SysReg,
     k_SysCR,
     k_Prefetch,
     k_Shifter,
     k_Extend,
     k_FPImm,
-    k_Barrier,
-    k_SystemRegister,
-    k_CPSRField
+    k_Barrier
   } Kind;
 
   SMLoc StartLoc, EndLoc, OffsetLoc;
@@ -187,14 +190,9 @@ private:
     unsigned Val; // Not the enum since not all values have names.
   };
 
-  struct SystemRegisterOp {
-    // 16-bit immediate, usually from the ARM64SYS::SystermRegister enum,
-    // but not limited to those values.
-    uint16_t Val;
-  };
-
-  struct CPSRFieldOp {
-    ARM64SYS::CPSRField Field;
+  struct SysRegOp {
+    const char *Data;
+    unsigned Length;
   };
 
   struct SysCRImmOp {
@@ -231,8 +229,7 @@ private:
     struct ImmOp Imm;
     struct FPImmOp FPImm;
     struct BarrierOp Barrier;
-    struct SystemRegisterOp SystemRegister;
-    struct CPSRFieldOp CPSRField;
+    struct SysRegOp SysReg;
     struct SysCRImmOp SysCRImm;
     struct PrefetchOp Prefetch;
     struct ShifterOp Shifter;
@@ -265,12 +262,6 @@ public:
     case k_Barrier:
       Barrier = o.Barrier;
       break;
-    case k_SystemRegister:
-      SystemRegister = o.SystemRegister;
-      break;
-    case k_CPSRField:
-      CPSRField = o.CPSRField;
-      break;
     case k_Register:
       Reg = o.Reg;
       break;
@@ -280,6 +271,9 @@ public:
     case k_VectorIndex:
       VectorIndex = o.VectorIndex;
       break;
+    case k_SysReg:
+      SysReg = o.SysReg;
+      break;
     case k_SysCR:
       SysCRImm = o.SysCRImm;
       break;
@@ -330,16 +324,6 @@ public:
     return Barrier.Val;
   }
 
-  uint16_t getSystemRegister() const {
-    assert(Kind == k_SystemRegister && "Invalid access!");
-    return SystemRegister.Val;
-  }
-
-  ARM64SYS::CPSRField getCPSRField() const {
-    assert(Kind == k_CPSRField && "Invalid access!");
-    return CPSRField.Field;
-  }
-
   unsigned getReg() const {
     assert(Kind == k_Register && "Invalid access!");
     return Reg.RegNum;
@@ -360,6 +344,11 @@ public:
     return VectorIndex.Val;
   }
 
+  StringRef getSysReg() const {
+    assert(Kind == k_SysReg && "Invalid access!");
+    return StringRef(SysReg.Data, SysReg.Length);
+  }
+
   unsigned getSysCR() const {
     assert(Kind == k_SysCR && "Invalid access!");
     return SysCRImm.Val;
@@ -569,7 +558,7 @@ public:
       return false;
     return (Val >= -(0x2000000 << 2) && Val <= (0x1ffffff << 2));
   }
-  bool isBranchTarget19() const {
+  bool isPCRelLabel19() const {
     if (!isImm())
       return false;
     const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
@@ -621,6 +610,7 @@ public:
 
   bool isMovZSymbolG2() const {
     static ARM64MCExpr::VariantKind Variants[] = { ARM64MCExpr::VK_ABS_G2,
+                                                   ARM64MCExpr::VK_ABS_G2_S,
                                                    ARM64MCExpr::VK_TPREL_G2,
                                                    ARM64MCExpr::VK_DTPREL_G2 };
     return isMovWSymbol(Variants);
@@ -628,6 +618,7 @@ public:
 
   bool isMovZSymbolG1() const {
     static ARM64MCExpr::VariantKind Variants[] = { ARM64MCExpr::VK_ABS_G1,
+                                                   ARM64MCExpr::VK_ABS_G1_S,
                                                    ARM64MCExpr::VK_GOTTPREL_G1,
                                                    ARM64MCExpr::VK_TPREL_G1,
                                                    ARM64MCExpr::VK_DTPREL_G1, };
@@ -636,11 +627,17 @@ public:
 
   bool isMovZSymbolG0() const {
     static ARM64MCExpr::VariantKind Variants[] = { ARM64MCExpr::VK_ABS_G0,
+                                                   ARM64MCExpr::VK_ABS_G0_S,
                                                    ARM64MCExpr::VK_TPREL_G0,
                                                    ARM64MCExpr::VK_DTPREL_G0 };
     return isMovWSymbol(Variants);
   }
 
+  bool isMovKSymbolG3() const {
+    static ARM64MCExpr::VariantKind Variants[] = { ARM64MCExpr::VK_ABS_G3 };
+    return isMovWSymbol(Variants);
+  }
+
   bool isMovKSymbolG2() const {
     static ARM64MCExpr::VariantKind Variants[] = { ARM64MCExpr::VK_ABS_G2_NC };
     return isMovWSymbol(Variants);
@@ -664,14 +661,31 @@ public:
 
   bool isFPImm() const { return Kind == k_FPImm; }
   bool isBarrier() const { return Kind == k_Barrier; }
-  bool isSystemRegister() const {
-    if (Kind == k_SystemRegister)
-      return true;
-    // SPSel is legal for both the system register and the CPSR-field
-    // variants of MSR, so special case that. Fugly.
-    return (Kind == k_CPSRField && getCPSRField() == ARM64SYS::cpsr_SPSel);
+  bool isSysReg() const { return Kind == k_SysReg; }
+  bool isMRSSystemRegister() const {
+    if (!isSysReg()) return false;
+
+    bool IsKnownRegister;
+    ARM64SysReg::MRSMapper().fromString(getSysReg(), IsKnownRegister);
+
+    return IsKnownRegister;
+  }
+  bool isMSRSystemRegister() const {
+    if (!isSysReg()) return false;
+
+    bool IsKnownRegister;
+    ARM64SysReg::MSRMapper().fromString(getSysReg(), IsKnownRegister);
+
+    return IsKnownRegister;
+  }
+  bool isSystemCPSRField() const {
+    if (!isSysReg()) return false;
+
+    bool IsKnownRegister;
+    ARM64PState::PStateMapper().fromString(getSysReg(), IsKnownRegister);
+
+    return IsKnownRegister;
   }
-  bool isSystemCPSRField() const { return Kind == k_CPSRField; }
   bool isReg() const { return Kind == k_Register && !Reg.isVector; }
   bool isVectorReg() const { return Kind == k_Register && Reg.isVector; }
 
@@ -714,7 +728,7 @@ public:
   bool isPrefetch() const { return Kind == k_Prefetch; }
   bool isShifter() const { return Kind == k_Shifter; }
   bool isExtend() const {
-    // lsl is an alias for UXTX but will be a parsed as a k_Shifter operand.
+    // lsl is an alias for UXTW but will be a parsed as a k_Shifter operand.
     if (isShifter()) {
       ARM64_AM::ShiftType ST = ARM64_AM::getShiftType(Shifter.Val);
       return ST == ARM64_AM::LSL;
@@ -997,13 +1011,33 @@ public:
   bool isAdrpLabel() const {
     // Validation was handled during parsing, so we just sanity check that
     // something didn't go haywire.
-    return isImm();
+    if (!isImm())
+        return false;
+
+    if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Imm.Val)) {
+      int64_t Val = CE->getValue();
+      int64_t Min = - (4096 * (1LL << (21 - 1)));
+      int64_t Max = 4096 * ((1LL << (21 - 1)) - 1);
+      return (Val % 4096) == 0 && Val >= Min && Val <= Max;
+    }
+
+    return true;
   }
 
   bool isAdrLabel() const {
     // Validation was handled during parsing, so we just sanity check that
     // something didn't go haywire.
-    return isImm();
+    if (!isImm())
+        return false;
+
+    if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Imm.Val)) {
+      int64_t Val = CE->getValue();
+      int64_t Min = - (1LL << (21 - 1));
+      int64_t Max = ((1LL << (21 - 1)) - 1);
+      return Val >= Min && Val <= Max;
+    }
+
+    return true;
   }
 
   void addExpr(MCInst &Inst, const MCExpr *Expr) const {
@@ -1077,7 +1111,12 @@ public:
   }
 
   void addAdrpLabelOperands(MCInst &Inst, unsigned N) const {
-    addImmOperands(Inst, N);
+    assert(N == 1 && "Invalid number of operands!");
+    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
+    if (!MCE)
+      addExpr(Inst, getImm());
+    else
+      Inst.addOperand(MCOperand::CreateImm(MCE->getValue() >> 12));
   }
 
   void addAdrLabelOperands(MCInst &Inst, unsigned N) const {
@@ -1241,7 +1280,7 @@ public:
     Inst.addOperand(MCOperand::CreateImm(MCE->getValue() >> 2));
   }
 
-  void addBranchTarget19Operands(MCInst &Inst, unsigned N) const {
+  void addPCRelLabel19Operands(MCInst &Inst, unsigned N) const {
     // Branch operands don't encode the low bits, so shift them off
     // here. If it's a label, however, just put it on directly as there's
     // not enough information now to do anything.
@@ -1279,19 +1318,31 @@ public:
     Inst.addOperand(MCOperand::CreateImm(getBarrier()));
   }
 
-  void addSystemRegisterOperands(MCInst &Inst, unsigned N) const {
+  void addMRSSystemRegisterOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    if (Kind == k_SystemRegister)
-      Inst.addOperand(MCOperand::CreateImm(getSystemRegister()));
-    else {
-      assert(Kind == k_CPSRField && getCPSRField() == ARM64SYS::cpsr_SPSel);
-      Inst.addOperand(MCOperand::CreateImm(ARM64SYS::SPSel));
-    }
+
+    bool Valid;
+    uint32_t Bits = ARM64SysReg::MRSMapper().fromString(getSysReg(), Valid);
+
+    Inst.addOperand(MCOperand::CreateImm(Bits));
+  }
+
+  void addMSRSystemRegisterOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+
+    bool Valid;
+    uint32_t Bits = ARM64SysReg::MSRMapper().fromString(getSysReg(), Valid);
+
+    Inst.addOperand(MCOperand::CreateImm(Bits));
   }
 
   void addSystemCPSRFieldOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::CreateImm(getCPSRField()));
+
+    bool Valid;
+    uint32_t Bits = ARM64PState::PStateMapper().fromString(getSysReg(), Valid);
+
+    Inst.addOperand(MCOperand::CreateImm(Bits));
   }
 
   void addSysCROperands(MCInst &Inst, unsigned N) const {
@@ -1346,10 +1397,10 @@ public:
 
   void addExtendOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    // lsl is an alias for UXTX but will be a parsed as a k_Shifter operand.
+    // lsl is an alias for UXTW but will be a parsed as a k_Shifter operand.
     if (isShifter()) {
       assert(ARM64_AM::getShiftType(getShifter()) == ARM64_AM::LSL);
-      unsigned imm = getArithExtendImm(ARM64_AM::UXTX,
+      unsigned imm = getArithExtendImm(ARM64_AM::UXTW,
                                        ARM64_AM::getShiftValue(getShifter()));
       Inst.addOperand(MCOperand::CreateImm(imm));
     } else
@@ -1377,7 +1428,7 @@ public:
     assert(N == 3 && "Invalid number of operands!");
 
     Inst.addOperand(MCOperand::CreateReg(Mem.BaseRegNum));
-    Inst.addOperand(MCOperand::CreateReg(Mem.OffsetRegNum));
+    Inst.addOperand(MCOperand::CreateReg(getXRegFromWReg(Mem.OffsetRegNum)));
     unsigned ExtendImm = ARM64_AM::getMemExtendImm(Mem.ExtType, DoShift);
     Inst.addOperand(MCOperand::CreateImm(ExtendImm));
   }
@@ -1605,19 +1656,10 @@ public:
     return Op;
   }
 
-  static ARM64Operand *CreateSystemRegister(uint16_t Val, SMLoc S,
-                                            MCContext &Ctx) {
-    ARM64Operand *Op = new ARM64Operand(k_SystemRegister, Ctx);
-    Op->SystemRegister.Val = Val;
-    Op->StartLoc = S;
-    Op->EndLoc = S;
-    return Op;
-  }
-
-  static ARM64Operand *CreateCPSRField(ARM64SYS::CPSRField Field, SMLoc S,
-                                       MCContext &Ctx) {
-    ARM64Operand *Op = new ARM64Operand(k_CPSRField, Ctx);
-    Op->CPSRField.Field = Field;
+  static ARM64Operand *CreateSysReg(StringRef Str, SMLoc S, MCContext &Ctx) {
+    ARM64Operand *Op = new ARM64Operand(k_SysReg, Ctx);
+    Op->SysReg.Data = Str.data();
+    Op->SysReg.Length = Str.size();
     Op->StartLoc = S;
     Op->EndLoc = S;
     return Op;
@@ -1702,30 +1744,12 @@ void ARM64Operand::print(raw_ostream &OS) const {
        << ") >";
     break;
   case k_Barrier: {
-    const char *Name =
-        ARM64SYS::getBarrierOptName((ARM64SYS::BarrierOption)getBarrier());
-    OS << "<barrier ";
-    if (Name)
-      OS << Name;
+    bool Valid;
+    StringRef Name = ARM64DB::DBarrierMapper().toString(getBarrier(), Valid);
+    if (Valid)
+      OS << "<barrier " << Name << ">";
     else
-      OS << getBarrier();
-    OS << ">";
-    break;
-  }
-  case k_SystemRegister: {
-    const char *Name = ARM64SYS::getSystemRegisterName(
-        (ARM64SYS::SystemRegister)getSystemRegister());
-    OS << "<systemreg ";
-    if (Name)
-      OS << Name;
-    else
-      OS << "#" << getSystemRegister();
-    OS << ">";
-    break;
-  }
-  case k_CPSRField: {
-    const char *Name = ARM64SYS::getCPSRFieldName(getCPSRField());
-    OS << "<cpsrfield " << Name << ">";
+      OS << "<barrier invalid #" << getBarrier() << ">";
     break;
   }
   case k_Immediate:
@@ -1748,20 +1772,24 @@ void ARM64Operand::print(raw_ostream &OS) const {
   case k_VectorIndex:
     OS << "<vectorindex " << getVectorIndex() << ">";
     break;
+  case k_SysReg:
+    OS << "<sysreg: " << getSysReg() << '>';
+    break;
   case k_Token:
     OS << "'" << getToken() << "'";
     break;
   case k_SysCR:
     OS << "c" << getSysCR();
     break;
-  case k_Prefetch:
-    OS << "<prfop ";
-    if (ARM64_AM::isNamedPrefetchOp(getPrefetch()))
-      OS << ARM64_AM::getPrefetchOpName((ARM64_AM::PrefetchOp)getPrefetch());
+  case k_Prefetch: {
+    bool Valid;
+    StringRef Name = ARM64PRFM::PRFMMapper().toString(getPrefetch(), Valid);
+    if (Valid)
+      OS << "<prfop " << Name << ">";
     else
-      OS << "#" << getPrefetch();
-    OS << ">";
+      OS << "<prfop invalid #" << getPrefetch() << ">";
     break;
+  }
   case k_Shifter: {
     unsigned Val = getShifter();
     OS << "<" << ARM64_AM::getShiftName(ARM64_AM::getShiftType(Val)) << " #"
@@ -1880,8 +1908,8 @@ int ARM64AsmParser::tryParseRegister() {
   // Also handle a few aliases of registers.
   if (RegNum == 0)
     RegNum = StringSwitch<unsigned>(lowerCase)
-                 .Case("x29", ARM64::FP)
-                 .Case("x30", ARM64::LR)
+                 .Case("fp",  ARM64::FP)
+                 .Case("lr",  ARM64::LR)
                  .Case("x31", ARM64::XZR)
                  .Case("w31", ARM64::WZR)
                  .Default(0);
@@ -1895,7 +1923,7 @@ int ARM64AsmParser::tryParseRegister() {
 
 /// tryMatchVectorRegister - Try to parse a vector register name with optional
 /// kind specifier. If it is a register specifier, eat the token and return it.
-int ARM64AsmParser::tryMatchVectorRegister(StringRef &Kind) {
+int ARM64AsmParser::tryMatchVectorRegister(StringRef &Kind, bool expected) {
   if (Parser.getTok().isNot(AsmToken::Identifier)) {
     TokError("vector register expected");
     return -1;
@@ -1918,6 +1946,9 @@ int ARM64AsmParser::tryMatchVectorRegister(StringRef &Kind) {
     Parser.Lex(); // Eat the register token.
     return RegNum;
   }
+
+  if (expected)
+    TokError("vector register expected");
   return -1;
 }
 
@@ -2004,8 +2035,10 @@ ARM64AsmParser::tryParsePrefetch(OperandVector &Operands) {
   SMLoc S = getLoc();
   const AsmToken &Tok = Parser.getTok();
   // Either an identifier for named values or a 5-bit immediate.
-  if (Tok.is(AsmToken::Hash)) {
-    Parser.Lex(); // Eat hash token.
+  bool Hash = Tok.is(AsmToken::Hash);
+  if (Hash || Tok.is(AsmToken::Integer)) {
+    if (Hash)
+      Parser.Lex(); // Eat hash token.
     const MCExpr *ImmVal;
     if (getParser().parseExpression(ImmVal))
       return MatchOperand_ParseFail;
@@ -2030,21 +2063,9 @@ ARM64AsmParser::tryParsePrefetch(OperandVector &Operands) {
     return MatchOperand_ParseFail;
   }
 
-  unsigned prfop = StringSwitch<unsigned>(Tok.getString())
-                       .Case("pldl1keep", ARM64_AM::PLDL1KEEP)
-                       .Case("pldl1strm", ARM64_AM::PLDL1STRM)
-                       .Case("pldl2keep", ARM64_AM::PLDL2KEEP)
-                       .Case("pldl2strm", ARM64_AM::PLDL2STRM)
-                       .Case("pldl3keep", ARM64_AM::PLDL3KEEP)
-                       .Case("pldl3strm", ARM64_AM::PLDL3STRM)
-                       .Case("pstl1keep", ARM64_AM::PSTL1KEEP)
-                       .Case("pstl1strm", ARM64_AM::PSTL1STRM)
-                       .Case("pstl2keep", ARM64_AM::PSTL2KEEP)
-                       .Case("pstl2strm", ARM64_AM::PSTL2STRM)
-                       .Case("pstl3keep", ARM64_AM::PSTL3KEEP)
-                       .Case("pstl3strm", ARM64_AM::PSTL3STRM)
-                       .Default(0xff);
-  if (prfop == 0xff) {
+  bool Valid;
+  unsigned prfop = ARM64PRFM::PRFMMapper().fromString(Tok.getString(), Valid);
+  if (!Valid) {
     TokError("pre-fetch hint expected");
     return MatchOperand_ParseFail;
   }
@@ -2060,40 +2081,43 @@ ARM64AsmParser::OperandMatchResultTy
 ARM64AsmParser::tryParseAdrpLabel(OperandVector &Operands) {
   SMLoc S = getLoc();
   const MCExpr *Expr;
+
+  if (Parser.getTok().is(AsmToken::Hash)) {
+    Parser.Lex(); // Eat hash token.
+  }
+
   if (parseSymbolicImmVal(Expr))
     return MatchOperand_ParseFail;
 
   ARM64MCExpr::VariantKind ELFRefKind;
   MCSymbolRefExpr::VariantKind DarwinRefKind;
   const MCConstantExpr *Addend;
-  if (!classifySymbolRef(Expr, ELFRefKind, DarwinRefKind, Addend)) {
-    Error(S, "modified label reference + constant expected");
-    return MatchOperand_ParseFail;
-  }
-
-  if (DarwinRefKind == MCSymbolRefExpr::VK_None &&
-      ELFRefKind == ARM64MCExpr::VK_INVALID) {
-    // No modifier was specified at all; this is the syntax for an ELF basic
-    // ADRP relocation (unfortunately).
-    Expr = ARM64MCExpr::Create(Expr, ARM64MCExpr::VK_ABS_PAGE, getContext());
-  } else if ((DarwinRefKind == MCSymbolRefExpr::VK_GOTPAGE ||
-              DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGE) &&
-             Addend != 0) {
-    Error(S, "gotpage label reference not allowed an addend");
-    return MatchOperand_ParseFail;
-  } else if (DarwinRefKind != MCSymbolRefExpr::VK_PAGE &&
-             DarwinRefKind != MCSymbolRefExpr::VK_GOTPAGE &&
-             DarwinRefKind != MCSymbolRefExpr::VK_TLVPPAGE &&
-             ELFRefKind != ARM64MCExpr::VK_GOT_PAGE &&
-             ELFRefKind != ARM64MCExpr::VK_GOTTPREL_PAGE &&
-             ELFRefKind != ARM64MCExpr::VK_TLSDESC_PAGE) {
-    // The operand must be an @page or @gotpage qualified symbolref.
-    Error(S, "page or gotpage label reference expected");
-    return MatchOperand_ParseFail;
+  if (classifySymbolRef(Expr, ELFRefKind, DarwinRefKind, Addend)) {
+    if (DarwinRefKind == MCSymbolRefExpr::VK_None &&
+        ELFRefKind == ARM64MCExpr::VK_INVALID) {
+      // No modifier was specified at all; this is the syntax for an ELF basic
+      // ADRP relocation (unfortunately).
+      Expr = ARM64MCExpr::Create(Expr, ARM64MCExpr::VK_ABS_PAGE, getContext());
+    } else if ((DarwinRefKind == MCSymbolRefExpr::VK_GOTPAGE ||
+                DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGE) &&
+               Addend != 0) {
+      Error(S, "gotpage label reference not allowed an addend");
+      return MatchOperand_ParseFail;
+    } else if (DarwinRefKind != MCSymbolRefExpr::VK_PAGE &&
+               DarwinRefKind != MCSymbolRefExpr::VK_GOTPAGE &&
+               DarwinRefKind != MCSymbolRefExpr::VK_TLVPPAGE &&
+               ELFRefKind != ARM64MCExpr::VK_GOT_PAGE &&
+               ELFRefKind != ARM64MCExpr::VK_GOTTPREL_PAGE &&
+               ELFRefKind != ARM64MCExpr::VK_TLSDESC_PAGE) {
+      // The operand must be an @page or @gotpage qualified symbolref.
+      Error(S, "page or gotpage label reference expected");
+      return MatchOperand_ParseFail;
+    }
   }
 
-  // We have a label reference possibly with addend. The addend is a raw value
-  // here. The linker will adjust it to only reference the page.
+  // We have either a label reference possibly with addend or an immediate. The
+  // addend is a raw value here. The linker will adjust it to only reference the
+  // page.
   SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
   Operands.push_back(ARM64Operand::CreateImm(Expr, S, E, getContext()));
 
@@ -2106,20 +2130,14 @@ ARM64AsmParser::OperandMatchResultTy
 ARM64AsmParser::tryParseAdrLabel(OperandVector &Operands) {
   SMLoc S = getLoc();
   const MCExpr *Expr;
-  if (getParser().parseExpression(Expr))
-    return MatchOperand_ParseFail;
 
-  // The operand must be an un-qualified assembler local symbolref.
-  // FIXME: wrong for ELF.
-  if (const MCSymbolRefExpr *SRE = dyn_cast<const MCSymbolRefExpr>(Expr)) {
-    // FIXME: Should reference the MachineAsmInfo to get the private prefix.
-    bool isTemporary = SRE->getSymbol().getName().startswith("L");
-    if (!isTemporary || SRE->getKind() != MCSymbolRefExpr::VK_None) {
-      Error(S, "unqualified, assembler-local label name expected");
-      return MatchOperand_ParseFail;
-    }
+  if (Parser.getTok().is(AsmToken::Hash)) {
+    Parser.Lex(); // Eat hash token.
   }
 
+  if (getParser().parseExpression(Expr))
+    return MatchOperand_ParseFail;
+
   SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
   Operands.push_back(ARM64Operand::CreateImm(Expr, S, E, getContext()));
 
@@ -2131,9 +2149,11 @@ ARM64AsmParser::OperandMatchResultTy
 ARM64AsmParser::tryParseFPImm(OperandVector &Operands) {
   SMLoc S = getLoc();
 
-  if (Parser.getTok().isNot(AsmToken::Hash))
-    return MatchOperand_NoMatch;
-  Parser.Lex(); // Eat the '#'.
+  bool Hash = false;
+  if (Parser.getTok().is(AsmToken::Hash)) {
+    Parser.Lex(); // Eat '#'
+    Hash = true;
+  }
 
   // Handle negation, as that still comes through as a separate token.
   bool isNegative = false;
@@ -2179,13 +2199,16 @@ ARM64AsmParser::tryParseFPImm(OperandVector &Operands) {
     return MatchOperand_Success;
   }
 
+  if (!Hash)
+    return MatchOperand_NoMatch;
+
   TokError("invalid floating point immediate");
   return MatchOperand_ParseFail;
 }
 
 /// parseCondCodeString - Parse a Condition Code string.
 unsigned ARM64AsmParser::parseCondCodeString(StringRef Cond) {
-  unsigned CC = StringSwitch<unsigned>(Cond)
+  unsigned CC = StringSwitch<unsigned>(Cond.lower())
                     .Case("eq", ARM64CC::EQ)
                     .Case("ne", ARM64CC::NE)
                     .Case("cs", ARM64CC::CS)
@@ -2203,25 +2226,8 @@ unsigned ARM64AsmParser::parseCondCodeString(StringRef Cond) {
                     .Case("gt", ARM64CC::GT)
                     .Case("le", ARM64CC::LE)
                     .Case("al", ARM64CC::AL)
-                // Upper case works too. Not mixed case, though.
-                    .Case("EQ", ARM64CC::EQ)
-                    .Case("NE", ARM64CC::NE)
-                    .Case("CS", ARM64CC::CS)
-                    .Case("HS", ARM64CC::CS)
-                    .Case("CC", ARM64CC::CC)
-                    .Case("LO", ARM64CC::CC)
-                    .Case("MI", ARM64CC::MI)
-                    .Case("PL", ARM64CC::PL)
-                    .Case("VS", ARM64CC::VS)
-                    .Case("VC", ARM64CC::VC)
-                    .Case("HI", ARM64CC::HI)
-                    .Case("LS", ARM64CC::LS)
-                    .Case("GE", ARM64CC::GE)
-                    .Case("LT", ARM64CC::LT)
-                    .Case("GT", ARM64CC::GT)
-                    .Case("LE", ARM64CC::LE)
-                    .Case("AL", ARM64CC::AL)
-                    .Default(~0U);
+                    .Case("nv", ARM64CC::NV)
+                    .Default(ARM64CC::Invalid);
   return CC;
 }
 
@@ -2234,7 +2240,7 @@ bool ARM64AsmParser::parseCondCode(OperandVector &Operands,
 
   StringRef Cond = Tok.getString();
   unsigned CC = parseCondCodeString(Cond);
-  if (CC == ~0U)
+  if (CC == ARM64CC::Invalid)
     return TokError("invalid condition code");
   Parser.Lex(); // Eat identifier token.
 
@@ -2270,9 +2276,12 @@ bool ARM64AsmParser::parseOptionalShift(OperandVector &Operands) {
   Parser.Lex();
 
   // We expect a number here.
-  if (getLexer().isNot(AsmToken::Hash))
+  bool Hash = getLexer().is(AsmToken::Hash);
+  if (!Hash && getLexer().isNot(AsmToken::Integer))
     return TokError("immediate value expected for shifter operand");
-  Parser.Lex(); // Eat the '#'.
+
+  if (Hash)
+    Parser.Lex(); // Eat the '#'.
 
   SMLoc ExprLoc = getLoc();
   const MCExpr *ImmVal;
@@ -2331,14 +2340,16 @@ bool ARM64AsmParser::parseOptionalExtend(OperandVector &Operands) {
     return false;
   }
 
-  if (getLexer().isNot(AsmToken::Hash)) {
+  bool Hash = getLexer().is(AsmToken::Hash);
+  if (!Hash && getLexer().isNot(AsmToken::Integer)) {
     SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
     Operands.push_back(
         ARM64Operand::CreateExtend(ExtOp, 0, S, E, getContext()));
     return false;
   }
 
-  Parser.Lex(); // Eat the '#'.
+  if (Hash)
+    Parser.Lex(); // Eat the '#'.
 
   const MCExpr *ImmVal;
   if (getParser().parseExpression(ImmVal))
@@ -2551,6 +2562,12 @@ bool ARM64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc,
     } else if (!Op.compare_lower("ipas2le1")) {
       // SYS #4, C8, C4, #5
       SYS_ALIAS(4, 8, 4, 5);
+    } else if (!Op.compare_lower("ipas2e1is")) {
+      // SYS #4, C8, C4, #1
+      SYS_ALIAS(4, 8, 0, 1);
+    } else if (!Op.compare_lower("ipas2le1is")) {
+      // SYS #4, C8, C4, #5
+      SYS_ALIAS(4, 8, 0, 5);
     } else if (!Op.compare_lower("vmalls12e1")) {
       // SYS #4, C8, C7, #6
       SYS_ALIAS(4, 8, 7, 6);
@@ -2566,12 +2583,17 @@ bool ARM64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc,
 
   Parser.Lex(); // Eat operand.
 
+  bool ExpectRegister = (Op.lower().find("all") == StringRef::npos);
+  bool HasRegister = false;
+
   // Check for the optional register operand.
   if (getLexer().is(AsmToken::Comma)) {
     Parser.Lex(); // Eat comma.
 
     if (Tok.isNot(AsmToken::Identifier) || parseRegister(Operands))
       return TokError("expected register operand");
+
+    HasRegister = true;
   }
 
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
@@ -2579,6 +2601,13 @@ bool ARM64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc,
     return TokError("unexpected token in argument list");
   }
 
+  if (ExpectRegister && !HasRegister) {
+    return TokError("specified " + Mnemonic + " op requires a register");
+  }
+  else if (!ExpectRegister && HasRegister) {
+    return TokError("specified " + Mnemonic + " op does not use a register");
+  }
+
   Parser.Lex(); // Consume the EndOfStatement
   return false;
 }
@@ -2588,9 +2617,11 @@ ARM64AsmParser::tryParseBarrierOperand(OperandVector &Operands) {
   const AsmToken &Tok = Parser.getTok();
 
   // Can be either a #imm style literal or an option name
-  if (Tok.is(AsmToken::Hash)) {
+  bool Hash = Tok.is(AsmToken::Hash);
+  if (Hash || Tok.is(AsmToken::Integer)) {
     // Immediate operand.
-    Parser.Lex(); // Eat the '#'
+    if (Hash)
+      Parser.Lex(); // Eat the '#'
     const MCExpr *ImmVal;
     SMLoc ExprLoc = getLoc();
     if (getParser().parseExpression(ImmVal))
@@ -2614,27 +2645,15 @@ ARM64AsmParser::tryParseBarrierOperand(OperandVector &Operands) {
     return MatchOperand_ParseFail;
   }
 
-  unsigned Opt = StringSwitch<unsigned>(Tok.getString())
-                     .Case("oshld", ARM64SYS::OSHLD)
-                     .Case("oshst", ARM64SYS::OSHST)
-                     .Case("osh", ARM64SYS::OSH)
-                     .Case("nshld", ARM64SYS::NSHLD)
-                     .Case("nshst", ARM64SYS::NSHST)
-                     .Case("nsh", ARM64SYS::NSH)
-                     .Case("ishld", ARM64SYS::ISHLD)
-                     .Case("ishst", ARM64SYS::ISHST)
-                     .Case("ish", ARM64SYS::ISH)
-                     .Case("ld", ARM64SYS::LD)
-                     .Case("st", ARM64SYS::ST)
-                     .Case("sy", ARM64SYS::SY)
-                     .Default(ARM64SYS::InvalidBarrier);
-  if (Opt == ARM64SYS::InvalidBarrier) {
+  bool Valid;
+  unsigned Opt = ARM64DB::DBarrierMapper().fromString(Tok.getString(), Valid);
+  if (!Valid) {
     TokError("invalid barrier option name");
     return MatchOperand_ParseFail;
   }
 
   // The only valid named option for ISB is 'sy'
-  if (Mnemonic == "isb" && Opt != ARM64SYS::SY) {
+  if (Mnemonic == "isb" && Opt != ARM64DB::SY) {
     TokError("'sy' or #imm operand expected");
     return MatchOperand_ParseFail;
   }
@@ -2646,352 +2665,15 @@ ARM64AsmParser::tryParseBarrierOperand(OperandVector &Operands) {
 }
 
 ARM64AsmParser::OperandMatchResultTy
-ARM64AsmParser::tryParseSystemRegister(OperandVector &Operands) {
+ARM64AsmParser::tryParseSysReg(OperandVector &Operands) {
   const AsmToken &Tok = Parser.getTok();
 
-  // It can be specified as a symbolic name.
   if (Tok.isNot(AsmToken::Identifier))
     return MatchOperand_NoMatch;
 
-  auto ID = Tok.getString().lower();
-  ARM64SYS::SystemRegister Reg =
-      StringSwitch<ARM64SYS::SystemRegister>(ID)
-          .Case("spsr_el1", ARM64SYS::SPSR_svc)
-          .Case("spsr_svc", ARM64SYS::SPSR_svc)
-          .Case("elr_el1", ARM64SYS::ELR_EL1)
-          .Case("sp_el0", ARM64SYS::SP_EL0)
-          .Case("spsel", ARM64SYS::SPSel)
-          .Case("daif", ARM64SYS::DAIF)
-          .Case("currentel", ARM64SYS::CurrentEL)
-          .Case("nzcv", ARM64SYS::NZCV)
-          .Case("fpcr", ARM64SYS::FPCR)
-          .Case("fpsr", ARM64SYS::FPSR)
-          .Case("dspsr", ARM64SYS::DSPSR)
-          .Case("dlr", ARM64SYS::DLR)
-          .Case("spsr_el2", ARM64SYS::SPSR_hyp)
-          .Case("spsr_hyp", ARM64SYS::SPSR_hyp)
-          .Case("elr_el2", ARM64SYS::ELR_EL2)
-          .Case("sp_el1", ARM64SYS::SP_EL1)
-          .Case("spsr_irq", ARM64SYS::SPSR_irq)
-          .Case("spsr_abt", ARM64SYS::SPSR_abt)
-          .Case("spsr_und", ARM64SYS::SPSR_und)
-          .Case("spsr_fiq", ARM64SYS::SPSR_fiq)
-          .Case("spsr_el3", ARM64SYS::SPSR_EL3)
-          .Case("elr_el3", ARM64SYS::ELR_EL3)
-          .Case("sp_el2", ARM64SYS::SP_EL2)
-          .Case("midr_el1", ARM64SYS::MIDR_EL1)
-          .Case("ctr_el0", ARM64SYS::CTR_EL0)
-          .Case("mpidr_el1", ARM64SYS::MPIDR_EL1)
-          .Case("ecoidr_el1", ARM64SYS::ECOIDR_EL1)
-          .Case("dczid_el0", ARM64SYS::DCZID_EL0)
-          .Case("mvfr0_el1", ARM64SYS::MVFR0_EL1)
-          .Case("mvfr1_el1", ARM64SYS::MVFR1_EL1)
-          .Case("id_aa64pfr0_el1", ARM64SYS::ID_AA64PFR0_EL1)
-          .Case("id_aa64pfr1_el1", ARM64SYS::ID_AA64PFR1_EL1)
-          .Case("id_aa64dfr0_el1", ARM64SYS::ID_AA64DFR0_EL1)
-          .Case("id_aa64dfr1_el1", ARM64SYS::ID_AA64DFR1_EL1)
-          .Case("id_aa64isar0_el1", ARM64SYS::ID_AA64ISAR0_EL1)
-          .Case("id_aa64isar1_el1", ARM64SYS::ID_AA64ISAR1_EL1)
-          .Case("id_aa64mmfr0_el1", ARM64SYS::ID_AA64MMFR0_EL1)
-          .Case("id_aa64mmfr1_el1", ARM64SYS::ID_AA64MMFR1_EL1)
-          .Case("ccsidr_el1", ARM64SYS::CCSIDR_EL1)
-          .Case("clidr_el1", ARM64SYS::CLIDR_EL1)
-          .Case("aidr_el1", ARM64SYS::AIDR_EL1)
-          .Case("csselr_el1", ARM64SYS::CSSELR_EL1)
-          .Case("vpidr_el2", ARM64SYS::VPIDR_EL2)
-          .Case("vmpidr_el2", ARM64SYS::VMPIDR_EL2)
-          .Case("sctlr_el1", ARM64SYS::SCTLR_EL1)
-          .Case("sctlr_el2", ARM64SYS::SCTLR_EL2)
-          .Case("sctlr_el3", ARM64SYS::SCTLR_EL3)
-          .Case("actlr_el1", ARM64SYS::ACTLR_EL1)
-          .Case("actlr_el2", ARM64SYS::ACTLR_EL2)
-          .Case("actlr_el3", ARM64SYS::ACTLR_EL3)
-          .Case("cpacr_el1", ARM64SYS::CPACR_EL1)
-          .Case("cptr_el2", ARM64SYS::CPTR_EL2)
-          .Case("cptr_el3", ARM64SYS::CPTR_EL3)
-          .Case("scr_el3", ARM64SYS::SCR_EL3)
-          .Case("hcr_el2", ARM64SYS::HCR_EL2)
-          .Case("mdcr_el2", ARM64SYS::MDCR_EL2)
-          .Case("mdcr_el3", ARM64SYS::MDCR_EL3)
-          .Case("hstr_el2", ARM64SYS::HSTR_EL2)
-          .Case("hacr_el2", ARM64SYS::HACR_EL2)
-          .Case("ttbr0_el1", ARM64SYS::TTBR0_EL1)
-          .Case("ttbr1_el1", ARM64SYS::TTBR1_EL1)
-          .Case("ttbr0_el2", ARM64SYS::TTBR0_EL2)
-          .Case("ttbr0_el3", ARM64SYS::TTBR0_EL3)
-          .Case("vttbr_el2", ARM64SYS::VTTBR_EL2)
-          .Case("tcr_el1", ARM64SYS::TCR_EL1)
-          .Case("tcr_el2", ARM64SYS::TCR_EL2)
-          .Case("tcr_el3", ARM64SYS::TCR_EL3)
-          .Case("vtcr_el2", ARM64SYS::VTCR_EL2)
-          .Case("adfsr_el1", ARM64SYS::ADFSR_EL1)
-          .Case("aifsr_el1", ARM64SYS::AIFSR_EL1)
-          .Case("adfsr_el2", ARM64SYS::ADFSR_EL2)
-          .Case("aifsr_el2", ARM64SYS::AIFSR_EL2)
-          .Case("adfsr_el3", ARM64SYS::ADFSR_EL3)
-          .Case("aifsr_el3", ARM64SYS::AIFSR_EL3)
-          .Case("esr_el1", ARM64SYS::ESR_EL1)
-          .Case("esr_el2", ARM64SYS::ESR_EL2)
-          .Case("esr_el3", ARM64SYS::ESR_EL3)
-          .Case("far_el1", ARM64SYS::FAR_EL1)
-          .Case("far_el2", ARM64SYS::FAR_EL2)
-          .Case("far_el3", ARM64SYS::FAR_EL3)
-          .Case("hpfar_el2", ARM64SYS::HPFAR_EL2)
-          .Case("par_el1", ARM64SYS::PAR_EL1)
-          .Case("mair_el1", ARM64SYS::MAIR_EL1)
-          .Case("mair_el2", ARM64SYS::MAIR_EL2)
-          .Case("mair_el3", ARM64SYS::MAIR_EL3)
-          .Case("amair_el1", ARM64SYS::AMAIR_EL1)
-          .Case("amair_el2", ARM64SYS::AMAIR_EL2)
-          .Case("amair_el3", ARM64SYS::AMAIR_EL3)
-          .Case("vbar_el1", ARM64SYS::VBAR_EL1)
-          .Case("vbar_el2", ARM64SYS::VBAR_EL2)
-          .Case("vbar_el3", ARM64SYS::VBAR_EL3)
-          .Case("rvbar_el1", ARM64SYS::RVBAR_EL1)
-          .Case("rvbar_el2", ARM64SYS::RVBAR_EL2)
-          .Case("rvbar_el3", ARM64SYS::RVBAR_EL3)
-          .Case("isr_el1", ARM64SYS::ISR_EL1)
-          .Case("contextidr_el1", ARM64SYS::CONTEXTIDR_EL1)
-          .Case("tpidr_el0", ARM64SYS::TPIDR_EL0)
-          .Case("tpidrro_el0", ARM64SYS::TPIDRRO_EL0)
-          .Case("tpidr_el1", ARM64SYS::TPIDR_EL1)
-          .Case("tpidr_el2", ARM64SYS::TPIDR_EL2)
-          .Case("tpidr_el3", ARM64SYS::TPIDR_EL3)
-          .Case("teecr32_el1", ARM64SYS::TEECR32_EL1)
-          .Case("cntfrq_el0", ARM64SYS::CNTFRQ_EL0)
-          .Case("cntpct_el0", ARM64SYS::CNTPCT_EL0)
-          .Case("cntvct_el0", ARM64SYS::CNTVCT_EL0)
-          .Case("cntvoff_el2", ARM64SYS::CNTVOFF_EL2)
-          .Case("cntkctl_el1", ARM64SYS::CNTKCTL_EL1)
-          .Case("cnthctl_el2", ARM64SYS::CNTHCTL_EL2)
-          .Case("cntp_tval_el0", ARM64SYS::CNTP_TVAL_EL0)
-          .Case("cntp_ctl_el0", ARM64SYS::CNTP_CTL_EL0)
-          .Case("cntp_cval_el0", ARM64SYS::CNTP_CVAL_EL0)
-          .Case("cntv_tval_el0", ARM64SYS::CNTV_TVAL_EL0)
-          .Case("cntv_ctl_el0", ARM64SYS::CNTV_CTL_EL0)
-          .Case("cntv_cval_el0", ARM64SYS::CNTV_CVAL_EL0)
-          .Case("cnthp_tval_el2", ARM64SYS::CNTHP_TVAL_EL2)
-          .Case("cnthp_ctl_el2", ARM64SYS::CNTHP_CTL_EL2)
-          .Case("cnthp_cval_el2", ARM64SYS::CNTHP_CVAL_EL2)
-          .Case("cntps_tval_el1", ARM64SYS::CNTPS_TVAL_EL1)
-          .Case("cntps_ctl_el1", ARM64SYS::CNTPS_CTL_EL1)
-          .Case("cntps_cval_el1", ARM64SYS::CNTPS_CVAL_EL1)
-          .Case("dacr32_el2", ARM64SYS::DACR32_EL2)
-          .Case("ifsr32_el2", ARM64SYS::IFSR32_EL2)
-          .Case("teehbr32_el1", ARM64SYS::TEEHBR32_EL1)
-          .Case("sder32_el3", ARM64SYS::SDER32_EL3)
-          .Case("fpexc32_el2", ARM64SYS::FPEXC32_EL2)
-          .Case("current_el", ARM64SYS::CurrentEL)
-          .Case("pmevcntr0_el0", ARM64SYS::PMEVCNTR0_EL0)
-          .Case("pmevcntr1_el0", ARM64SYS::PMEVCNTR1_EL0)
-          .Case("pmevcntr2_el0", ARM64SYS::PMEVCNTR2_EL0)
-          .Case("pmevcntr3_el0", ARM64SYS::PMEVCNTR3_EL0)
-          .Case("pmevcntr4_el0", ARM64SYS::PMEVCNTR4_EL0)
-          .Case("pmevcntr5_el0", ARM64SYS::PMEVCNTR5_EL0)
-          .Case("pmevcntr6_el0", ARM64SYS::PMEVCNTR6_EL0)
-          .Case("pmevcntr7_el0", ARM64SYS::PMEVCNTR7_EL0)
-          .Case("pmevcntr8_el0", ARM64SYS::PMEVCNTR8_EL0)
-          .Case("pmevcntr9_el0", ARM64SYS::PMEVCNTR9_EL0)
-          .Case("pmevcntr10_el0", ARM64SYS::PMEVCNTR10_EL0)
-          .Case("pmevcntr11_el0", ARM64SYS::PMEVCNTR11_EL0)
-          .Case("pmevcntr12_el0", ARM64SYS::PMEVCNTR12_EL0)
-          .Case("pmevcntr13_el0", ARM64SYS::PMEVCNTR13_EL0)
-          .Case("pmevcntr14_el0", ARM64SYS::PMEVCNTR14_EL0)
-          .Case("pmevcntr15_el0", ARM64SYS::PMEVCNTR15_EL0)
-          .Case("pmevcntr16_el0", ARM64SYS::PMEVCNTR16_EL0)
-          .Case("pmevcntr17_el0", ARM64SYS::PMEVCNTR17_EL0)
-          .Case("pmevcntr18_el0", ARM64SYS::PMEVCNTR18_EL0)
-          .Case("pmevcntr19_el0", ARM64SYS::PMEVCNTR19_EL0)
-          .Case("pmevcntr20_el0", ARM64SYS::PMEVCNTR20_EL0)
-          .Case("pmevcntr21_el0", ARM64SYS::PMEVCNTR21_EL0)
-          .Case("pmevcntr22_el0", ARM64SYS::PMEVCNTR22_EL0)
-          .Case("pmevcntr23_el0", ARM64SYS::PMEVCNTR23_EL0)
-          .Case("pmevcntr24_el0", ARM64SYS::PMEVCNTR24_EL0)
-          .Case("pmevcntr25_el0", ARM64SYS::PMEVCNTR25_EL0)
-          .Case("pmevcntr26_el0", ARM64SYS::PMEVCNTR26_EL0)
-          .Case("pmevcntr27_el0", ARM64SYS::PMEVCNTR27_EL0)
-          .Case("pmevcntr28_el0", ARM64SYS::PMEVCNTR28_EL0)
-          .Case("pmevcntr29_el0", ARM64SYS::PMEVCNTR29_EL0)
-          .Case("pmevcntr30_el0", ARM64SYS::PMEVCNTR30_EL0)
-          .Case("pmevtyper0_el0", ARM64SYS::PMEVTYPER0_EL0)
-          .Case("pmevtyper1_el0", ARM64SYS::PMEVTYPER1_EL0)
-          .Case("pmevtyper2_el0", ARM64SYS::PMEVTYPER2_EL0)
-          .Case("pmevtyper3_el0", ARM64SYS::PMEVTYPER3_EL0)
-          .Case("pmevtyper4_el0", ARM64SYS::PMEVTYPER4_EL0)
-          .Case("pmevtyper5_el0", ARM64SYS::PMEVTYPER5_EL0)
-          .Case("pmevtyper6_el0", ARM64SYS::PMEVTYPER6_EL0)
-          .Case("pmevtyper7_el0", ARM64SYS::PMEVTYPER7_EL0)
-          .Case("pmevtyper8_el0", ARM64SYS::PMEVTYPER8_EL0)
-          .Case("pmevtyper9_el0", ARM64SYS::PMEVTYPER9_EL0)
-          .Case("pmevtyper10_el0", ARM64SYS::PMEVTYPER10_EL0)
-          .Case("pmevtyper11_el0", ARM64SYS::PMEVTYPER11_EL0)
-          .Case("pmevtyper12_el0", ARM64SYS::PMEVTYPER12_EL0)
-          .Case("pmevtyper13_el0", ARM64SYS::PMEVTYPER13_EL0)
-          .Case("pmevtyper14_el0", ARM64SYS::PMEVTYPER14_EL0)
-          .Case("pmevtyper15_el0", ARM64SYS::PMEVTYPER15_EL0)
-          .Case("pmevtyper16_el0", ARM64SYS::PMEVTYPER16_EL0)
-          .Case("pmevtyper17_el0", ARM64SYS::PMEVTYPER17_EL0)
-          .Case("pmevtyper18_el0", ARM64SYS::PMEVTYPER18_EL0)
-          .Case("pmevtyper19_el0", ARM64SYS::PMEVTYPER19_EL0)
-          .Case("pmevtyper20_el0", ARM64SYS::PMEVTYPER20_EL0)
-          .Case("pmevtyper21_el0", ARM64SYS::PMEVTYPER21_EL0)
-          .Case("pmevtyper22_el0", ARM64SYS::PMEVTYPER22_EL0)
-          .Case("pmevtyper23_el0", ARM64SYS::PMEVTYPER23_EL0)
-          .Case("pmevtyper24_el0", ARM64SYS::PMEVTYPER24_EL0)
-          .Case("pmevtyper25_el0", ARM64SYS::PMEVTYPER25_EL0)
-          .Case("pmevtyper26_el0", ARM64SYS::PMEVTYPER26_EL0)
-          .Case("pmevtyper27_el0", ARM64SYS::PMEVTYPER27_EL0)
-          .Case("pmevtyper28_el0", ARM64SYS::PMEVTYPER28_EL0)
-          .Case("pmevtyper29_el0", ARM64SYS::PMEVTYPER29_EL0)
-          .Case("pmevtyper30_el0", ARM64SYS::PMEVTYPER30_EL0)
-          .Case("pmccfiltr_el0", ARM64SYS::PMCCFILTR_EL0)
-          .Case("rmr_el3", ARM64SYS::RMR_EL3)
-          .Case("rmr_el2", ARM64SYS::RMR_EL2)
-          .Case("rmr_el1", ARM64SYS::RMR_EL1)
-          .Case("cpm_ioacc_ctl_el3", ARM64SYS::CPM_IOACC_CTL_EL3)
-          .Case("mdccsr_el0", ARM64SYS::MDCCSR_EL0)
-          .Case("mdccint_el1", ARM64SYS::MDCCINT_EL1)
-          .Case("dbgdtr_el0", ARM64SYS::DBGDTR_EL0)
-          .Case("dbgdtrrx_el0", ARM64SYS::DBGDTRRX_EL0)
-          .Case("dbgdtrtx_el0", ARM64SYS::DBGDTRTX_EL0)
-          .Case("dbgvcr32_el2", ARM64SYS::DBGVCR32_EL2)
-          .Case("osdtrrx_el1", ARM64SYS::OSDTRRX_EL1)
-          .Case("mdscr_el1", ARM64SYS::MDSCR_EL1)
-          .Case("osdtrtx_el1", ARM64SYS::OSDTRTX_EL1)
-          .Case("oseccr_el11", ARM64SYS::OSECCR_EL11)
-          .Case("dbgbvr0_el1", ARM64SYS::DBGBVR0_EL1)
-          .Case("dbgbvr1_el1", ARM64SYS::DBGBVR1_EL1)
-          .Case("dbgbvr2_el1", ARM64SYS::DBGBVR2_EL1)
-          .Case("dbgbvr3_el1", ARM64SYS::DBGBVR3_EL1)
-          .Case("dbgbvr4_el1", ARM64SYS::DBGBVR4_EL1)
-          .Case("dbgbvr5_el1", ARM64SYS::DBGBVR5_EL1)
-          .Case("dbgbvr6_el1", ARM64SYS::DBGBVR6_EL1)
-          .Case("dbgbvr7_el1", ARM64SYS::DBGBVR7_EL1)
-          .Case("dbgbvr8_el1", ARM64SYS::DBGBVR8_EL1)
-          .Case("dbgbvr9_el1", ARM64SYS::DBGBVR9_EL1)
-          .Case("dbgbvr10_el1", ARM64SYS::DBGBVR10_EL1)
-          .Case("dbgbvr11_el1", ARM64SYS::DBGBVR11_EL1)
-          .Case("dbgbvr12_el1", ARM64SYS::DBGBVR12_EL1)
-          .Case("dbgbvr13_el1", ARM64SYS::DBGBVR13_EL1)
-          .Case("dbgbvr14_el1", ARM64SYS::DBGBVR14_EL1)
-          .Case("dbgbvr15_el1", ARM64SYS::DBGBVR15_EL1)
-          .Case("dbgbcr0_el1", ARM64SYS::DBGBCR0_EL1)
-          .Case("dbgbcr1_el1", ARM64SYS::DBGBCR1_EL1)
-          .Case("dbgbcr2_el1", ARM64SYS::DBGBCR2_EL1)
-          .Case("dbgbcr3_el1", ARM64SYS::DBGBCR3_EL1)
-          .Case("dbgbcr4_el1", ARM64SYS::DBGBCR4_EL1)
-          .Case("dbgbcr5_el1", ARM64SYS::DBGBCR5_EL1)
-          .Case("dbgbcr6_el1", ARM64SYS::DBGBCR6_EL1)
-          .Case("dbgbcr7_el1", ARM64SYS::DBGBCR7_EL1)
-          .Case("dbgbcr8_el1", ARM64SYS::DBGBCR8_EL1)
-          .Case("dbgbcr9_el1", ARM64SYS::DBGBCR9_EL1)
-          .Case("dbgbcr10_el1", ARM64SYS::DBGBCR10_EL1)
-          .Case("dbgbcr11_el1", ARM64SYS::DBGBCR11_EL1)
-          .Case("dbgbcr12_el1", ARM64SYS::DBGBCR12_EL1)
-          .Case("dbgbcr13_el1", ARM64SYS::DBGBCR13_EL1)
-          .Case("dbgbcr14_el1", ARM64SYS::DBGBCR14_EL1)
-          .Case("dbgbcr15_el1", ARM64SYS::DBGBCR15_EL1)
-          .Case("dbgwvr0_el1", ARM64SYS::DBGWVR0_EL1)
-          .Case("dbgwvr1_el1", ARM64SYS::DBGWVR1_EL1)
-          .Case("dbgwvr2_el1", ARM64SYS::DBGWVR2_EL1)
-          .Case("dbgwvr3_el1", ARM64SYS::DBGWVR3_EL1)
-          .Case("dbgwvr4_el1", ARM64SYS::DBGWVR4_EL1)
-          .Case("dbgwvr5_el1", ARM64SYS::DBGWVR5_EL1)
-          .Case("dbgwvr6_el1", ARM64SYS::DBGWVR6_EL1)
-          .Case("dbgwvr7_el1", ARM64SYS::DBGWVR7_EL1)
-          .Case("dbgwvr8_el1", ARM64SYS::DBGWVR8_EL1)
-          .Case("dbgwvr9_el1", ARM64SYS::DBGWVR9_EL1)
-          .Case("dbgwvr10_el1", ARM64SYS::DBGWVR10_EL1)
-          .Case("dbgwvr11_el1", ARM64SYS::DBGWVR11_EL1)
-          .Case("dbgwvr12_el1", ARM64SYS::DBGWVR12_EL1)
-          .Case("dbgwvr13_el1", ARM64SYS::DBGWVR13_EL1)
-          .Case("dbgwvr14_el1", ARM64SYS::DBGWVR14_EL1)
-          .Case("dbgwvr15_el1", ARM64SYS::DBGWVR15_EL1)
-          .Case("dbgwcr0_el1", ARM64SYS::DBGWCR0_EL1)
-          .Case("dbgwcr1_el1", ARM64SYS::DBGWCR1_EL1)
-          .Case("dbgwcr2_el1", ARM64SYS::DBGWCR2_EL1)
-          .Case("dbgwcr3_el1", ARM64SYS::DBGWCR3_EL1)
-          .Case("dbgwcr4_el1", ARM64SYS::DBGWCR4_EL1)
-          .Case("dbgwcr5_el1", ARM64SYS::DBGWCR5_EL1)
-          .Case("dbgwcr6_el1", ARM64SYS::DBGWCR6_EL1)
-          .Case("dbgwcr7_el1", ARM64SYS::DBGWCR7_EL1)
-          .Case("dbgwcr8_el1", ARM64SYS::DBGWCR8_EL1)
-          .Case("dbgwcr9_el1", ARM64SYS::DBGWCR9_EL1)
-          .Case("dbgwcr10_el1", ARM64SYS::DBGWCR10_EL1)
-          .Case("dbgwcr11_el1", ARM64SYS::DBGWCR11_EL1)
-          .Case("dbgwcr12_el1", ARM64SYS::DBGWCR12_EL1)
-          .Case("dbgwcr13_el1", ARM64SYS::DBGWCR13_EL1)
-          .Case("dbgwcr14_el1", ARM64SYS::DBGWCR14_EL1)
-          .Case("dbgwcr15_el1", ARM64SYS::DBGWCR15_EL1)
-          .Case("mdrar_el1", ARM64SYS::MDRAR_EL1)
-          .Case("oslar_el1", ARM64SYS::OSLAR_EL1)
-          .Case("oslsr_el1", ARM64SYS::OSLSR_EL1)
-          .Case("osdlr_el1", ARM64SYS::OSDLR_EL1)
-          .Case("dbgprcr_el1", ARM64SYS::DBGPRCR_EL1)
-          .Case("dbgclaimset_el1", ARM64SYS::DBGCLAIMSET_EL1)
-          .Case("dbgclaimclr_el1", ARM64SYS::DBGCLAIMCLR_EL1)
-          .Case("dbgauthstatus_el1", ARM64SYS::DBGAUTHSTATUS_EL1)
-          .Case("dbgdevid2", ARM64SYS::DBGDEVID2)
-          .Case("dbgdevid1", ARM64SYS::DBGDEVID1)
-          .Case("dbgdevid0", ARM64SYS::DBGDEVID0)
-          .Case("id_pfr0_el1", ARM64SYS::ID_PFR0_EL1)
-          .Case("id_pfr1_el1", ARM64SYS::ID_PFR1_EL1)
-          .Case("id_dfr0_el1", ARM64SYS::ID_DFR0_EL1)
-          .Case("id_afr0_el1", ARM64SYS::ID_AFR0_EL1)
-          .Case("id_isar0_el1", ARM64SYS::ID_ISAR0_EL1)
-          .Case("id_isar1_el1", ARM64SYS::ID_ISAR1_EL1)
-          .Case("id_isar2_el1", ARM64SYS::ID_ISAR2_EL1)
-          .Case("id_isar3_el1", ARM64SYS::ID_ISAR3_EL1)
-          .Case("id_isar4_el1", ARM64SYS::ID_ISAR4_EL1)
-          .Case("id_isar5_el1", ARM64SYS::ID_ISAR5_EL1)
-          .Case("afsr1_el1", ARM64SYS::AFSR1_EL1)
-          .Case("afsr0_el1", ARM64SYS::AFSR0_EL1)
-          .Case("revidr_el1", ARM64SYS::REVIDR_EL1)
-          .Default(ARM64SYS::InvalidSystemReg);
-  if (Reg != ARM64SYS::InvalidSystemReg) {
-    // We matched a reg name, so create the operand.
-    Operands.push_back(
-        ARM64Operand::CreateSystemRegister(Reg, getLoc(), getContext()));
-    Parser.Lex(); // Consume the register name.
-    return MatchOperand_Success;
-  }
-
-  // Or we may have an identifier that encodes the sub-operands.
-  // For example, s3_2_c15_c0_0.
-  unsigned op0, op1, CRn, CRm, op2;
-  std::string Desc = ID;
-  if (std::sscanf(Desc.c_str(), "s%u_%u_c%u_c%u_%u", &op0, &op1, &CRn, &CRm,
-                  &op2) != 5)
-    return MatchOperand_NoMatch;
-  if ((op0 != 2 && op0 != 3) || op1 > 7 || CRn > 15 || CRm > 15 || op2 > 7)
-    return MatchOperand_NoMatch;
-
-  unsigned Val = op0 << 14 | op1 << 11 | CRn << 7 | CRm << 3 | op2;
-  Operands.push_back(
-      ARM64Operand::CreateSystemRegister(Val, getLoc(), getContext()));
-  Parser.Lex(); // Consume the register name.
-
-  return MatchOperand_Success;
-}
-
-ARM64AsmParser::OperandMatchResultTy
-ARM64AsmParser::tryParseCPSRField(OperandVector &Operands) {
-  const AsmToken &Tok = Parser.getTok();
-
-  if (Tok.isNot(AsmToken::Identifier))
-    return MatchOperand_NoMatch;
-
-  ARM64SYS::CPSRField Field =
-      StringSwitch<ARM64SYS::CPSRField>(Tok.getString().lower())
-          .Case("spsel", ARM64SYS::cpsr_SPSel)
-          .Case("daifset", ARM64SYS::cpsr_DAIFSet)
-          .Case("daifclr", ARM64SYS::cpsr_DAIFClr)
-          .Default(ARM64SYS::InvalidCPSRField);
-  if (Field == ARM64SYS::InvalidCPSRField)
-    return MatchOperand_NoMatch;
-  Operands.push_back(
-      ARM64Operand::CreateCPSRField(Field, getLoc(), getContext()));
-  Parser.Lex(); // Consume the register name.
+  Operands.push_back(ARM64Operand::CreateSysReg(Tok.getString(), getLoc(),
+                     getContext()));
+  Parser.Lex(); // Eat identifier
 
   return MatchOperand_Success;
 }
@@ -3004,7 +2686,7 @@ bool ARM64AsmParser::tryParseVectorRegister(OperandVector &Operands) {
   SMLoc S = getLoc();
   // Check for a vector register specifier first.
   StringRef Kind;
-  int64_t Reg = tryMatchVectorRegister(Kind);
+  int64_t Reg = tryMatchVectorRegister(Kind, false);
   if (Reg == -1)
     return true;
   Operands.push_back(
@@ -3021,17 +2703,17 @@ bool ARM64AsmParser::tryParseVectorRegister(OperandVector &Operands) {
 
     const MCExpr *ImmVal;
     if (getParser().parseExpression(ImmVal))
-      return MatchOperand_ParseFail;
+      return false;
     const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
     if (!MCE) {
       TokError("immediate value expected for vector index");
-      return MatchOperand_ParseFail;
+      return false;
     }
 
     SMLoc E = getLoc();
     if (Parser.getTok().isNot(AsmToken::RBrac)) {
       Error(E, "']' expected");
-      return MatchOperand_ParseFail;
+      return false;
     }
 
     Parser.Lex(); // Eat right bracket token.
@@ -3174,13 +2856,27 @@ bool ARM64AsmParser::parseMemory(OperandVector &Operands) {
 
         Parser.Lex(); // Eat the extend op.
 
+        // A 32-bit offset register is only valid for [SU]/XTW extend
+        // operators.
+        if (ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(Reg2)) {
+         if (ExtOp != ARM64_AM::UXTW &&
+            ExtOp != ARM64_AM::SXTW)
+          return Error(ExtLoc, "32-bit general purpose offset register "
+                               "requires sxtw or uxtw extend");
+        } else if (!ARM64MCRegisterClasses[ARM64::GPR64allRegClassID].contains(
+                       Reg2))
+          return Error(OffsetLoc,
+                       "64-bit general purpose offset register expected");
+
+        bool Hash = getLexer().is(AsmToken::Hash);
         if (getLexer().is(AsmToken::RBrac)) {
           // No immediate operand.
           if (ExtOp == ARM64_AM::UXTX)
             return Error(ExtLoc, "LSL extend requires immediate operand");
-        } else if (getLexer().is(AsmToken::Hash)) {
+        } else if (Hash || getLexer().is(AsmToken::Integer)) {
           // Immediate operand.
-          Parser.Lex(); // Eat the '#'
+          if (Hash)
+            Parser.Lex(); // Eat the '#'
           const MCExpr *ImmVal;
           SMLoc ExprLoc = getLoc();
           if (getParser().parseExpression(ImmVal))
@@ -3208,8 +2904,10 @@ bool ARM64AsmParser::parseMemory(OperandVector &Operands) {
       return false;
 
       // Immediate expressions.
-    } else if (Parser.getTok().is(AsmToken::Hash)) {
-      Parser.Lex(); // Eat hash token.
+    } else if (Parser.getTok().is(AsmToken::Hash) ||
+               Parser.getTok().is(AsmToken::Integer)) {
+      if (Parser.getTok().is(AsmToken::Hash))
+        Parser.Lex(); // Eat hash token.
 
       if (parseSymbolicImmVal(OffsetExpr))
         return true;
@@ -3295,10 +2993,13 @@ bool ARM64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
                   .Case("lo12", ARM64MCExpr::VK_LO12)
                   .Case("abs_g3", ARM64MCExpr::VK_ABS_G3)
                   .Case("abs_g2", ARM64MCExpr::VK_ABS_G2)
+                  .Case("abs_g2_s", ARM64MCExpr::VK_ABS_G2_S)
                   .Case("abs_g2_nc", ARM64MCExpr::VK_ABS_G2_NC)
                   .Case("abs_g1", ARM64MCExpr::VK_ABS_G1)
+                  .Case("abs_g1_s", ARM64MCExpr::VK_ABS_G1_S)
                   .Case("abs_g1_nc", ARM64MCExpr::VK_ABS_G1_NC)
                   .Case("abs_g0", ARM64MCExpr::VK_ABS_G0)
+                  .Case("abs_g0_s", ARM64MCExpr::VK_ABS_G0_S)
                   .Case("abs_g0_nc", ARM64MCExpr::VK_ABS_G0_NC)
                   .Case("dtprel_g2", ARM64MCExpr::VK_DTPREL_G2)
                   .Case("dtprel_g1", ARM64MCExpr::VK_DTPREL_G1)
@@ -3354,36 +3055,57 @@ bool ARM64AsmParser::parseVectorList(OperandVector &Operands) {
   SMLoc S = getLoc();
   Parser.Lex(); // Eat left bracket token.
   StringRef Kind;
-  int64_t FirstReg = tryMatchVectorRegister(Kind);
+  int64_t FirstReg = tryMatchVectorRegister(Kind, true);
   if (FirstReg == -1)
-    return Error(getLoc(), "vector register expected");
+    return true;
   int64_t PrevReg = FirstReg;
   unsigned Count = 1;
-  while (Parser.getTok().isNot(AsmToken::RCurly)) {
-    if (Parser.getTok().is(AsmToken::EndOfStatement))
-      Error(getLoc(), "'}' expected");
 
-    if (Parser.getTok().isNot(AsmToken::Comma))
-      return Error(getLoc(), "',' expected");
-    Parser.Lex(); // Eat the comma token.
+  if (Parser.getTok().is(AsmToken::Minus)) {
+    Parser.Lex(); // Eat the minus.
 
     SMLoc Loc = getLoc();
     StringRef NextKind;
-    int64_t Reg = tryMatchVectorRegister(NextKind);
+    int64_t Reg = tryMatchVectorRegister(NextKind, true);
     if (Reg == -1)
-      return Error(Loc, "vector register expected");
+      return true;
     // Any Kind suffices must match on all regs in the list.
     if (Kind != NextKind)
       return Error(Loc, "mismatched register size suffix");
 
-    // Registers must be incremental (with wraparound at 31)
-    if (getContext().getRegisterInfo()->getEncodingValue(Reg) !=
-        (getContext().getRegisterInfo()->getEncodingValue(PrevReg) + 1) % 32)
-      return Error(Loc, "registers must be sequential");
+    unsigned Space = (PrevReg < Reg) ? (Reg - PrevReg) : (Reg + 32 - PrevReg);
 
-    PrevReg = Reg;
-    ++Count;
+    if (Space == 0 || Space > 3) {
+      return Error(Loc, "invalid number of vectors");
+    }
+
+    Count += Space;
+  }
+  else {
+    while (Parser.getTok().is(AsmToken::Comma)) {
+      Parser.Lex(); // Eat the comma token.
+
+      SMLoc Loc = getLoc();
+      StringRef NextKind;
+      int64_t Reg = tryMatchVectorRegister(NextKind, true);
+      if (Reg == -1)
+        return true;
+      // Any Kind suffices must match on all regs in the list.
+      if (Kind != NextKind)
+        return Error(Loc, "mismatched register size suffix");
+
+      // Registers must be incremental (with wraparound at 31)
+      if (getContext().getRegisterInfo()->getEncodingValue(Reg) !=
+          (getContext().getRegisterInfo()->getEncodingValue(PrevReg) + 1) % 32)
+       return Error(Loc, "registers must be sequential");
+
+      PrevReg = Reg;
+      ++Count;
+    }
   }
+
+  if (Parser.getTok().is(AsmToken::EndOfStatement))
+    Error(getLoc(), "'}' expected");
   Parser.Lex(); // Eat the '}' token.
 
   unsigned NumElements = 0;
@@ -3401,17 +3123,17 @@ bool ARM64AsmParser::parseVectorList(OperandVector &Operands) {
 
     const MCExpr *ImmVal;
     if (getParser().parseExpression(ImmVal))
-      return MatchOperand_ParseFail;
+      return false;
     const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
     if (!MCE) {
       TokError("immediate value expected for vector index");
-      return MatchOperand_ParseFail;
+      return false;
     }
 
     SMLoc E = getLoc();
     if (Parser.getTok().isNot(AsmToken::RBrac)) {
       Error(E, "']' expected");
-      return MatchOperand_ParseFail;
+      return false;
     }
 
     Parser.Lex(); // Eat right bracket token.
@@ -3482,10 +3204,13 @@ bool ARM64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode,
     Operands.push_back(ARM64Operand::CreateImm(IdVal, S, E, getContext()));
     return false;
   }
+  case AsmToken::Integer:
+  case AsmToken::Real:
   case AsmToken::Hash: {
     // #42 -> immediate.
     S = getLoc();
-    Parser.Lex();
+    if (getLexer().is(AsmToken::Hash))
+      Parser.Lex();
 
     // The only Real that should come through here is a literal #0.0 for
     // the fcmp[e] r, #0.0 instructions. They expect raw token operands,
@@ -3542,7 +3267,7 @@ bool ARM64AsmParser::ParseInstruction(ParseInstructionInfo &Info,
     SMLoc SuffixLoc = SMLoc::getFromPointer(NameLoc.getPointer() +
                                             (Head.data() - Name.data()));
     unsigned CC = parseCondCodeString(Head);
-    if (CC == ~0U)
+    if (CC == ARM64CC::Invalid)
       return Error(SuffixLoc, "invalid condition code");
     const MCExpr *CCExpr = MCConstantExpr::Create(CC, getContext());
     Operands.push_back(
@@ -3612,57 +3337,6 @@ bool ARM64AsmParser::ParseInstruction(ParseInstructionInfo &Info,
   return false;
 }
 
-/// isFPR32Register - Check if a register is in the FPR32 register class.
-/// (The parser does not have the target register info to check the register
-/// class directly.)
-static bool isFPR32Register(unsigned Reg) {
-  using namespace ARM64;
-  switch (Reg) {
-  default:
-    break;
-  case S0:  case S1:  case S2:  case S3:  case S4:  case S5:  case S6:
-  case S7:  case S8:  case S9:  case S10:  case S11:  case S12:  case S13:
-  case S14:  case S15:  case S16:  case S17:  case S18:  case S19:  case S20:
-  case S21:  case S22:  case S23:  case S24:  case S25:  case S26:  case S27:
-  case S28:  case S29:  case S30:  case S31:
-    return true;
-  }
-  return false;
-}
-
-/// isGPR32Register - Check if a register is in the GPR32sp register class.
-/// (The parser does not have the target register info to check the register
-/// class directly.)
-static bool isGPR32Register(unsigned Reg) {
-  using namespace ARM64;
-  switch (Reg) {
-  default:
-    break;
-  case W0:  case W1:  case W2:  case W3:  case W4:  case W5:  case W6:
-  case W7:  case W8:  case W9:  case W10:  case W11:  case W12:  case W13:
-  case W14:  case W15:  case W16:  case W17:  case W18:  case W19:  case W20:
-  case W21:  case W22:  case W23:  case W24:  case W25:  case W26:  case W27:
-  case W28:  case W29:  case W30:  case WSP:
-    return true;
-  }
-  return false;
-}
-
-static bool isGPR64Reg(unsigned Reg) {
-  using namespace ARM64;
-  switch (Reg) {
-  case X0:  case X1:  case X2:  case X3:  case X4:  case X5:  case X6:
-  case X7:  case X8:  case X9:  case X10:  case X11:  case X12:  case X13:
-  case X14:  case X15:  case X16:  case X17:  case X18:  case X19:  case X20:
-  case X21:  case X22:  case X23:  case X24:  case X25:  case X26:  case X27:
-  case X28:  case FP:  case LR:  case SP:  case XZR:
-    return true;
-  default:
-    return false;
-  }
-}
-
-
 // FIXME: This entire function is a giant hack to provide us with decent
 // operand range validation/diagnostics until TableGen/MC can be extended
 // to support autogeneration of this kind of validation.
@@ -4055,9 +3729,9 @@ bool ARM64AsmParser::validateInstruction(MCInst &Inst,
   }
 }
 
-static void rewriteMOV(ARM64AsmParser::OperandVector &Operands,
-                       StringRef mnemonic, uint64_t imm, unsigned shift,
-                       MCContext &Context) {
+static void rewriteMOVI(ARM64AsmParser::OperandVector &Operands,
+                        StringRef mnemonic, uint64_t imm, unsigned shift,
+                        MCContext &Context) {
   ARM64Operand *Op = static_cast<ARM64Operand *>(Operands[0]);
   ARM64Operand *Op2 = static_cast<ARM64Operand *>(Operands[2]);
   Operands[0] =
@@ -4073,6 +3747,44 @@ static void rewriteMOV(ARM64AsmParser::OperandVector &Operands,
   delete Op;
 }
 
+static void rewriteMOVRSP(ARM64AsmParser::OperandVector &Operands,
+                        MCContext &Context) {
+  ARM64Operand *Op = static_cast<ARM64Operand *>(Operands[0]);
+  ARM64Operand *Op2 = static_cast<ARM64Operand *>(Operands[2]);
+  Operands[0] =
+    ARM64Operand::CreateToken("add", false, Op->getStartLoc(), Context);
+
+  const MCExpr *Imm = MCConstantExpr::Create(0, Context);
+  Operands.push_back(ARM64Operand::CreateImm(Imm, Op2->getStartLoc(),
+                                             Op2->getEndLoc(), Context));
+  Operands.push_back(ARM64Operand::CreateShifter(
+      ARM64_AM::LSL, 0, Op2->getStartLoc(), Op2->getEndLoc(), Context));
+
+  delete Op;
+}
+
+static void rewriteMOVR(ARM64AsmParser::OperandVector &Operands,
+                        MCContext &Context) {
+  ARM64Operand *Op = static_cast<ARM64Operand *>(Operands[0]);
+  ARM64Operand *Op2 = static_cast<ARM64Operand *>(Operands[2]);
+  Operands[0] =
+    ARM64Operand::CreateToken("orr", false, Op->getStartLoc(), Context);
+
+  // Operands[2] becomes Operands[3].
+  Operands.push_back(Operands[2]);
+  // And Operands[2] becomes ZR.
+  unsigned ZeroReg = ARM64::XZR;
+  if (ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(
+          Operands[2]->getReg()))
+    ZeroReg = ARM64::WZR;
+
+  Operands[2] =
+    ARM64Operand::CreateReg(ZeroReg, false, Op2->getStartLoc(),
+                            Op2->getEndLoc(), Context);
+
+  delete Op;
+}
+
 bool ARM64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode) {
   switch (ErrCode) {
   case Match_MissingFeature:
@@ -4108,6 +3820,8 @@ bool ARM64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode) {
     return Error(Loc, "immediate must be an integer in range [1,32].");
   case Match_InvalidImm1_64:
     return Error(Loc, "immediate must be an integer in range [1,64].");
+  case Match_InvalidLabel:
+    return Error(Loc, "expected label or encodable integer pc offset");
   case Match_MnemonicFail:
     return Error(Loc, "unrecognized instruction mnemonic");
   default:
@@ -4116,6 +3830,8 @@ bool ARM64AsmParser::showMatchError(SMLoc Loc, unsigned ErrCode) {
   }
 }
 
+static const char *getSubtargetFeatureName(unsigned Val);
+
 bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                              OperandVector &Operands,
                                              MCStreamer &Out,
@@ -4142,7 +3858,8 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     ARM64Operand *RegOp = static_cast<ARM64Operand *>(Operands[1]);
     unsigned ZeroReg;
     if (RegOp->isReg() &&
-        (isGPR32Register(RegOp->getReg()) || RegOp->getReg() == ARM64::WZR))
+        ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(
+            RegOp->getReg()))
       ZeroReg = ARM64::WZR;
     else
       ZeroReg = ARM64::XZR;
@@ -4162,6 +3879,7 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     // FIXME: Catching this here is a total hack, and we should use tblgen
     // support to implement this instead as soon as it is available.
 
+    ARM64Operand *Op1 = static_cast<ARM64Operand *>(Operands[1]);
     ARM64Operand *Op2 = static_cast<ARM64Operand *>(Operands[2]);
     if (Op2->isImm()) {
       if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op2->getImm())) {
@@ -4172,42 +3890,59 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
         // set, clear the complemented upper 32-bits so the logic below works
         // for 32-bit registers too.
         ARM64Operand *Op1 = static_cast<ARM64Operand *>(Operands[1]);
-        if (Op1->isReg() && isGPR32Register(Op1->getReg()) &&
+        if (Op1->isReg() &&
+            ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(
+                Op1->getReg()) &&
             (Val & 0xFFFFFFFFULL) == Val)
           NVal &= 0x00000000FFFFFFFFULL;
 
         // MOVK Rd, imm << 0
         if ((Val & 0xFFFF) == Val)
-          rewriteMOV(Operands, "movz", Val, 0, getContext());
+          rewriteMOVI(Operands, "movz", Val, 0, getContext());
 
         // MOVK Rd, imm << 16
         else if ((Val & 0xFFFF0000ULL) == Val)
-          rewriteMOV(Operands, "movz", Val, 16, getContext());
+          rewriteMOVI(Operands, "movz", Val, 16, getContext());
 
         // MOVK Rd, imm << 32
         else if ((Val & 0xFFFF00000000ULL) == Val)
-          rewriteMOV(Operands, "movz", Val, 32, getContext());
+          rewriteMOVI(Operands, "movz", Val, 32, getContext());
 
         // MOVK Rd, imm << 48
         else if ((Val & 0xFFFF000000000000ULL) == Val)
-          rewriteMOV(Operands, "movz", Val, 48, getContext());
+          rewriteMOVI(Operands, "movz", Val, 48, getContext());
 
         // MOVN Rd, (~imm << 0)
         else if ((NVal & 0xFFFFULL) == NVal)
-          rewriteMOV(Operands, "movn", NVal, 0, getContext());
+          rewriteMOVI(Operands, "movn", NVal, 0, getContext());
 
         // MOVN Rd, ~(imm << 16)
         else if ((NVal & 0xFFFF0000ULL) == NVal)
-          rewriteMOV(Operands, "movn", NVal, 16, getContext());
+          rewriteMOVI(Operands, "movn", NVal, 16, getContext());
 
         // MOVN Rd, ~(imm << 32)
         else if ((NVal & 0xFFFF00000000ULL) == NVal)
-          rewriteMOV(Operands, "movn", NVal, 32, getContext());
+          rewriteMOVI(Operands, "movn", NVal, 32, getContext());
 
         // MOVN Rd, ~(imm << 48)
         else if ((NVal & 0xFFFF000000000000ULL) == NVal)
-          rewriteMOV(Operands, "movn", NVal, 48, getContext());
+          rewriteMOVI(Operands, "movn", NVal, 48, getContext());
       }
+    } else if (Op1->isReg() && Op2->isReg()) {
+      // reg->reg move.
+      unsigned Reg1 = Op1->getReg();
+      unsigned Reg2 = Op2->getReg();
+      if ((Reg1 == ARM64::SP &&
+           ARM64MCRegisterClasses[ARM64::GPR64allRegClassID].contains(Reg2)) ||
+          (Reg2 == ARM64::SP &&
+           ARM64MCRegisterClasses[ARM64::GPR64allRegClassID].contains(Reg1)) ||
+          (Reg1 == ARM64::WSP &&
+           ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(Reg2)) ||
+          (Reg2 == ARM64::WSP &&
+           ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(Reg1)))
+        rewriteMOVRSP(Operands, getContext());
+      else
+        rewriteMOVR(Operands, getContext());
     }
   } else if (NumOperands == 4) {
     if (Tok == "add" || Tok == "adds" || Tok == "sub" || Tok == "subs") {
@@ -4251,7 +3986,8 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
           uint64_t Op3Val = Op3CE->getValue();
           uint64_t NewOp3Val = 0;
           uint64_t NewOp4Val = 0;
-          if (isGPR32Register(Op2->getReg()) || Op2->getReg() == ARM64::WZR) {
+          if (ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(
+                  Op2->getReg())) {
             NewOp3Val = (32 - Op3Val) & 0x1f;
             NewOp4Val = 31 - Op3Val;
           } else {
@@ -4318,7 +4054,8 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
           uint64_t Op4Val = Op4CE->getValue();
 
           uint64_t NewOp3Val = 0;
-          if (isGPR32Register(Op1->getReg()))
+          if (ARM64MCRegisterClasses[ARM64::GPR32allRegClassID].contains(
+                  Op1->getReg()))
             NewOp3Val = (32 - Op3Val) & 0x1f;
           else
             NewOp3Val = (64 - Op3Val) & 0x3f;
@@ -4431,7 +4168,9 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   else if (NumOperands == 3 &&
            (Tok == "sxtb" || Tok == "uxtb" || Tok == "sxth" || Tok == "uxth")) {
     ARM64Operand *Op = static_cast<ARM64Operand *>(Operands[1]);
-    if (Op->isReg() && isGPR64Reg(Op->getReg())) {
+    if (Op->isReg() &&
+        ARM64MCRegisterClasses[ARM64::GPR64allRegClassID].contains(
+            Op->getReg())) {
       // The source register can be Wn here, but the matcher expects a
       // GPR64. Twiddle it here if necessary.
       ARM64Operand *Op = static_cast<ARM64Operand *>(Operands[2]);
@@ -4450,8 +4189,10 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     ARM64Operand *ImmOp = static_cast<ARM64Operand *>(Operands[2]);
     if (RegOp->isReg() && ImmOp->isFPImm() &&
         ImmOp->getFPImm() == (unsigned)-1) {
-      unsigned zreg =
-          isFPR32Register(RegOp->getReg()) ? ARM64::WZR : ARM64::XZR;
+      unsigned zreg = ARM64MCRegisterClasses[ARM64::FPR32RegClassID].contains(
+                          RegOp->getReg())
+                          ? ARM64::WZR
+                          : ARM64::XZR;
       Operands[2] = ARM64Operand::CreateReg(zreg, false, Op->getStartLoc(),
                                             Op->getEndLoc(), getContext());
       delete ImmOp;
@@ -4470,6 +4211,7 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     if (Op->isVectorIndexD() && Op->getVectorIndex() == 1) {
       SMLoc Loc = Op->getStartLoc();
       Operands.pop_back();
+      delete Op;
       Operands.push_back(
           ARM64Operand::CreateToken("[", false, Loc, getContext()));
       Operands.push_back(
@@ -4491,6 +4233,7 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
         Operands.insert(
             Operands.begin() + OpNo + 2,
             ARM64Operand::CreateToken("]", false, Loc, getContext()));
+        delete Op;
       }
     }
   }
@@ -4521,7 +4264,21 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     Out.EmitInstruction(Inst, STI);
     return false;
   }
-  case Match_MissingFeature:
+  case Match_MissingFeature: {
+    assert(ErrorInfo && "Unknown missing feature!");
+    // Special case the error message for the very common case where only
+    // a single subtarget feature is missing (neon, e.g.).
+    std::string Msg = "instruction requires:";
+    unsigned Mask = 1;
+    for (unsigned i = 0; i < (sizeof(ErrorInfo)*8-1); ++i) {
+      if (ErrorInfo & Mask) {
+        Msg += " ";
+        Msg += getSubtargetFeatureName(ErrorInfo & Mask);
+      }
+      Mask <<= 1;
+    }
+    return Error(IDLoc, Msg);
+  }
   case Match_MnemonicFail:
     return showMatchError(IDLoc, MatchResult);
   case Match_InvalidOperand: {
@@ -4557,8 +4314,9 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
       MatchResult = Match_InvalidMemoryIndexed64;
       if (ErrorInfo) {
         ARM64Operand *PrevOp = (ARM64Operand *)Operands[ErrorInfo - 1];
-        if (PrevOp->isReg() && ARM64MCRegisterClasses[ARM64::GPR32RegClassID]
-                                   .contains(PrevOp->getReg()))
+        if (PrevOp->isReg() &&
+            ARM64MCRegisterClasses[ARM64::GPR32RegClassID].contains(
+                PrevOp->getReg()))
           MatchResult = Match_InvalidMemoryIndexed32;
       }
     }
@@ -4584,7 +4342,8 @@ bool ARM64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidImm1_8:
   case Match_InvalidImm1_16:
   case Match_InvalidImm1_32:
-  case Match_InvalidImm1_64: {
+  case Match_InvalidImm1_64:
+  case Match_InvalidLabel: {
     // Any time we get here, there's nothing fancy to do. Just get the
     // operand SMLoc and display the diagnostic.
     SMLoc ErrorLoc = ((ARM64Operand *)Operands[ErrorInfo])->getStartLoc();
@@ -4762,10 +4521,12 @@ ARM64AsmParser::classifySymbolRef(const MCExpr *Expr,
 
 /// Force static initialization.
 extern "C" void LLVMInitializeARM64AsmParser() {
-  RegisterMCAsmParser<ARM64AsmParser> X(TheARM64Target);
+  RegisterMCAsmParser<ARM64AsmParser> X(TheARM64leTarget);
+  RegisterMCAsmParser<ARM64AsmParser> Y(TheARM64beTarget);
 }
 
 #define GET_REGISTER_MATCHER
+#define GET_SUBTARGET_FEATURE_NAME
 #define GET_MATCHER_IMPLEMENTATION
 #include "ARM64GenAsmMatcher.inc"