1 //===- ARM64InstrAtomics.td - ARM64 Atomic codegen support -*- tablegen -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // ARM64 Atomic operand code-gen constructs.
12 //===----------------------------------------------------------------------===//
14 //===----------------------------------
16 //===----------------------------------
17 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
18 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
20 //===----------------------------------
22 //===----------------------------------
24 // When they're actually atomic, only one addressing mode (GPR64sp) is
25 // supported, but when they're relaxed and anything can be used, all the
26 // standard modes would be valid and may give efficiency gains.
28 // A atomic load operation that actually needs acquire semantics.
29 class acquiring_load<PatFrag base>
30 : PatFrag<(ops node:$ptr), (base node:$ptr), [{
31 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
32 assert(Ordering != AcquireRelease && "unexpected load ordering");
33 return Ordering == Acquire || Ordering == SequentiallyConsistent;
36 // An atomic load operation that does not need either acquire or release
38 class relaxed_load<PatFrag base>
39 : PatFrag<(ops node:$ptr), (base node:$ptr), [{
40 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
41 return Ordering == Monotonic || Ordering == Unordered;
45 def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
46 def : Pat<(relaxed_load<atomic_load_8> ro_indexed8:$addr),
47 (LDRBBro ro_indexed8:$addr)>;
48 def : Pat<(relaxed_load<atomic_load_8> am_indexed8:$addr),
49 (LDRBBui am_indexed8:$addr)>;
50 def : Pat<(relaxed_load<atomic_load_8> am_unscaled8:$addr),
51 (LDURBBi am_unscaled8:$addr)>;
54 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
55 def : Pat<(relaxed_load<atomic_load_16> ro_indexed16:$addr),
56 (LDRHHro ro_indexed16:$addr)>;
57 def : Pat<(relaxed_load<atomic_load_16> am_indexed16:$addr),
58 (LDRHHui am_indexed16:$addr)>;
59 def : Pat<(relaxed_load<atomic_load_16> am_unscaled16:$addr),
60 (LDURHHi am_unscaled16:$addr)>;
63 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
64 def : Pat<(relaxed_load<atomic_load_32> ro_indexed32:$addr),
65 (LDRWro ro_indexed32:$addr)>;
66 def : Pat<(relaxed_load<atomic_load_32> am_indexed32:$addr),
67 (LDRWui am_indexed32:$addr)>;
68 def : Pat<(relaxed_load<atomic_load_32> am_unscaled32:$addr),
69 (LDURWi am_unscaled32:$addr)>;
72 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
73 def : Pat<(relaxed_load<atomic_load_64> ro_indexed64:$addr),
74 (LDRXro ro_indexed64:$addr)>;
75 def : Pat<(relaxed_load<atomic_load_64> am_indexed64:$addr),
76 (LDRXui am_indexed64:$addr)>;
77 def : Pat<(relaxed_load<atomic_load_64> am_unscaled64:$addr),
78 (LDURXi am_unscaled64:$addr)>;
80 //===----------------------------------
82 //===----------------------------------
84 // When they're actually atomic, only one addressing mode (GPR64sp) is
85 // supported, but when they're relaxed and anything can be used, all the
86 // standard modes would be valid and may give efficiency gains.
88 // A store operation that actually needs release semantics.
89 class releasing_store<PatFrag base>
90 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
91 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
92 assert(Ordering != AcquireRelease && "unexpected store ordering");
93 return Ordering == Release || Ordering == SequentiallyConsistent;
96 // An atomic store operation that doesn't actually need to be atomic on ARM64.
97 class relaxed_store<PatFrag base>
98 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
99 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
100 return Ordering == Monotonic || Ordering == Unordered;
104 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
105 (STLRB GPR32:$val, GPR64sp:$ptr)>;
106 def : Pat<(relaxed_store<atomic_store_8> ro_indexed8:$ptr, GPR32:$val),
107 (STRBBro GPR32:$val, ro_indexed8:$ptr)>;
108 def : Pat<(relaxed_store<atomic_store_8> am_indexed8:$ptr, GPR32:$val),
109 (STRBBui GPR32:$val, am_indexed8:$ptr)>;
110 def : Pat<(relaxed_store<atomic_store_8> am_unscaled8:$ptr, GPR32:$val),
111 (STURBBi GPR32:$val, am_unscaled8:$ptr)>;
114 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
115 (STLRH GPR32:$val, GPR64sp:$ptr)>;
116 def : Pat<(relaxed_store<atomic_store_16> ro_indexed16:$ptr, GPR32:$val),
117 (STRHHro GPR32:$val, ro_indexed16:$ptr)>;
118 def : Pat<(relaxed_store<atomic_store_16> am_indexed16:$ptr, GPR32:$val),
119 (STRHHui GPR32:$val, am_indexed16:$ptr)>;
120 def : Pat<(relaxed_store<atomic_store_16> am_unscaled16:$ptr, GPR32:$val),
121 (STURHHi GPR32:$val, am_unscaled16:$ptr)>;
124 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
125 (STLRW GPR32:$val, GPR64sp:$ptr)>;
126 def : Pat<(relaxed_store<atomic_store_32> ro_indexed32:$ptr, GPR32:$val),
127 (STRWro GPR32:$val, ro_indexed32:$ptr)>;
128 def : Pat<(relaxed_store<atomic_store_32> am_indexed32:$ptr, GPR32:$val),
129 (STRWui GPR32:$val, am_indexed32:$ptr)>;
130 def : Pat<(relaxed_store<atomic_store_32> am_unscaled32:$ptr, GPR32:$val),
131 (STURWi GPR32:$val, am_unscaled32:$ptr)>;
134 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
135 (STLRX GPR64:$val, GPR64sp:$ptr)>;
136 def : Pat<(relaxed_store<atomic_store_64> ro_indexed64:$ptr, GPR64:$val),
137 (STRXro GPR64:$val, ro_indexed64:$ptr)>;
138 def : Pat<(relaxed_store<atomic_store_64> am_indexed64:$ptr, GPR64:$val),
139 (STRXui GPR64:$val, am_indexed64:$ptr)>;
140 def : Pat<(relaxed_store<atomic_store_64> am_unscaled64:$ptr, GPR64:$val),
141 (STURXi GPR64:$val, am_unscaled64:$ptr)>;
143 //===----------------------------------
144 // Atomic read-modify-write operations
145 //===----------------------------------
147 // More complicated operations need lots of C++ support, so we just create
148 // skeletons here for the C++ code to refer to.
150 let usesCustomInserter = 1, hasCtrlDep = 1, mayLoad = 1, mayStore = 1 in {
151 multiclass AtomicSizes {
152 def _I8 : Pseudo<(outs GPR32:$dst),
153 (ins GPR64sp:$ptr, GPR32:$incr, i32imm:$ordering), []>;
154 def _I16 : Pseudo<(outs GPR32:$dst),
155 (ins GPR64sp:$ptr, GPR32:$incr, i32imm:$ordering), []>;
156 def _I32 : Pseudo<(outs GPR32:$dst),
157 (ins GPR64sp:$ptr, GPR32:$incr, i32imm:$ordering), []>;
158 def _I64 : Pseudo<(outs GPR64:$dst),
159 (ins GPR64sp:$ptr, GPR64:$incr, i32imm:$ordering), []>;
160 def _I128 : Pseudo<(outs GPR64:$dstlo, GPR64:$dsthi),
161 (ins GPR64sp:$ptr, GPR64:$incrlo, GPR64:$incrhi,
162 i32imm:$ordering), []>;
166 defm ATOMIC_LOAD_ADD : AtomicSizes;
167 defm ATOMIC_LOAD_SUB : AtomicSizes;
168 defm ATOMIC_LOAD_AND : AtomicSizes;
169 defm ATOMIC_LOAD_OR : AtomicSizes;
170 defm ATOMIC_LOAD_XOR : AtomicSizes;
171 defm ATOMIC_LOAD_NAND : AtomicSizes;
172 defm ATOMIC_SWAP : AtomicSizes;
173 let Defs = [CPSR] in {
174 // These operations need a CMP to calculate the correct value
175 defm ATOMIC_LOAD_MIN : AtomicSizes;
176 defm ATOMIC_LOAD_MAX : AtomicSizes;
177 defm ATOMIC_LOAD_UMIN : AtomicSizes;
178 defm ATOMIC_LOAD_UMAX : AtomicSizes;
181 class AtomicCmpSwap<RegisterClass GPRData>
182 : Pseudo<(outs GPRData:$dst),
183 (ins GPR64sp:$ptr, GPRData:$old, GPRData:$new,
184 i32imm:$ordering), []> {
185 let usesCustomInserter = 1;
192 def ATOMIC_CMP_SWAP_I8 : AtomicCmpSwap<GPR32>;
193 def ATOMIC_CMP_SWAP_I16 : AtomicCmpSwap<GPR32>;
194 def ATOMIC_CMP_SWAP_I32 : AtomicCmpSwap<GPR32>;
195 def ATOMIC_CMP_SWAP_I64 : AtomicCmpSwap<GPR64>;
197 def ATOMIC_CMP_SWAP_I128
198 : Pseudo<(outs GPR64:$dstlo, GPR64:$dsthi),
199 (ins GPR64sp:$ptr, GPR64:$oldlo, GPR64:$oldhi,
200 GPR64:$newlo, GPR64:$newhi, i32imm:$ordering), []> {
201 let usesCustomInserter = 1;
208 //===----------------------------------
209 // Low-level exclusive operations
210 //===----------------------------------
214 def ldxr_1 : PatFrag<(ops node:$ptr), (int_arm64_ldxr node:$ptr), [{
215 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
218 def ldxr_2 : PatFrag<(ops node:$ptr), (int_arm64_ldxr node:$ptr), [{
219 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
222 def ldxr_4 : PatFrag<(ops node:$ptr), (int_arm64_ldxr node:$ptr), [{
223 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
226 def ldxr_8 : PatFrag<(ops node:$ptr), (int_arm64_ldxr node:$ptr), [{
227 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
230 def : Pat<(ldxr_1 am_noindex:$addr),
231 (SUBREG_TO_REG (i64 0), (LDXRB am_noindex:$addr), sub_32)>;
232 def : Pat<(ldxr_2 am_noindex:$addr),
233 (SUBREG_TO_REG (i64 0), (LDXRH am_noindex:$addr), sub_32)>;
234 def : Pat<(ldxr_4 am_noindex:$addr),
235 (SUBREG_TO_REG (i64 0), (LDXRW am_noindex:$addr), sub_32)>;
236 def : Pat<(ldxr_8 am_noindex:$addr), (LDXRX am_noindex:$addr)>;
238 def : Pat<(and (ldxr_1 am_noindex:$addr), 0xff),
239 (SUBREG_TO_REG (i64 0), (LDXRB am_noindex:$addr), sub_32)>;
240 def : Pat<(and (ldxr_2 am_noindex:$addr), 0xffff),
241 (SUBREG_TO_REG (i64 0), (LDXRH am_noindex:$addr), sub_32)>;
242 def : Pat<(and (ldxr_4 am_noindex:$addr), 0xffffffff),
243 (SUBREG_TO_REG (i64 0), (LDXRW am_noindex:$addr), sub_32)>;
247 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_arm64_ldaxr node:$ptr), [{
248 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
251 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_arm64_ldaxr node:$ptr), [{
252 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
255 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_arm64_ldaxr node:$ptr), [{
256 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
259 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_arm64_ldaxr node:$ptr), [{
260 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
263 def : Pat<(ldaxr_1 am_noindex:$addr),
264 (SUBREG_TO_REG (i64 0), (LDAXRB am_noindex:$addr), sub_32)>;
265 def : Pat<(ldaxr_2 am_noindex:$addr),
266 (SUBREG_TO_REG (i64 0), (LDAXRH am_noindex:$addr), sub_32)>;
267 def : Pat<(ldaxr_4 am_noindex:$addr),
268 (SUBREG_TO_REG (i64 0), (LDAXRW am_noindex:$addr), sub_32)>;
269 def : Pat<(ldaxr_8 am_noindex:$addr), (LDAXRX am_noindex:$addr)>;
271 def : Pat<(and (ldaxr_1 am_noindex:$addr), 0xff),
272 (SUBREG_TO_REG (i64 0), (LDAXRB am_noindex:$addr), sub_32)>;
273 def : Pat<(and (ldaxr_2 am_noindex:$addr), 0xffff),
274 (SUBREG_TO_REG (i64 0), (LDAXRH am_noindex:$addr), sub_32)>;
275 def : Pat<(and (ldaxr_4 am_noindex:$addr), 0xffffffff),
276 (SUBREG_TO_REG (i64 0), (LDAXRW am_noindex:$addr), sub_32)>;
280 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
281 (int_arm64_stxr node:$val, node:$ptr), [{
282 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
285 def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
286 (int_arm64_stxr node:$val, node:$ptr), [{
287 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
290 def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
291 (int_arm64_stxr node:$val, node:$ptr), [{
292 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
295 def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
296 (int_arm64_stxr node:$val, node:$ptr), [{
297 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
301 def : Pat<(stxr_1 GPR64:$val, am_noindex:$addr),
302 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
303 def : Pat<(stxr_2 GPR64:$val, am_noindex:$addr),
304 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
305 def : Pat<(stxr_4 GPR64:$val, am_noindex:$addr),
306 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
307 def : Pat<(stxr_8 GPR64:$val, am_noindex:$addr),
308 (STXRX GPR64:$val, am_noindex:$addr)>;
310 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), am_noindex:$addr),
311 (STXRB GPR32:$val, am_noindex:$addr)>;
312 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), am_noindex:$addr),
313 (STXRH GPR32:$val, am_noindex:$addr)>;
314 def : Pat<(stxr_4 (zext GPR32:$val), am_noindex:$addr),
315 (STXRW GPR32:$val, am_noindex:$addr)>;
317 def : Pat<(stxr_1 (and GPR64:$val, 0xff), am_noindex:$addr),
318 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
319 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), am_noindex:$addr),
320 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
321 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), am_noindex:$addr),
322 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
324 // Store-release-exclusives.
326 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
327 (int_arm64_stlxr node:$val, node:$ptr), [{
328 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
331 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
332 (int_arm64_stlxr node:$val, node:$ptr), [{
333 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
336 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
337 (int_arm64_stlxr node:$val, node:$ptr), [{
338 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
341 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
342 (int_arm64_stlxr node:$val, node:$ptr), [{
343 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
347 def : Pat<(stlxr_1 GPR64:$val, am_noindex:$addr),
348 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
349 def : Pat<(stlxr_2 GPR64:$val, am_noindex:$addr),
350 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
351 def : Pat<(stlxr_4 GPR64:$val, am_noindex:$addr),
352 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
353 def : Pat<(stlxr_8 GPR64:$val, am_noindex:$addr),
354 (STLXRX GPR64:$val, am_noindex:$addr)>;
356 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), am_noindex:$addr),
357 (STLXRB GPR32:$val, am_noindex:$addr)>;
358 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), am_noindex:$addr),
359 (STLXRH GPR32:$val, am_noindex:$addr)>;
360 def : Pat<(stlxr_4 (zext GPR32:$val), am_noindex:$addr),
361 (STLXRW GPR32:$val, am_noindex:$addr)>;
363 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), am_noindex:$addr),
364 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
365 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), am_noindex:$addr),
366 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
367 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), am_noindex:$addr),
368 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), am_noindex:$addr)>;
371 // And clear exclusive.
373 def : Pat<(int_arm64_clrex), (CLREX 0xf)>;