1 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // AArch64 Atomic operand code-gen constructs.
12 //===----------------------------------------------------------------------===//
14 //===----------------------------------
16 //===----------------------------------
17 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
18 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
20 //===----------------------------------
22 //===----------------------------------
24 // When they're actually atomic, only one addressing mode (GPR64sp) is
25 // supported, but when they're relaxed and anything can be used, all the
26 // standard modes would be valid and may give efficiency gains.
28 // A atomic load operation that actually needs acquire semantics.
29 class acquiring_load<PatFrag base>
30 : PatFrag<(ops node:$ptr), (base node:$ptr), [{
31 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
32 assert(Ordering != AcquireRelease && "unexpected load ordering");
33 return Ordering == Acquire || Ordering == SequentiallyConsistent;
36 // An atomic load operation that does not need either acquire or release
38 class relaxed_load<PatFrag base>
39 : PatFrag<(ops node:$ptr), (base node:$ptr), [{
40 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
41 return Ordering == Monotonic || Ordering == Unordered;
45 def : Pat<(acquiring_load<atomic_load_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
46 def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
47 ro_Wextend8:$offset)),
48 (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
49 def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
50 ro_Xextend8:$offset)),
51 (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
52 def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn,
54 (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
55 def : Pat<(relaxed_load<atomic_load_8>
56 (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
57 (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
60 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
61 def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
62 ro_Wextend16:$extend)),
63 (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
64 def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
65 ro_Xextend16:$extend)),
66 (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
67 def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn,
69 (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
70 def : Pat<(relaxed_load<atomic_load_16>
71 (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
72 (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
75 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
76 def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
77 ro_Wextend32:$extend)),
78 (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
79 def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
80 ro_Xextend32:$extend)),
81 (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
82 def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
84 (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
85 def : Pat<(relaxed_load<atomic_load_32>
86 (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
87 (LDURWi GPR64sp:$Rn, simm9:$offset)>;
90 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
91 def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
92 ro_Wextend64:$extend)),
93 (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
94 def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
95 ro_Xextend64:$extend)),
96 (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
97 def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
99 (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
100 def : Pat<(relaxed_load<atomic_load_64>
101 (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
102 (LDURXi GPR64sp:$Rn, simm9:$offset)>;
104 //===----------------------------------
106 //===----------------------------------
108 // When they're actually atomic, only one addressing mode (GPR64sp) is
109 // supported, but when they're relaxed and anything can be used, all the
110 // standard modes would be valid and may give efficiency gains.
112 // A store operation that actually needs release semantics.
113 class releasing_store<PatFrag base>
114 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
115 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
116 assert(Ordering != AcquireRelease && "unexpected store ordering");
117 return Ordering == Release || Ordering == SequentiallyConsistent;
120 // An atomic store operation that doesn't actually need to be atomic on AArch64.
121 class relaxed_store<PatFrag base>
122 : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
123 AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getOrdering();
124 return Ordering == Monotonic || Ordering == Unordered;
128 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
129 (STLRB GPR32:$val, GPR64sp:$ptr)>;
130 def : Pat<(relaxed_store<atomic_store_8>
131 (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
133 (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
134 def : Pat<(relaxed_store<atomic_store_8>
135 (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
137 (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
138 def : Pat<(relaxed_store<atomic_store_8>
139 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
140 (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
141 def : Pat<(relaxed_store<atomic_store_8>
142 (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
143 (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
146 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
147 (STLRH GPR32:$val, GPR64sp:$ptr)>;
148 def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
149 ro_Wextend16:$extend),
151 (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
152 def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
153 ro_Xextend16:$extend),
155 (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
156 def : Pat<(relaxed_store<atomic_store_16>
157 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
158 (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
159 def : Pat<(relaxed_store<atomic_store_16>
160 (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
161 (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
164 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
165 (STLRW GPR32:$val, GPR64sp:$ptr)>;
166 def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
167 ro_Wextend32:$extend),
169 (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
170 def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
171 ro_Xextend32:$extend),
173 (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
174 def : Pat<(relaxed_store<atomic_store_32>
175 (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
176 (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
177 def : Pat<(relaxed_store<atomic_store_32>
178 (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
179 (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
182 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
183 (STLRX GPR64:$val, GPR64sp:$ptr)>;
184 def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
185 ro_Wextend16:$extend),
187 (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
188 def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
189 ro_Xextend16:$extend),
191 (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
192 def : Pat<(relaxed_store<atomic_store_64>
193 (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
194 (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
195 def : Pat<(relaxed_store<atomic_store_64>
196 (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
197 (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
199 //===----------------------------------
200 // Low-level exclusive operations
201 //===----------------------------------
205 def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
206 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
209 def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
210 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
213 def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
214 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
217 def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
218 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
221 def : Pat<(ldxr_1 GPR64sp:$addr),
222 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
223 def : Pat<(ldxr_2 GPR64sp:$addr),
224 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
225 def : Pat<(ldxr_4 GPR64sp:$addr),
226 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
227 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
229 def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
230 (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
231 def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
232 (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
233 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
234 (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
238 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
239 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
242 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
243 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
246 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
247 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
250 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
251 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
254 def : Pat<(ldaxr_1 GPR64sp:$addr),
255 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
256 def : Pat<(ldaxr_2 GPR64sp:$addr),
257 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
258 def : Pat<(ldaxr_4 GPR64sp:$addr),
259 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
260 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
262 def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
263 (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
264 def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
265 (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
266 def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
267 (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
271 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
272 (int_aarch64_stxr node:$val, node:$ptr), [{
273 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
276 def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
277 (int_aarch64_stxr node:$val, node:$ptr), [{
278 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
281 def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
282 (int_aarch64_stxr node:$val, node:$ptr), [{
283 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
286 def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
287 (int_aarch64_stxr node:$val, node:$ptr), [{
288 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
292 def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
293 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
294 def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
295 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
296 def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
297 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
298 def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
299 (STXRX GPR64:$val, GPR64sp:$addr)>;
301 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
302 (STXRB GPR32:$val, GPR64sp:$addr)>;
303 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
304 (STXRH GPR32:$val, GPR64sp:$addr)>;
305 def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
306 (STXRW GPR32:$val, GPR64sp:$addr)>;
308 def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
309 (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
310 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
311 (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
312 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
313 (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
315 // Store-release-exclusives.
317 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
318 (int_aarch64_stlxr node:$val, node:$ptr), [{
319 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
322 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
323 (int_aarch64_stlxr node:$val, node:$ptr), [{
324 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
327 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
328 (int_aarch64_stlxr node:$val, node:$ptr), [{
329 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
332 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
333 (int_aarch64_stlxr node:$val, node:$ptr), [{
334 return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
338 def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
339 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
340 def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
341 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
342 def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
343 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
344 def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
345 (STLXRX GPR64:$val, GPR64sp:$addr)>;
347 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
348 (STLXRB GPR32:$val, GPR64sp:$addr)>;
349 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
350 (STLXRH GPR32:$val, GPR64sp:$addr)>;
351 def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
352 (STLXRW GPR32:$val, GPR64sp:$addr)>;
354 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
355 (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
356 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
357 (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
358 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
359 (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
362 // And clear exclusive.
364 def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;