1 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the AArch64 implementation of the TargetInstrInfo class.
11 //
12 //===----------------------------------------------------------------------===//
14 #include "AArch64InstrInfo.h"
15 #include "AArch64Subtarget.h"
16 #include "MCTargetDesc/AArch64AddressingModes.h"
17 #include "AArch64MachineCombinerPattern.h"
18 #include "llvm/CodeGen/MachineFrameInfo.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineMemOperand.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/CodeGen/PseudoSourceValue.h"
23 #include "llvm/MC/MCInst.h"
24 #include "llvm/Support/ErrorHandling.h"
25 #include "llvm/Support/TargetRegistry.h"
27 using namespace llvm;
29 #define GET_INSTRINFO_CTOR_DTOR
30 #include "AArch64GenInstrInfo.inc"
32 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
33 : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP),
34 RI(this, &STI), Subtarget(STI) {}
36 /// GetInstSize - Return the number of bytes of code the specified
37 /// instruction may be. This returns the maximum number of bytes.
38 unsigned AArch64InstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
39 const MachineBasicBlock &MBB = *MI->getParent();
40 const MachineFunction *MF = MBB.getParent();
41 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
43 if (MI->getOpcode() == AArch64::INLINEASM)
44 return getInlineAsmLength(MI->getOperand(0).getSymbolName(), *MAI);
46 const MCInstrDesc &Desc = MI->getDesc();
47 switch (Desc.getOpcode()) {
48 default:
49 // Anything not explicitly designated otherwise is a nomal 4-byte insn.
50 return 4;
51 case TargetOpcode::DBG_VALUE:
52 case TargetOpcode::EH_LABEL:
53 case TargetOpcode::IMPLICIT_DEF:
54 case TargetOpcode::KILL:
55 return 0;
56 }
58 llvm_unreachable("GetInstSizeInBytes()- Unable to determin insn size");
59 }
61 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
62 SmallVectorImpl<MachineOperand> &Cond) {
63 // Block ends with fall-through condbranch.
64 switch (LastInst->getOpcode()) {
65 default:
66 llvm_unreachable("Unknown branch instruction?");
67 case AArch64::Bcc:
68 Target = LastInst->getOperand(1).getMBB();
69 Cond.push_back(LastInst->getOperand(0));
70 break;
71 case AArch64::CBZW:
72 case AArch64::CBZX:
73 case AArch64::CBNZW:
74 case AArch64::CBNZX:
75 Target = LastInst->getOperand(1).getMBB();
76 Cond.push_back(MachineOperand::CreateImm(-1));
77 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
78 Cond.push_back(LastInst->getOperand(0));
79 break;
80 case AArch64::TBZW:
81 case AArch64::TBZX:
82 case AArch64::TBNZW:
83 case AArch64::TBNZX:
84 Target = LastInst->getOperand(2).getMBB();
85 Cond.push_back(MachineOperand::CreateImm(-1));
86 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
87 Cond.push_back(LastInst->getOperand(0));
88 Cond.push_back(LastInst->getOperand(1));
89 }
90 }
92 // Branch analysis.
93 bool AArch64InstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
94 MachineBasicBlock *&TBB,
95 MachineBasicBlock *&FBB,
96 SmallVectorImpl<MachineOperand> &Cond,
97 bool AllowModify) const {
98 // If the block has no terminators, it just falls into the block after it.
99 MachineBasicBlock::iterator I = MBB.end();
100 if (I == MBB.begin())
101 return false;
102 --I;
103 while (I->isDebugValue()) {
104 if (I == MBB.begin())
105 return false;
106 --I;
107 }
108 if (!isUnpredicatedTerminator(I))
109 return false;
111 // Get the last instruction in the block.
112 MachineInstr *LastInst = I;
114 // If there is only one terminator instruction, process it.
115 unsigned LastOpc = LastInst->getOpcode();
116 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
117 if (isUncondBranchOpcode(LastOpc)) {
118 TBB = LastInst->getOperand(0).getMBB();
119 return false;
120 }
121 if (isCondBranchOpcode(LastOpc)) {
122 // Block ends with fall-through condbranch.
123 parseCondBranch(LastInst, TBB, Cond);
124 return false;
125 }
126 return true; // Can't handle indirect branch.
127 }
129 // Get the instruction before it if it is a terminator.
130 MachineInstr *SecondLastInst = I;
131 unsigned SecondLastOpc = SecondLastInst->getOpcode();
133 // If AllowModify is true and the block ends with two or more unconditional
134 // branches, delete all but the first unconditional branch.
135 if (AllowModify && isUncondBranchOpcode(LastOpc)) {
136 while (isUncondBranchOpcode(SecondLastOpc)) {
137 LastInst->eraseFromParent();
138 LastInst = SecondLastInst;
139 LastOpc = LastInst->getOpcode();
140 if (I == MBB.begin() || !isUnpredicatedTerminator(--I)) {
141 // Return now the only terminator is an unconditional branch.
142 TBB = LastInst->getOperand(0).getMBB();
143 return false;
144 } else {
145 SecondLastInst = I;
146 SecondLastOpc = SecondLastInst->getOpcode();
147 }
148 }
149 }
151 // If there are three terminators, we don't know what sort of block this is.
152 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(--I))
153 return true;
155 // If the block ends with a B and a Bcc, handle it.
156 if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
157 parseCondBranch(SecondLastInst, TBB, Cond);
158 FBB = LastInst->getOperand(0).getMBB();
159 return false;
160 }
162 // If the block ends with two unconditional branches, handle it. The second
163 // one is not executed, so remove it.
164 if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
165 TBB = SecondLastInst->getOperand(0).getMBB();
166 I = LastInst;
167 if (AllowModify)
168 I->eraseFromParent();
169 return false;
170 }
172 // ...likewise if it ends with an indirect branch followed by an unconditional
173 // branch.
174 if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
175 I = LastInst;
176 if (AllowModify)
177 I->eraseFromParent();
178 return true;
179 }
181 // Otherwise, can't handle this.
182 return true;
183 }
185 bool AArch64InstrInfo::ReverseBranchCondition(
186 SmallVectorImpl<MachineOperand> &Cond) const {
187 if (Cond[0].getImm() != -1) {
188 // Regular Bcc
189 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
190 Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
191 } else {
192 // Folded compare-and-branch
193 switch (Cond[1].getImm()) {
194 default:
195 llvm_unreachable("Unknown conditional branch!");
196 case AArch64::CBZW:
197 Cond[1].setImm(AArch64::CBNZW);
198 break;
199 case AArch64::CBNZW:
200 Cond[1].setImm(AArch64::CBZW);
201 break;
202 case AArch64::CBZX:
203 Cond[1].setImm(AArch64::CBNZX);
204 break;
205 case AArch64::CBNZX:
206 Cond[1].setImm(AArch64::CBZX);
207 break;
208 case AArch64::TBZW:
209 Cond[1].setImm(AArch64::TBNZW);
210 break;
211 case AArch64::TBNZW:
212 Cond[1].setImm(AArch64::TBZW);
213 break;
214 case AArch64::TBZX:
215 Cond[1].setImm(AArch64::TBNZX);
216 break;
217 case AArch64::TBNZX:
218 Cond[1].setImm(AArch64::TBZX);
219 break;
220 }
221 }
223 return false;
224 }
226 unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
227 MachineBasicBlock::iterator I = MBB.end();
228 if (I == MBB.begin())
229 return 0;
230 --I;
231 while (I->isDebugValue()) {
232 if (I == MBB.begin())
233 return 0;
234 --I;
235 }
236 if (!isUncondBranchOpcode(I->getOpcode()) &&
237 !isCondBranchOpcode(I->getOpcode()))
238 return 0;
240 // Remove the branch.
241 I->eraseFromParent();
243 I = MBB.end();
245 if (I == MBB.begin())
246 return 1;
247 --I;
248 if (!isCondBranchOpcode(I->getOpcode()))
249 return 1;
251 // Remove the branch.
252 I->eraseFromParent();
253 return 2;
254 }
256 void AArch64InstrInfo::instantiateCondBranch(
257 MachineBasicBlock &MBB, DebugLoc DL, MachineBasicBlock *TBB,
258 const SmallVectorImpl<MachineOperand> &Cond) const {
259 if (Cond[0].getImm() != -1) {
260 // Regular Bcc
261 BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
262 } else {
263 // Folded compare-and-branch
264 const MachineInstrBuilder MIB =
265 BuildMI(&MBB, DL, get(Cond[1].getImm())).addReg(Cond[2].getReg());
266 if (Cond.size() > 3)
267 MIB.addImm(Cond[3].getImm());
268 MIB.addMBB(TBB);
269 }
270 }
272 unsigned AArch64InstrInfo::InsertBranch(
273 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
274 const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const {
275 // Shouldn't be a fall through.
276 assert(TBB && "InsertBranch must not be told to insert a fallthrough");
278 if (!FBB) {
279 if (Cond.empty()) // Unconditional branch?
280 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
281 else
282 instantiateCondBranch(MBB, DL, TBB, Cond);
283 return 1;
284 }
286 // Two-way conditional branch.
287 instantiateCondBranch(MBB, DL, TBB, Cond);
288 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
289 return 2;
290 }
292 // Find the original register that VReg is copied from.
293 static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
294 while (TargetRegisterInfo::isVirtualRegister(VReg)) {
295 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
296 if (!DefMI->isFullCopy())
297 return VReg;
298 VReg = DefMI->getOperand(1).getReg();
299 }
300 return VReg;
301 }
303 // Determine if VReg is defined by an instruction that can be folded into a
304 // csel instruction. If so, return the folded opcode, and the replacement
305 // register.
306 static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
307 unsigned *NewVReg = nullptr) {
308 VReg = removeCopies(MRI, VReg);
309 if (!TargetRegisterInfo::isVirtualRegister(VReg))
310 return 0;
312 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
313 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
314 unsigned Opc = 0;
315 unsigned SrcOpNum = 0;
316 switch (DefMI->getOpcode()) {
317 case AArch64::ADDSXri:
318 case AArch64::ADDSWri:
319 // if NZCV is used, do not fold.
320 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
321 return 0;
322 // fall-through to ADDXri and ADDWri.
323 case AArch64::ADDXri:
324 case AArch64::ADDWri:
325 // add x, 1 -> csinc.
326 if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
327 DefMI->getOperand(3).getImm() != 0)
328 return 0;
329 SrcOpNum = 1;
330 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
331 break;
333 case AArch64::ORNXrr:
334 case AArch64::ORNWrr: {
335 // not x -> csinv, represented as orn dst, xzr, src.
336 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
337 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
338 return 0;
339 SrcOpNum = 2;
340 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
341 break;
342 }
344 case AArch64::SUBSXrr:
345 case AArch64::SUBSWrr:
346 // if NZCV is used, do not fold.
347 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
348 return 0;
349 // fall-through to SUBXrr and SUBWrr.
350 case AArch64::SUBXrr:
351 case AArch64::SUBWrr: {
352 // neg x -> csneg, represented as sub dst, xzr, src.
353 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
354 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
355 return 0;
356 SrcOpNum = 2;
357 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
358 break;
359 }
360 default:
361 return 0;
362 }
363 assert(Opc && SrcOpNum && "Missing parameters");
365 if (NewVReg)
366 *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
367 return Opc;
368 }
370 bool AArch64InstrInfo::canInsertSelect(
371 const MachineBasicBlock &MBB, const SmallVectorImpl<MachineOperand> &Cond,
372 unsigned TrueReg, unsigned FalseReg, int &CondCycles, int &TrueCycles,
373 int &FalseCycles) const {
374 // Check register classes.
375 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
376 const TargetRegisterClass *RC =
377 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
378 if (!RC)
379 return false;
381 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
382 unsigned ExtraCondLat = Cond.size() != 1;
384 // GPRs are handled by csel.
385 // FIXME: Fold in x+1, -x, and ~x when applicable.
386 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
387 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
388 // Single-cycle csel, csinc, csinv, and csneg.
389 CondCycles = 1 + ExtraCondLat;
390 TrueCycles = FalseCycles = 1;
391 if (canFoldIntoCSel(MRI, TrueReg))
392 TrueCycles = 0;
393 else if (canFoldIntoCSel(MRI, FalseReg))
394 FalseCycles = 0;
395 return true;
396 }
398 // Scalar floating point is handled by fcsel.
399 // FIXME: Form fabs, fmin, and fmax when applicable.
400 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
401 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
402 CondCycles = 5 + ExtraCondLat;
403 TrueCycles = FalseCycles = 2;
404 return true;
405 }
407 // Can't do vectors.
408 return false;
409 }
411 void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
412 MachineBasicBlock::iterator I, DebugLoc DL,
413 unsigned DstReg,
414 const SmallVectorImpl<MachineOperand> &Cond,
415 unsigned TrueReg, unsigned FalseReg) const {
416 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
418 // Parse the condition code, see parseCondBranch() above.
419 AArch64CC::CondCode CC;
420 switch (Cond.size()) {
421 default:
422 llvm_unreachable("Unknown condition opcode in Cond");
423 case 1: // b.cc
424 CC = AArch64CC::CondCode(Cond[0].getImm());
425 break;
426 case 3: { // cbz/cbnz
427 // We must insert a compare against 0.
428 bool Is64Bit;
429 switch (Cond[1].getImm()) {
430 default:
431 llvm_unreachable("Unknown branch opcode in Cond");
432 case AArch64::CBZW:
433 Is64Bit = 0;
434 CC = AArch64CC::EQ;
435 break;
436 case AArch64::CBZX:
437 Is64Bit = 1;
438 CC = AArch64CC::EQ;
439 break;
440 case AArch64::CBNZW:
441 Is64Bit = 0;
442 CC = AArch64CC::NE;
443 break;
444 case AArch64::CBNZX:
445 Is64Bit = 1;
446 CC = AArch64CC::NE;
447 break;
448 }
449 unsigned SrcReg = Cond[2].getReg();
450 if (Is64Bit) {
451 // cmp reg, #0 is actually subs xzr, reg, #0.
452 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
453 BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
454 .addReg(SrcReg)
455 .addImm(0)
456 .addImm(0);
457 } else {
458 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
459 BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
460 .addReg(SrcReg)
461 .addImm(0)
462 .addImm(0);
463 }
464 break;
465 }
466 case 4: { // tbz/tbnz
467 // We must insert a tst instruction.
468 switch (Cond[1].getImm()) {
469 default:
470 llvm_unreachable("Unknown branch opcode in Cond");
471 case AArch64::TBZW:
472 case AArch64::TBZX:
473 CC = AArch64CC::EQ;
474 break;
475 case AArch64::TBNZW:
476 case AArch64::TBNZX:
477 CC = AArch64CC::NE;
478 break;
479 }
480 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
481 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
482 BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
483 .addReg(Cond[2].getReg())
484 .addImm(
485 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
486 else
487 BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
488 .addReg(Cond[2].getReg())
489 .addImm(
490 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
491 break;
492 }
493 }
495 unsigned Opc = 0;
496 const TargetRegisterClass *RC = nullptr;
497 bool TryFold = false;
498 if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
499 RC = &AArch64::GPR64RegClass;
500 Opc = AArch64::CSELXr;
501 TryFold = true;
502 } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
503 RC = &AArch64::GPR32RegClass;
504 Opc = AArch64::CSELWr;
505 TryFold = true;
506 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
507 RC = &AArch64::FPR64RegClass;
508 Opc = AArch64::FCSELDrrr;
509 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
510 RC = &AArch64::FPR32RegClass;
511 Opc = AArch64::FCSELSrrr;
512 }
513 assert(RC && "Unsupported regclass");
515 // Try folding simple instructions into the csel.
516 if (TryFold) {
517 unsigned NewVReg = 0;
518 unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
519 if (FoldedOpc) {
520 // The folded opcodes csinc, csinc and csneg apply the operation to
521 // FalseReg, so we need to invert the condition.
522 CC = AArch64CC::getInvertedCondCode(CC);
523 TrueReg = FalseReg;
524 } else
525 FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
527 // Fold the operation. Leave any dead instructions for DCE to clean up.
528 if (FoldedOpc) {
529 FalseReg = NewVReg;
530 Opc = FoldedOpc;
531 // The extends the live range of NewVReg.
532 MRI.clearKillFlags(NewVReg);
533 }
534 }
536 // Pull all virtual register into the appropriate class.
537 MRI.constrainRegClass(TrueReg, RC);
538 MRI.constrainRegClass(FalseReg, RC);
540 // Insert the csel.
541 BuildMI(MBB, I, DL, get(Opc), DstReg).addReg(TrueReg).addReg(FalseReg).addImm(
542 CC);
543 }
545 // FIXME: this implementation should be micro-architecture dependent, so a
546 // micro-architecture target hook should be introduced here in future.
547 bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr *MI) const {
548 if (!Subtarget.isCortexA57() && !Subtarget.isCortexA53())
549 return MI->isAsCheapAsAMove();
551 switch (MI->getOpcode()) {
552 default:
553 return false;
555 // add/sub on register without shift
556 case AArch64::ADDWri:
557 case AArch64::ADDXri:
558 case AArch64::SUBWri:
559 case AArch64::SUBXri:
560 return (MI->getOperand(3).getImm() == 0);
562 // logical ops on immediate
563 case AArch64::ANDWri:
564 case AArch64::ANDXri:
565 case AArch64::EORWri:
566 case AArch64::EORXri:
567 case AArch64::ORRWri:
568 case AArch64::ORRXri:
569 return true;
571 // logical ops on register without shift
572 case AArch64::ANDWrr:
573 case AArch64::ANDXrr:
574 case AArch64::BICWrr:
575 case AArch64::BICXrr:
576 case AArch64::EONWrr:
577 case AArch64::EONXrr:
578 case AArch64::EORWrr:
579 case AArch64::EORXrr:
580 case AArch64::ORNWrr:
581 case AArch64::ORNXrr:
582 case AArch64::ORRWrr:
583 case AArch64::ORRXrr:
584 return true;
585 }
587 llvm_unreachable("Unknown opcode to check as cheap as a move!");
588 }
590 bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
591 unsigned &SrcReg, unsigned &DstReg,
592 unsigned &SubIdx) const {
593 switch (MI.getOpcode()) {
594 default:
595 return false;
596 case AArch64::SBFMXri: // aka sxtw
597 case AArch64::UBFMXri: // aka uxtw
598 // Check for the 32 -> 64 bit extension case, these instructions can do
599 // much more.
600 if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
601 return false;
602 // This is a signed or unsigned 32 -> 64 bit extension.
603 SrcReg = MI.getOperand(1).getReg();
604 DstReg = MI.getOperand(0).getReg();
605 SubIdx = AArch64::sub_32;
606 return true;
607 }
608 }
610 bool
611 AArch64InstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr *MIa,
612 MachineInstr *MIb,
613 AliasAnalysis *AA) const {
614 const TargetRegisterInfo *TRI = &getRegisterInfo();
615 unsigned BaseRegA = 0, BaseRegB = 0;
616 int OffsetA = 0, OffsetB = 0;
617 int WidthA = 0, WidthB = 0;
619 assert(MIa && (MIa->mayLoad() || MIa->mayStore()) &&
620 "MIa must be a store or a load");
621 assert(MIb && (MIb->mayLoad() || MIb->mayStore()) &&
622 "MIb must be a store or a load");
624 if (MIa->hasUnmodeledSideEffects() || MIb->hasUnmodeledSideEffects() ||
625 MIa->hasOrderedMemoryRef() || MIb->hasOrderedMemoryRef())
626 return false;
628 // Retrieve the base register, offset from the base register and width. Width
629 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
630 // base registers are identical, and the offset of a lower memory access +
631 // the width doesn't overlap the offset of a higher memory access,
632 // then the memory accesses are different.
633 if (getLdStBaseRegImmOfsWidth(MIa, BaseRegA, OffsetA, WidthA, TRI) &&
634 getLdStBaseRegImmOfsWidth(MIb, BaseRegB, OffsetB, WidthB, TRI)) {
635 if (BaseRegA == BaseRegB) {
636 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
637 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
638 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
639 if (LowOffset + LowWidth <= HighOffset)
640 return true;
641 }
642 }
643 return false;
644 }
646 /// analyzeCompare - For a comparison instruction, return the source registers
647 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
648 /// Return true if the comparison instruction can be analyzed.
649 bool AArch64InstrInfo::analyzeCompare(const MachineInstr *MI, unsigned &SrcReg,
650 unsigned &SrcReg2, int &CmpMask,
651 int &CmpValue) const {
652 switch (MI->getOpcode()) {
653 default:
654 break;
655 case AArch64::SUBSWrr:
656 case AArch64::SUBSWrs:
657 case AArch64::SUBSWrx:
658 case AArch64::SUBSXrr:
659 case AArch64::SUBSXrs:
660 case AArch64::SUBSXrx:
661 case AArch64::ADDSWrr:
662 case AArch64::ADDSWrs:
663 case AArch64::ADDSWrx:
664 case AArch64::ADDSXrr:
665 case AArch64::ADDSXrs:
666 case AArch64::ADDSXrx:
667 // Replace SUBSWrr with SUBWrr if NZCV is not used.
668 SrcReg = MI->getOperand(1).getReg();
669 SrcReg2 = MI->getOperand(2).getReg();
670 CmpMask = ~0;
671 CmpValue = 0;
672 return true;
673 case AArch64::SUBSWri:
674 case AArch64::ADDSWri:
675 case AArch64::SUBSXri:
676 case AArch64::ADDSXri:
677 SrcReg = MI->getOperand(1).getReg();
678 SrcReg2 = 0;
679 CmpMask = ~0;
680 // FIXME: In order to convert CmpValue to 0 or 1
681 CmpValue = (MI->getOperand(2).getImm() != 0);
682 return true;
683 case AArch64::ANDSWri:
684 case AArch64::ANDSXri:
685 // ANDS does not use the same encoding scheme as the others xxxS
686 // instructions.
687 SrcReg = MI->getOperand(1).getReg();
688 SrcReg2 = 0;
689 CmpMask = ~0;
690 // FIXME:The return val type of decodeLogicalImmediate is uint64_t,
691 // while the type of CmpValue is int. When converting uint64_t to int,
692 // the high 32 bits of uint64_t will be lost.
693 // In fact it causes a bug in spec2006-483.xalancbmk
694 // CmpValue is only used to compare with zero in OptimizeCompareInstr
695 CmpValue = (AArch64_AM::decodeLogicalImmediate(
696 MI->getOperand(2).getImm(),
697 MI->getOpcode() == AArch64::ANDSWri ? 32 : 64) != 0);
698 return true;
699 }
701 return false;
702 }
704 static bool UpdateOperandRegClass(MachineInstr *Instr) {
705 MachineBasicBlock *MBB = Instr->getParent();
706 assert(MBB && "Can't get MachineBasicBlock here");
707 MachineFunction *MF = MBB->getParent();
708 assert(MF && "Can't get MachineFunction here");
709 const TargetMachine *TM = &MF->getTarget();
710 const TargetInstrInfo *TII = TM->getSubtargetImpl()->getInstrInfo();
711 const TargetRegisterInfo *TRI = TM->getSubtargetImpl()->getRegisterInfo();
712 MachineRegisterInfo *MRI = &MF->getRegInfo();
714 for (unsigned OpIdx = 0, EndIdx = Instr->getNumOperands(); OpIdx < EndIdx;
715 ++OpIdx) {
716 MachineOperand &MO = Instr->getOperand(OpIdx);
717 const TargetRegisterClass *OpRegCstraints =
718 Instr->getRegClassConstraint(OpIdx, TII, TRI);
720 // If there's no constraint, there's nothing to do.
721 if (!OpRegCstraints)
722 continue;
723 // If the operand is a frame index, there's nothing to do here.
724 // A frame index operand will resolve correctly during PEI.
725 if (MO.isFI())
726 continue;
728 assert(MO.isReg() &&
729 "Operand has register constraints without being a register!");
731 unsigned Reg = MO.getReg();
732 if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
733 if (!OpRegCstraints->contains(Reg))
734 return false;
735 } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
736 !MRI->constrainRegClass(Reg, OpRegCstraints))
737 return false;
738 }
740 return true;
741 }
743 /// convertFlagSettingOpcode - return opcode that does not
744 /// set flags when possible. The caller is responsible to do
745 /// the actual substitution and legality checking.
746 static unsigned convertFlagSettingOpcode(MachineInstr *MI) {
747 unsigned NewOpc;
748 switch (MI->getOpcode()) {
749 default:
750 return false;
751 case AArch64::ADDSWrr: NewOpc = AArch64::ADDWrr; break;
752 case AArch64::ADDSWri: NewOpc = AArch64::ADDWri; break;
753 case AArch64::ADDSWrs: NewOpc = AArch64::ADDWrs; break;
754 case AArch64::ADDSWrx: NewOpc = AArch64::ADDWrx; break;
755 case AArch64::ADDSXrr: NewOpc = AArch64::ADDXrr; break;
756 case AArch64::ADDSXri: NewOpc = AArch64::ADDXri; break;
757 case AArch64::ADDSXrs: NewOpc = AArch64::ADDXrs; break;
758 case AArch64::ADDSXrx: NewOpc = AArch64::ADDXrx; break;
759 case AArch64::SUBSWrr: NewOpc = AArch64::SUBWrr; break;
760 case AArch64::SUBSWri: NewOpc = AArch64::SUBWri; break;
761 case AArch64::SUBSWrs: NewOpc = AArch64::SUBWrs; break;
762 case AArch64::SUBSWrx: NewOpc = AArch64::SUBWrx; break;
763 case AArch64::SUBSXrr: NewOpc = AArch64::SUBXrr; break;
764 case AArch64::SUBSXri: NewOpc = AArch64::SUBXri; break;
765 case AArch64::SUBSXrs: NewOpc = AArch64::SUBXrs; break;
766 case AArch64::SUBSXrx: NewOpc = AArch64::SUBXrx; break;
767 }
768 return NewOpc;
769 }
771 /// optimizeCompareInstr - Convert the instruction supplying the argument to the
772 /// comparison into one that sets the zero bit in the flags register.
773 bool AArch64InstrInfo::optimizeCompareInstr(
774 MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
775 int CmpValue, const MachineRegisterInfo *MRI) const {
777 // Replace SUBSWrr with SUBWrr if NZCV is not used.
778 int Cmp_NZCV = CmpInstr->findRegisterDefOperandIdx(AArch64::NZCV, true);
779 if (Cmp_NZCV != -1) {
780 unsigned Opc = CmpInstr->getOpcode();
781 unsigned NewOpc = convertFlagSettingOpcode(CmpInstr);
782 if (NewOpc == Opc)
783 return false;
784 const MCInstrDesc &MCID = get(NewOpc);
785 CmpInstr->setDesc(MCID);
786 CmpInstr->RemoveOperand(Cmp_NZCV);
787 bool succeeded = UpdateOperandRegClass(CmpInstr);
788 (void)succeeded;
789 assert(succeeded && "Some operands reg class are incompatible!");
790 return true;
791 }
793 // Continue only if we have a "ri" where immediate is zero.
794 // FIXME:CmpValue has already been converted to 0 or 1 in analyzeCompare
795 // function.
796 assert((CmpValue == 0 || CmpValue == 1) && "CmpValue must be 0 or 1!");
797 if (CmpValue != 0 || SrcReg2 != 0)
798 return false;
800 // CmpInstr is a Compare instruction if destination register is not used.
801 if (!MRI->use_nodbg_empty(CmpInstr->getOperand(0).getReg()))
802 return false;
804 // Get the unique definition of SrcReg.
805 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
806 if (!MI)
807 return false;
809 // We iterate backward, starting from the instruction before CmpInstr and
810 // stop when reaching the definition of the source register or done with the
811 // basic block, to check whether NZCV is used or modified in between.
812 MachineBasicBlock::iterator I = CmpInstr, E = MI,
813 B = CmpInstr->getParent()->begin();
815 // Early exit if CmpInstr is at the beginning of the BB.
816 if (I == B)
817 return false;
819 // Check whether the definition of SrcReg is in the same basic block as
820 // Compare. If not, we can't optimize away the Compare.
821 if (MI->getParent() != CmpInstr->getParent())
822 return false;
824 // Check that NZCV isn't set between the comparison instruction and the one we
825 // want to change.
826 const TargetRegisterInfo *TRI = &getRegisterInfo();
827 for (--I; I != E; --I) {
828 const MachineInstr &Instr = *I;
830 if (Instr.modifiesRegister(AArch64::NZCV, TRI) ||
831 Instr.readsRegister(AArch64::NZCV, TRI))
832 // This instruction modifies or uses NZCV after the one we want to
833 // change. We can't do this transformation.
834 return false;
835 if (I == B)
836 // The 'and' is below the comparison instruction.
837 return false;
838 }
840 unsigned NewOpc = MI->getOpcode();
841 switch (MI->getOpcode()) {
842 default:
843 return false;
844 case AArch64::ADDSWrr:
845 case AArch64::ADDSWri:
846 case AArch64::ADDSXrr:
847 case AArch64::ADDSXri:
848 case AArch64::SUBSWrr:
849 case AArch64::SUBSWri:
850 case AArch64::SUBSXrr:
851 case AArch64::SUBSXri:
852 break;
853 case AArch64::ADDWrr: NewOpc = AArch64::ADDSWrr; break;
854 case AArch64::ADDWri: NewOpc = AArch64::ADDSWri; break;
855 case AArch64::ADDXrr: NewOpc = AArch64::ADDSXrr; break;
856 case AArch64::ADDXri: NewOpc = AArch64::ADDSXri; break;
857 case AArch64::ADCWr: NewOpc = AArch64::ADCSWr; break;
858 case AArch64::ADCXr: NewOpc = AArch64::ADCSXr; break;
859 case AArch64::SUBWrr: NewOpc = AArch64::SUBSWrr; break;
860 case AArch64::SUBWri: NewOpc = AArch64::SUBSWri; break;
861 case AArch64::SUBXrr: NewOpc = AArch64::SUBSXrr; break;
862 case AArch64::SUBXri: NewOpc = AArch64::SUBSXri; break;
863 case AArch64::SBCWr: NewOpc = AArch64::SBCSWr; break;
864 case AArch64::SBCXr: NewOpc = AArch64::SBCSXr; break;
865 case AArch64::ANDWri: NewOpc = AArch64::ANDSWri; break;
866 case AArch64::ANDXri: NewOpc = AArch64::ANDSXri; break;
867 }
869 // Scan forward for the use of NZCV.
870 // When checking against MI: if it's a conditional code requires
871 // checking of V bit, then this is not safe to do.
872 // It is safe to remove CmpInstr if NZCV is redefined or killed.
873 // If we are done with the basic block, we need to check whether NZCV is
874 // live-out.
875 bool IsSafe = false;
876 for (MachineBasicBlock::iterator I = CmpInstr,
877 E = CmpInstr->getParent()->end();
878 !IsSafe && ++I != E;) {
879 const MachineInstr &Instr = *I;
880 for (unsigned IO = 0, EO = Instr.getNumOperands(); !IsSafe && IO != EO;
881 ++IO) {
882 const MachineOperand &MO = Instr.getOperand(IO);
883 if (MO.isRegMask() && MO.clobbersPhysReg(AArch64::NZCV)) {
884 IsSafe = true;
885 break;
886 }
887 if (!MO.isReg() || MO.getReg() != AArch64::NZCV)
888 continue;
889 if (MO.isDef()) {
890 IsSafe = true;
891 break;
892 }
894 // Decode the condition code.
895 unsigned Opc = Instr.getOpcode();
896 AArch64CC::CondCode CC;
897 switch (Opc) {
898 default:
899 return false;
900 case AArch64::Bcc:
901 CC = (AArch64CC::CondCode)Instr.getOperand(IO - 2).getImm();
902 break;
903 case AArch64::CSINVWr:
904 case AArch64::CSINVXr:
905 case AArch64::CSINCWr:
906 case AArch64::CSINCXr:
907 case AArch64::CSELWr:
908 case AArch64::CSELXr:
909 case AArch64::CSNEGWr:
910 case AArch64::CSNEGXr:
911 case AArch64::FCSELSrrr:
912 case AArch64::FCSELDrrr:
913 CC = (AArch64CC::CondCode)Instr.getOperand(IO - 1).getImm();
914 break;
915 }
917 // It is not safe to remove Compare instruction if Overflow(V) is used.
918 switch (CC) {
919 default:
920 // NZCV can be used multiple times, we should continue.
921 break;
922 case AArch64CC::VS:
923 case AArch64CC::VC:
924 case AArch64CC::GE:
925 case AArch64CC::LT:
926 case AArch64CC::GT:
927 case AArch64CC::LE:
928 return false;
929 }
930 }
931 }
933 // If NZCV is not killed nor re-defined, we should check whether it is
934 // live-out. If it is live-out, do not optimize.
935 if (!IsSafe) {
936 MachineBasicBlock *ParentBlock = CmpInstr->getParent();
937 for (auto *MBB : ParentBlock->successors())
938 if (MBB->isLiveIn(AArch64::NZCV))
939 return false;
940 }
942 // Update the instruction to set NZCV.
943 MI->setDesc(get(NewOpc));
944 CmpInstr->eraseFromParent();
945 bool succeeded = UpdateOperandRegClass(MI);
946 (void)succeeded;
947 assert(succeeded && "Some operands reg class are incompatible!");
948 MI->addRegisterDefined(AArch64::NZCV, TRI);
949 return true;
950 }
952 bool
953 AArch64InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
954 if (MI->getOpcode() != TargetOpcode::LOAD_STACK_GUARD)
955 return false;
957 MachineBasicBlock &MBB = *MI->getParent();
958 DebugLoc DL = MI->getDebugLoc();
959 unsigned Reg = MI->getOperand(0).getReg();
960 const GlobalValue *GV =
961 cast<GlobalValue>((*MI->memoperands_begin())->getValue());
962 const TargetMachine &TM = MBB.getParent()->getTarget();
963 unsigned char OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
964 const unsigned char MO_NC = AArch64II::MO_NC;
966 if ((OpFlags & AArch64II::MO_GOT) != 0) {
967 BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
968 .addGlobalAddress(GV, 0, AArch64II::MO_GOT);
969 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
970 .addReg(Reg, RegState::Kill).addImm(0)
971 .addMemOperand(*MI->memoperands_begin());
972 } else if (TM.getCodeModel() == CodeModel::Large) {
973 BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
974 .addGlobalAddress(GV, 0, AArch64II::MO_G3).addImm(48);
975 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
976 .addReg(Reg, RegState::Kill)
977 .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC).addImm(32);
978 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
979 .addReg(Reg, RegState::Kill)
980 .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC).addImm(16);
981 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
982 .addReg(Reg, RegState::Kill)
983 .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC).addImm(0);
984 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
985 .addReg(Reg, RegState::Kill).addImm(0)
986 .addMemOperand(*MI->memoperands_begin());
987 } else {
988 BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
989 .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
990 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
991 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
992 .addReg(Reg, RegState::Kill)
993 .addGlobalAddress(GV, 0, LoFlags)
994 .addMemOperand(*MI->memoperands_begin());
995 }
997 MBB.erase(MI);
999 return true;
1000 }
1002 /// Return true if this is this instruction has a non-zero immediate
1003 bool AArch64InstrInfo::hasShiftedReg(const MachineInstr *MI) const {
1004 switch (MI->getOpcode()) {
1005 default:
1006 break;
1007 case AArch64::ADDSWrs:
1008 case AArch64::ADDSXrs:
1009 case AArch64::ADDWrs:
1010 case AArch64::ADDXrs:
1011 case AArch64::ANDSWrs:
1012 case AArch64::ANDSXrs:
1013 case AArch64::ANDWrs:
1014 case AArch64::ANDXrs:
1015 case AArch64::BICSWrs:
1016 case AArch64::BICSXrs:
1017 case AArch64::BICWrs:
1018 case AArch64::BICXrs:
1019 case AArch64::CRC32Brr:
1020 case AArch64::CRC32CBrr:
1021 case AArch64::CRC32CHrr:
1022 case AArch64::CRC32CWrr:
1023 case AArch64::CRC32CXrr:
1024 case AArch64::CRC32Hrr:
1025 case AArch64::CRC32Wrr:
1026 case AArch64::CRC32Xrr:
1027 case AArch64::EONWrs:
1028 case AArch64::EONXrs:
1029 case AArch64::EORWrs:
1030 case AArch64::EORXrs:
1031 case AArch64::ORNWrs:
1032 case AArch64::ORNXrs:
1033 case AArch64::ORRWrs:
1034 case AArch64::ORRXrs:
1035 case AArch64::SUBSWrs:
1036 case AArch64::SUBSXrs:
1037 case AArch64::SUBWrs:
1038 case AArch64::SUBXrs:
1039 if (MI->getOperand(3).isImm()) {
1040 unsigned val = MI->getOperand(3).getImm();
1041 return (val != 0);
1042 }
1043 break;
1044 }
1045 return false;
1046 }
1048 /// Return true if this is this instruction has a non-zero immediate
1049 bool AArch64InstrInfo::hasExtendedReg(const MachineInstr *MI) const {
1050 switch (MI->getOpcode()) {
1051 default:
1052 break;
1053 case AArch64::ADDSWrx:
1054 case AArch64::ADDSXrx:
1055 case AArch64::ADDSXrx64:
1056 case AArch64::ADDWrx:
1057 case AArch64::ADDXrx:
1058 case AArch64::ADDXrx64:
1059 case AArch64::SUBSWrx:
1060 case AArch64::SUBSXrx:
1061 case AArch64::SUBSXrx64:
1062 case AArch64::SUBWrx:
1063 case AArch64::SUBXrx:
1064 case AArch64::SUBXrx64:
1065 if (MI->getOperand(3).isImm()) {
1066 unsigned val = MI->getOperand(3).getImm();
1067 return (val != 0);
1068 }
1069 break;
1070 }
1072 return false;
1073 }
1075 // Return true if this instruction simply sets its single destination register
1076 // to zero. This is equivalent to a register rename of the zero-register.
1077 bool AArch64InstrInfo::isGPRZero(const MachineInstr *MI) const {
1078 switch (MI->getOpcode()) {
1079 default:
1080 break;
1081 case AArch64::MOVZWi:
1082 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
1083 if (MI->getOperand(1).isImm() && MI->getOperand(1).getImm() == 0) {
1084 assert(MI->getDesc().getNumOperands() == 3 &&
1085 MI->getOperand(2).getImm() == 0 && "invalid MOVZi operands");
1086 return true;
1087 }
1088 break;
1089 case AArch64::ANDWri: // and Rd, Rzr, #imm
1090 return MI->getOperand(1).getReg() == AArch64::WZR;
1091 case AArch64::ANDXri:
1092 return MI->getOperand(1).getReg() == AArch64::XZR;
1093 case TargetOpcode::COPY:
1094 return MI->getOperand(1).getReg() == AArch64::WZR;
1095 }
1096 return false;
1097 }
1099 // Return true if this instruction simply renames a general register without
1100 // modifying bits.
1101 bool AArch64InstrInfo::isGPRCopy(const MachineInstr *MI) const {
1102 switch (MI->getOpcode()) {
1103 default:
1104 break;
1105 case TargetOpcode::COPY: {
1106 // GPR32 copies will by lowered to ORRXrs
1107 unsigned DstReg = MI->getOperand(0).getReg();
1108 return (AArch64::GPR32RegClass.contains(DstReg) ||
1109 AArch64::GPR64RegClass.contains(DstReg));
1110 }
1111 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
1112 if (MI->getOperand(1).getReg() == AArch64::XZR) {
1113 assert(MI->getDesc().getNumOperands() == 4 &&
1114 MI->getOperand(3).getImm() == 0 && "invalid ORRrs operands");
1115 return true;
1116 }
1117 break;
1118 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
1119 if (MI->getOperand(2).getImm() == 0) {
1120 assert(MI->getDesc().getNumOperands() == 4 &&
1121 MI->getOperand(3).getImm() == 0 && "invalid ADDXri operands");
1122 return true;
1123 }
1124 break;
1125 }
1126 return false;
1127 }
1129 // Return true if this instruction simply renames a general register without
1130 // modifying bits.
1131 bool AArch64InstrInfo::isFPRCopy(const MachineInstr *MI) const {
1132 switch (MI->getOpcode()) {
1133 default:
1134 break;
1135 case TargetOpcode::COPY: {
1136 // FPR64 copies will by lowered to ORR.16b
1137 unsigned DstReg = MI->getOperand(0).getReg();
1138 return (AArch64::FPR64RegClass.contains(DstReg) ||
1139 AArch64::FPR128RegClass.contains(DstReg));
1140 }
1141 case AArch64::ORRv16i8:
1142 if (MI->getOperand(1).getReg() == MI->getOperand(2).getReg()) {
1143 assert(MI->getDesc().getNumOperands() == 3 && MI->getOperand(0).isReg() &&
1144 "invalid ORRv16i8 operands");
1145 return true;
1146 }
1147 break;
1148 }
1149 return false;
1150 }
1152 unsigned AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr *MI,
1153 int &FrameIndex) const {
1154 switch (MI->getOpcode()) {
1155 default:
1156 break;
1157 case AArch64::LDRWui:
1158 case AArch64::LDRXui:
1159 case AArch64::LDRBui:
1160 case AArch64::LDRHui:
1161 case AArch64::LDRSui:
1162 case AArch64::LDRDui:
1163 case AArch64::LDRQui:
1164 if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
1165 MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
1166 FrameIndex = MI->getOperand(1).getIndex();
1167 return MI->getOperand(0).getReg();
1168 }
1169 break;
1170 }
1172 return 0;
1173 }
1175 unsigned AArch64InstrInfo::isStoreToStackSlot(const MachineInstr *MI,
1176 int &FrameIndex) const {
1177 switch (MI->getOpcode()) {
1178 default:
1179 break;
1180 case AArch64::STRWui:
1181 case AArch64::STRXui:
1182 case AArch64::STRBui:
1183 case AArch64::STRHui:
1184 case AArch64::STRSui:
1185 case AArch64::STRDui:
1186 case AArch64::STRQui:
1187 if (MI->getOperand(0).getSubReg() == 0 && MI->getOperand(1).isFI() &&
1188 MI->getOperand(2).isImm() && MI->getOperand(2).getImm() == 0) {
1189 FrameIndex = MI->getOperand(1).getIndex();
1190 return MI->getOperand(0).getReg();
1191 }
1192 break;
1193 }
1194 return 0;
1195 }
1197 /// Return true if this is load/store scales or extends its register offset.
1198 /// This refers to scaling a dynamic index as opposed to scaled immediates.
1199 /// MI should be a memory op that allows scaled addressing.
1200 bool AArch64InstrInfo::isScaledAddr(const MachineInstr *MI) const {
1201 switch (MI->getOpcode()) {
1202 default:
1203 break;
1204 case AArch64::LDRBBroW:
1205 case AArch64::LDRBroW:
1206 case AArch64::LDRDroW:
1207 case AArch64::LDRHHroW:
1208 case AArch64::LDRHroW:
1209 case AArch64::LDRQroW:
1210 case AArch64::LDRSBWroW:
1211 case AArch64::LDRSBXroW:
1212 case AArch64::LDRSHWroW:
1213 case AArch64::LDRSHXroW:
1214 case AArch64::LDRSWroW:
1215 case AArch64::LDRSroW:
1216 case AArch64::LDRWroW:
1217 case AArch64::LDRXroW:
1218 case AArch64::STRBBroW:
1219 case AArch64::STRBroW:
1220 case AArch64::STRDroW:
1221 case AArch64::STRHHroW:
1222 case AArch64::STRHroW:
1223 case AArch64::STRQroW:
1224 case AArch64::STRSroW:
1225 case AArch64::STRWroW:
1226 case AArch64::STRXroW:
1227 case AArch64::LDRBBroX:
1228 case AArch64::LDRBroX:
1229 case AArch64::LDRDroX:
1230 case AArch64::LDRHHroX:
1231 case AArch64::LDRHroX:
1232 case AArch64::LDRQroX:
1233 case AArch64::LDRSBWroX:
1234 case AArch64::LDRSBXroX:
1235 case AArch64::LDRSHWroX:
1236 case AArch64::LDRSHXroX:
1237 case AArch64::LDRSWroX:
1238 case AArch64::LDRSroX:
1239 case AArch64::LDRWroX:
1240 case AArch64::LDRXroX:
1241 case AArch64::STRBBroX:
1242 case AArch64::STRBroX:
1243 case AArch64::STRDroX:
1244 case AArch64::STRHHroX:
1245 case AArch64::STRHroX:
1246 case AArch64::STRQroX:
1247 case AArch64::STRSroX:
1248 case AArch64::STRWroX:
1249 case AArch64::STRXroX:
1251 unsigned Val = MI->getOperand(3).getImm();
1252 AArch64_AM::ShiftExtendType ExtType = AArch64_AM::getMemExtendType(Val);
1253 return (ExtType != AArch64_AM::UXTX) || AArch64_AM::getMemDoShift(Val);
1254 }
1255 return false;
1256 }
1258 /// Check all MachineMemOperands for a hint to suppress pairing.
1259 bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr *MI) const {
1260 assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
1261 "Too many target MO flags");
1262 for (auto *MM : MI->memoperands()) {
1263 if (MM->getFlags() &
1264 (MOSuppressPair << MachineMemOperand::MOTargetStartBit)) {
1265 return true;
1266 }
1267 }
1268 return false;
1269 }
1271 /// Set a flag on the first MachineMemOperand to suppress pairing.
1272 void AArch64InstrInfo::suppressLdStPair(MachineInstr *MI) const {
1273 if (MI->memoperands_empty())
1274 return;
1276 assert(MOSuppressPair < (1 << MachineMemOperand::MOTargetNumBits) &&
1277 "Too many target MO flags");
1278 (*MI->memoperands_begin())
1279 ->setFlags(MOSuppressPair << MachineMemOperand::MOTargetStartBit);
1280 }
1282 bool
1283 AArch64InstrInfo::getLdStBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg,
1284 unsigned &Offset,
1285 const TargetRegisterInfo *TRI) const {
1286 switch (LdSt->getOpcode()) {
1287 default:
1288 return false;
1289 case AArch64::STRSui:
1290 case AArch64::STRDui:
1291 case AArch64::STRQui:
1292 case AArch64::STRXui:
1293 case AArch64::STRWui:
1294 case AArch64::LDRSui:
1295 case AArch64::LDRDui:
1296 case AArch64::LDRQui:
1297 case AArch64::LDRXui:
1298 case AArch64::LDRWui:
1299 if (!LdSt->getOperand(1).isReg() || !LdSt->getOperand(2).isImm())
1300 return false;
1301 BaseReg = LdSt->getOperand(1).getReg();
1302 MachineFunction &MF = *LdSt->getParent()->getParent();
1303 unsigned Width = getRegClass(LdSt->getDesc(), 0, TRI, MF)->getSize();
1304 Offset = LdSt->getOperand(2).getImm() * Width;
1305 return true;
1306 };
1307 }
1309 bool AArch64InstrInfo::getLdStBaseRegImmOfsWidth(
1310 MachineInstr *LdSt, unsigned &BaseReg, int &Offset, int &Width,
1311 const TargetRegisterInfo *TRI) const {
1312 // Handle only loads/stores with base register followed by immediate offset.
1313 if (LdSt->getNumOperands() != 3)
1314 return false;
1315 if (!LdSt->getOperand(1).isReg() || !LdSt->getOperand(2).isImm())
1316 return false;
1318 // Offset is calculated as the immediate operand multiplied by the scaling factor.
1319 // Unscaled instructions have scaling factor set to 1.
1320 int Scale = 0;
1321 switch (LdSt->getOpcode()) {
1322 default:
1323 return false;
1324 case AArch64::LDURQi:
1325 case AArch64::STURQi:
1326 Width = 16;
1327 Scale = 1;
1328 break;
1329 case AArch64::LDURXi:
1330 case AArch64::LDURDi:
1331 case AArch64::STURXi:
1332 case AArch64::STURDi:
1333 Width = 8;
1334 Scale = 1;
1335 break;
1336 case AArch64::LDURWi:
1337 case AArch64::LDURSi:
1338 case AArch64::LDURSWi:
1339 case AArch64::STURWi:
1340 case AArch64::STURSi:
1341 Width = 4;
1342 Scale = 1;
1343 break;
1344 case AArch64::LDURHi:
1345 case AArch64::LDURHHi:
1346 case AArch64::LDURSHXi:
1347 case AArch64::LDURSHWi:
1348 case AArch64::STURHi:
1349 case AArch64::STURHHi:
1350 Width = 2;
1351 Scale = 1;
1352 break;
1353 case AArch64::LDURBi:
1354 case AArch64::LDURBBi:
1355 case AArch64::LDURSBXi:
1356 case AArch64::LDURSBWi:
1357 case AArch64::STURBi:
1358 case AArch64::STURBBi:
1359 Width = 1;
1360 Scale = 1;
1361 break;
1362 case AArch64::LDRXui:
1363 case AArch64::STRXui:
1364 Scale = Width = 8;
1365 break;
1366 case AArch64::LDRWui:
1367 case AArch64::STRWui:
1368 Scale = Width = 4;
1369 break;
1370 case AArch64::LDRBui:
1371 case AArch64::STRBui:
1372 Scale = Width = 1;
1373 break;
1374 case AArch64::LDRHui:
1375 case AArch64::STRHui:
1376 Scale = Width = 2;
1377 break;
1378 case AArch64::LDRSui:
1379 case AArch64::STRSui:
1380 Scale = Width = 4;
1381 break;
1382 case AArch64::LDRDui:
1383 case AArch64::STRDui:
1384 Scale = Width = 8;
1385 break;
1386 case AArch64::LDRQui:
1387 case AArch64::STRQui:
1388 Scale = Width = 16;
1389 break;
1390 case AArch64::LDRBBui:
1391 case AArch64::STRBBui:
1392 Scale = Width = 1;
1393 break;
1394 case AArch64::LDRHHui:
1395 case AArch64::STRHHui:
1396 Scale = Width = 2;
1397 break;
1398 };
1400 BaseReg = LdSt->getOperand(1).getReg();
1401 Offset = LdSt->getOperand(2).getImm() * Scale;
1402 return true;
1403 }
1405 /// Detect opportunities for ldp/stp formation.
1406 ///
1407 /// Only called for LdSt for which getLdStBaseRegImmOfs returns true.
1408 bool AArch64InstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt,
1409 MachineInstr *SecondLdSt,
1410 unsigned NumLoads) const {
1411 // Only cluster up to a single pair.
1412 if (NumLoads > 1)
1413 return false;
1414 if (FirstLdSt->getOpcode() != SecondLdSt->getOpcode())
1415 return false;
1416 // getLdStBaseRegImmOfs guarantees that oper 2 isImm.
1417 unsigned Ofs1 = FirstLdSt->getOperand(2).getImm();
1418 // Allow 6 bits of positive range.
1419 if (Ofs1 > 64)
1420 return false;
1421 // The caller should already have ordered First/SecondLdSt by offset.
1422 unsigned Ofs2 = SecondLdSt->getOperand(2).getImm();
1423 return Ofs1 + 1 == Ofs2;
1424 }
1426 bool AArch64InstrInfo::shouldScheduleAdjacent(MachineInstr *First,
1427 MachineInstr *Second) const {
1428 // Cyclone can fuse CMN, CMP followed by Bcc.
1430 // FIXME: B0 can also fuse:
1431 // AND, BIC, ORN, ORR, or EOR (optional S) followed by Bcc or CBZ or CBNZ.
1432 if (Second->getOpcode() != AArch64::Bcc)
1433 return false;
1434 switch (First->getOpcode()) {
1435 default:
1436 return false;
1437 case AArch64::SUBSWri:
1438 case AArch64::ADDSWri:
1439 case AArch64::ANDSWri:
1440 case AArch64::SUBSXri:
1441 case AArch64::ADDSXri:
1442 case AArch64::ANDSXri:
1443 return true;
1444 }
1445 }
1447 MachineInstr *AArch64InstrInfo::emitFrameIndexDebugValue(
1448 MachineFunction &MF, int FrameIx, uint64_t Offset, const MDNode *Var,
1449 const MDNode *Expr, DebugLoc DL) const {
1450 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AArch64::DBG_VALUE))
1451 .addFrameIndex(FrameIx)
1452 .addImm(0)
1453 .addImm(Offset)
1454 .addMetadata(Var)
1455 .addMetadata(Expr);
1456 return &*MIB;
1457 }
1459 static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
1460 unsigned Reg, unsigned SubIdx,
1461 unsigned State,
1462 const TargetRegisterInfo *TRI) {
1463 if (!SubIdx)
1464 return MIB.addReg(Reg, State);
1466 if (TargetRegisterInfo::isPhysicalRegister(Reg))
1467 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
1468 return MIB.addReg(Reg, State, SubIdx);
1469 }
1471 static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
1472 unsigned NumRegs) {
1473 // We really want the positive remainder mod 32 here, that happens to be
1474 // easily obtainable with a mask.
1475 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
1476 }
1478 void AArch64InstrInfo::copyPhysRegTuple(
1479 MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL,
1480 unsigned DestReg, unsigned SrcReg, bool KillSrc, unsigned Opcode,
1481 llvm::ArrayRef<unsigned> Indices) const {
1482 assert(Subtarget.hasNEON() &&
1483 "Unexpected register copy without NEON");
1484 const TargetRegisterInfo *TRI = &getRegisterInfo();
1485 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
1486 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
1487 unsigned NumRegs = Indices.size();
1489 int SubReg = 0, End = NumRegs, Incr = 1;
1490 if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
1491 SubReg = NumRegs - 1;
1492 End = -1;
1493 Incr = -1;
1494 }
1496 for (; SubReg != End; SubReg += Incr) {
1497 const MachineInstrBuilder &MIB = BuildMI(MBB, I, DL, get(Opcode));
1498 AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
1499 AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
1500 AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
1501 }
1502 }
1504 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
1505 MachineBasicBlock::iterator I, DebugLoc DL,
1506 unsigned DestReg, unsigned SrcReg,
1507 bool KillSrc) const {
1508 if (AArch64::GPR32spRegClass.contains(DestReg) &&
1509 (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
1510 const TargetRegisterInfo *TRI = &getRegisterInfo();
1512 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
1513 // If either operand is WSP, expand to ADD #0.
1514 if (Subtarget.hasZeroCycleRegMove()) {
1515 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
1516 unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
1517 &AArch64::GPR64spRegClass);
1518 unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
1519 &AArch64::GPR64spRegClass);
1520 // This instruction is reading and writing X registers. This may upset
1521 // the register scavenger and machine verifier, so we need to indicate
1522 // that we are reading an undefined value from SrcRegX, but a proper
1523 // value from SrcReg.
1524 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
1525 .addReg(SrcRegX, RegState::Undef)
1526 .addImm(0)
1527 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1528 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
1529 } else {
1530 BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
1531 .addReg(SrcReg, getKillRegState(KillSrc))
1532 .addImm(0)
1533 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1534 }
1535 } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroing()) {
1536 BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg).addImm(0).addImm(
1537 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1538 } else {
1539 if (Subtarget.hasZeroCycleRegMove()) {
1540 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
1541 unsigned DestRegX = TRI->getMatchingSuperReg(DestReg, AArch64::sub_32,
1542 &AArch64::GPR64spRegClass);
1543 unsigned SrcRegX = TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
1544 &AArch64::GPR64spRegClass);
1545 // This instruction is reading and writing X registers. This may upset
1546 // the register scavenger and machine verifier, so we need to indicate
1547 // that we are reading an undefined value from SrcRegX, but a proper
1548 // value from SrcReg.
1549 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
1550 .addReg(AArch64::XZR)
1551 .addReg(SrcRegX, RegState::Undef)
1552 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
1553 } else {
1554 // Otherwise, expand to ORR WZR.
1555 BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
1556 .addReg(AArch64::WZR)
1557 .addReg(SrcReg, getKillRegState(KillSrc));
1558 }
1559 }
1560 return;
1561 }
1563 if (AArch64::GPR64spRegClass.contains(DestReg) &&
1564 (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
1565 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
1566 // If either operand is SP, expand to ADD #0.
1567 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
1568 .addReg(SrcReg, getKillRegState(KillSrc))
1569 .addImm(0)
1570 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1571 } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroing()) {
1572 BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg).addImm(0).addImm(
1573 AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
1574 } else {
1575 // Otherwise, expand to ORR XZR.
1576 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
1577 .addReg(AArch64::XZR)
1578 .addReg(SrcReg, getKillRegState(KillSrc));
1579 }
1580 return;
1581 }
1583 // Copy a DDDD register quad by copying the individual sub-registers.
1584 if (AArch64::DDDDRegClass.contains(DestReg) &&
1585 AArch64::DDDDRegClass.contains(SrcReg)) {
1586 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1,
1587 AArch64::dsub2, AArch64::dsub3 };
1588 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1589 Indices);
1590 return;
1591 }
1593 // Copy a DDD register triple by copying the individual sub-registers.
1594 if (AArch64::DDDRegClass.contains(DestReg) &&
1595 AArch64::DDDRegClass.contains(SrcReg)) {
1596 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1,
1597 AArch64::dsub2 };
1598 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1599 Indices);
1600 return;
1601 }
1603 // Copy a DD register pair by copying the individual sub-registers.
1604 if (AArch64::DDRegClass.contains(DestReg) &&
1605 AArch64::DDRegClass.contains(SrcReg)) {
1606 static const unsigned Indices[] = { AArch64::dsub0, AArch64::dsub1 };
1607 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
1608 Indices);
1609 return;
1610 }
1612 // Copy a QQQQ register quad by copying the individual sub-registers.
1613 if (AArch64::QQQQRegClass.contains(DestReg) &&
1614 AArch64::QQQQRegClass.contains(SrcReg)) {
1615 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1,
1616 AArch64::qsub2, AArch64::qsub3 };
1617 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1618 Indices);
1619 return;
1620 }
1622 // Copy a QQQ register triple by copying the individual sub-registers.
1623 if (AArch64::QQQRegClass.contains(DestReg) &&
1624 AArch64::QQQRegClass.contains(SrcReg)) {
1625 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1,
1626 AArch64::qsub2 };
1627 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1628 Indices);
1629 return;
1630 }
1632 // Copy a QQ register pair by copying the individual sub-registers.
1633 if (AArch64::QQRegClass.contains(DestReg) &&
1634 AArch64::QQRegClass.contains(SrcReg)) {
1635 static const unsigned Indices[] = { AArch64::qsub0, AArch64::qsub1 };
1636 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
1637 Indices);
1638 return;
1639 }
1641 if (AArch64::FPR128RegClass.contains(DestReg) &&
1642 AArch64::FPR128RegClass.contains(SrcReg)) {
1643 if(Subtarget.hasNEON()) {
1644 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1645 .addReg(SrcReg)
1646 .addReg(SrcReg, getKillRegState(KillSrc));
1647 } else {
1648 BuildMI(MBB, I, DL, get(AArch64::STRQpre))
1649 .addReg(AArch64::SP, RegState::Define)
1650 .addReg(SrcReg, getKillRegState(KillSrc))
1651 .addReg(AArch64::SP)
1652 .addImm(-16);
1653 BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
1654 .addReg(AArch64::SP, RegState::Define)
1655 .addReg(DestReg, RegState::Define)
1656 .addReg(AArch64::SP)
1657 .addImm(16);
1658 }
1659 return;
1660 }
1662 if (AArch64::FPR64RegClass.contains(DestReg) &&
1663 AArch64::FPR64RegClass.contains(SrcReg)) {
1664 if(Subtarget.hasNEON()) {
1665 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
1666 &AArch64::FPR128RegClass);
1667 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
1668 &AArch64::FPR128RegClass);
1669 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1670 .addReg(SrcReg)
1671 .addReg(SrcReg, getKillRegState(KillSrc));
1672 } else {
1673 BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
1674 .addReg(SrcReg, getKillRegState(KillSrc));
1675 }
1676 return;
1677 }
1679 if (AArch64::FPR32RegClass.contains(DestReg) &&
1680 AArch64::FPR32RegClass.contains(SrcReg)) {
1681 if(Subtarget.hasNEON()) {
1682 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
1683 &AArch64::FPR128RegClass);
1684 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
1685 &AArch64::FPR128RegClass);
1686 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1687 .addReg(SrcReg)
1688 .addReg(SrcReg, getKillRegState(KillSrc));
1689 } else {
1690 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1691 .addReg(SrcReg, getKillRegState(KillSrc));
1692 }
1693 return;
1694 }
1696 if (AArch64::FPR16RegClass.contains(DestReg) &&
1697 AArch64::FPR16RegClass.contains(SrcReg)) {
1698 if(Subtarget.hasNEON()) {
1699 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
1700 &AArch64::FPR128RegClass);
1701 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
1702 &AArch64::FPR128RegClass);
1703 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1704 .addReg(SrcReg)
1705 .addReg(SrcReg, getKillRegState(KillSrc));
1706 } else {
1707 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
1708 &AArch64::FPR32RegClass);
1709 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
1710 &AArch64::FPR32RegClass);
1711 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1712 .addReg(SrcReg, getKillRegState(KillSrc));
1713 }
1714 return;
1715 }
1717 if (AArch64::FPR8RegClass.contains(DestReg) &&
1718 AArch64::FPR8RegClass.contains(SrcReg)) {
1719 if(Subtarget.hasNEON()) {
1720 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
1721 &AArch64::FPR128RegClass);
1722 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
1723 &AArch64::FPR128RegClass);
1724 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
1725 .addReg(SrcReg)
1726 .addReg(SrcReg, getKillRegState(KillSrc));
1727 } else {
1728 DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
1729 &AArch64::FPR32RegClass);
1730 SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
1731 &AArch64::FPR32RegClass);
1732 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
1733 .addReg(SrcReg, getKillRegState(KillSrc));
1734 }
1735 return;
1736 }
1738 // Copies between GPR64 and FPR64.
1739 if (AArch64::FPR64RegClass.contains(DestReg) &&
1740 AArch64::GPR64RegClass.contains(SrcReg)) {
1741 BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
1742 .addReg(SrcReg, getKillRegState(KillSrc));
1743 return;
1744 }
1745 if (AArch64::GPR64RegClass.contains(DestReg) &&
1746 AArch64::FPR64RegClass.contains(SrcReg)) {
1747 BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
1748 .addReg(SrcReg, getKillRegState(KillSrc));
1749 return;
1750 }
1751 // Copies between GPR32 and FPR32.
1752 if (AArch64::FPR32RegClass.contains(DestReg) &&
1753 AArch64::GPR32RegClass.contains(SrcReg)) {
1754 BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
1755 .addReg(SrcReg, getKillRegState(KillSrc));
1756 return;
1757 }
1758 if (AArch64::GPR32RegClass.contains(DestReg) &&
1759 AArch64::FPR32RegClass.contains(SrcReg)) {
1760 BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
1761 .addReg(SrcReg, getKillRegState(KillSrc));
1762 return;
1763 }
1765 if (DestReg == AArch64::NZCV) {
1766 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
1767 BuildMI(MBB, I, DL, get(AArch64::MSR))
1768 .addImm(AArch64SysReg::NZCV)
1769 .addReg(SrcReg, getKillRegState(KillSrc))
1770 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
1771 return;
1772 }
1774 if (SrcReg == AArch64::NZCV) {
1775 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
1776 BuildMI(MBB, I, DL, get(AArch64::MRS))
1777 .addReg(DestReg)
1778 .addImm(AArch64SysReg::NZCV)
1779 .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
1780 return;
1781 }
1783 llvm_unreachable("unimplemented reg-to-reg copy");
1784 }
1786 void AArch64InstrInfo::storeRegToStackSlot(
1787 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned SrcReg,
1788 bool isKill, int FI, const TargetRegisterClass *RC,
1789 const TargetRegisterInfo *TRI) const {
1790 DebugLoc DL;
1791 if (MBBI != MBB.end())
1792 DL = MBBI->getDebugLoc();
1793 MachineFunction &MF = *MBB.getParent();
1794 MachineFrameInfo &MFI = *MF.getFrameInfo();
1795 unsigned Align = MFI.getObjectAlignment(FI);
1797 MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
1798 MachineMemOperand *MMO = MF.getMachineMemOperand(
1799 PtrInfo, MachineMemOperand::MOStore, MFI.getObjectSize(FI), Align);
1800 unsigned Opc = 0;
1801 bool Offset = true;
1802 switch (RC->getSize()) {
1803 case 1:
1804 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
1805 Opc = AArch64::STRBui;
1806 break;
1807 case 2:
1808 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
1809 Opc = AArch64::STRHui;
1810 break;
1811 case 4:
1812 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1813 Opc = AArch64::STRWui;
1814 if (TargetRegisterInfo::isVirtualRegister(SrcReg))
1815 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
1816 else
1817 assert(SrcReg != AArch64::WSP);
1818 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
1819 Opc = AArch64::STRSui;
1820 break;
1821 case 8:
1822 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
1823 Opc = AArch64::STRXui;
1824 if (TargetRegisterInfo::isVirtualRegister(SrcReg))
1825 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
1826 else
1827 assert(SrcReg != AArch64::SP);
1828 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
1829 Opc = AArch64::STRDui;
1830 break;
1831 case 16:
1832 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
1833 Opc = AArch64::STRQui;
1834 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
1835 assert(Subtarget.hasNEON() &&
1836 "Unexpected register store without NEON");
1837 Opc = AArch64::ST1Twov1d, Offset = false;
1838 }
1839 break;
1840 case 24:
1841 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
1842 assert(Subtarget.hasNEON() &&
1843 "Unexpected register store without NEON");
1844 Opc = AArch64::ST1Threev1d, Offset = false;
1845 }
1846 break;
1847 case 32:
1848 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
1849 assert(Subtarget.hasNEON() &&
1850 "Unexpected register store without NEON");
1851 Opc = AArch64::ST1Fourv1d, Offset = false;
1852 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
1853 assert(Subtarget.hasNEON() &&
1854 "Unexpected register store without NEON");
1855 Opc = AArch64::ST1Twov2d, Offset = false;
1856 }
1857 break;
1858 case 48:
1859 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
1860 assert(Subtarget.hasNEON() &&
1861 "Unexpected register store without NEON");
1862 Opc = AArch64::ST1Threev2d, Offset = false;
1863 }
1864 break;
1865 case 64:
1866 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
1867 assert(Subtarget.hasNEON() &&
1868 "Unexpected register store without NEON");
1869 Opc = AArch64::ST1Fourv2d, Offset = false;
1870 }
1871 break;
1872 }
1873 assert(Opc && "Unknown register class");
1875 const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc))
1876 .addReg(SrcReg, getKillRegState(isKill))
1877 .addFrameIndex(FI);
1879 if (Offset)
1880 MI.addImm(0);
1881 MI.addMemOperand(MMO);
1882 }
1884 void AArch64InstrInfo::loadRegFromStackSlot(
1885 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned DestReg,
1886 int FI, const TargetRegisterClass *RC,
1887 const TargetRegisterInfo *TRI) const {
1888 DebugLoc DL;
1889 if (MBBI != MBB.end())
1890 DL = MBBI->getDebugLoc();
1891 MachineFunction &MF = *MBB.getParent();
1892 MachineFrameInfo &MFI = *MF.getFrameInfo();
1893 unsigned Align = MFI.getObjectAlignment(FI);
1894 MachinePointerInfo PtrInfo(PseudoSourceValue::getFixedStack(FI));
1895 MachineMemOperand *MMO = MF.getMachineMemOperand(
1896 PtrInfo, MachineMemOperand::MOLoad, MFI.getObjectSize(FI), Align);
1898 unsigned Opc = 0;
1899 bool Offset = true;
1900 switch (RC->getSize()) {
1901 case 1:
1902 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
1903 Opc = AArch64::LDRBui;
1904 break;
1905 case 2:
1906 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
1907 Opc = AArch64::LDRHui;
1908 break;
1909 case 4:
1910 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1911 Opc = AArch64::LDRWui;
1912 if (TargetRegisterInfo::isVirtualRegister(DestReg))
1913 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
1914 else
1915 assert(DestReg != AArch64::WSP);
1916 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
1917 Opc = AArch64::LDRSui;
1918 break;
1919 case 8:
1920 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
1921 Opc = AArch64::LDRXui;
1922 if (TargetRegisterInfo::isVirtualRegister(DestReg))
1923 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
1924 else
1925 assert(DestReg != AArch64::SP);
1926 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
1927 Opc = AArch64::LDRDui;
1928 break;
1929 case 16:
1930 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
1931 Opc = AArch64::LDRQui;
1932 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
1933 assert(Subtarget.hasNEON() &&
1934 "Unexpected register load without NEON");
1935 Opc = AArch64::LD1Twov1d, Offset = false;
1936 }
1937 break;
1938 case 24:
1939 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
1940 assert(Subtarget.hasNEON() &&
1941 "Unexpected register load without NEON");
1942 Opc = AArch64::LD1Threev1d, Offset = false;
1943 }
1944 break;
1945 case 32:
1946 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
1947 assert(Subtarget.hasNEON() &&
1948 "Unexpected register load without NEON");
1949 Opc = AArch64::LD1Fourv1d, Offset = false;
1950 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
1951 assert(Subtarget.hasNEON() &&
1952 "Unexpected register load without NEON");
1953 Opc = AArch64::LD1Twov2d, Offset = false;
1954 }
1955 break;
1956 case 48:
1957 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
1958 assert(Subtarget.hasNEON() &&
1959 "Unexpected register load without NEON");
1960 Opc = AArch64::LD1Threev2d, Offset = false;
1961 }
1962 break;
1963 case 64:
1964 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
1965 assert(Subtarget.hasNEON() &&
1966 "Unexpected register load without NEON");
1967 Opc = AArch64::LD1Fourv2d, Offset = false;
1968 }
1969 break;
1970 }
1971 assert(Opc && "Unknown register class");
1973 const MachineInstrBuilder &MI = BuildMI(MBB, MBBI, DL, get(Opc))
1974 .addReg(DestReg, getDefRegState(true))
1975 .addFrameIndex(FI);
1976 if (Offset)
1977 MI.addImm(0);
1978 MI.addMemOperand(MMO);
1979 }
1981 void llvm::emitFrameOffset(MachineBasicBlock &MBB,
1982 MachineBasicBlock::iterator MBBI, DebugLoc DL,
1983 unsigned DestReg, unsigned SrcReg, int Offset,
1984 const TargetInstrInfo *TII,
1985 MachineInstr::MIFlag Flag, bool SetNZCV) {
1986 if (DestReg == SrcReg && Offset == 0)
1987 return;
1989 bool isSub = Offset < 0;
1990 if (isSub)
1991 Offset = -Offset;
1993 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
1994 // scratch register. If DestReg is a virtual register, use it as the
1995 // scratch register; otherwise, create a new virtual register (to be
1996 // replaced by the scavenger at the end of PEI). That case can be optimized
1997 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
1998 // register can be loaded with offset%8 and the add/sub can use an extending
1999 // instruction with LSL#3.
2000 // Currently the function handles any offsets but generates a poor sequence
2001 // of code.
2002 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
2004 unsigned Opc;
2005 if (SetNZCV)
2006 Opc = isSub ? AArch64::SUBSXri : AArch64::ADDSXri;
2007 else
2008 Opc = isSub ? AArch64::SUBXri : AArch64::ADDXri;
2009 const unsigned MaxEncoding = 0xfff;
2010 const unsigned ShiftSize = 12;
2011 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
2012 while (((unsigned)Offset) >= (1 << ShiftSize)) {
2013 unsigned ThisVal;
2014 if (((unsigned)Offset) > MaxEncodableValue) {
2015 ThisVal = MaxEncodableValue;
2016 } else {
2017 ThisVal = Offset & MaxEncodableValue;
2018 }
2019 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
2020 "Encoding cannot handle value that big");
2021 BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
2022 .addReg(SrcReg)
2023 .addImm(ThisVal >> ShiftSize)
2024 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftSize))
2025 .setMIFlag(Flag);
2027 SrcReg = DestReg;
2028 Offset -= ThisVal;
2029 if (Offset == 0)
2030 return;
2031 }
2032 BuildMI(MBB, MBBI, DL, TII->get(Opc), DestReg)
2033 .addReg(SrcReg)
2034 .addImm(Offset)
2035 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
2036 .setMIFlag(Flag);
2037 }
2039 MachineInstr *
2040 AArch64InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr *MI,
2041 const SmallVectorImpl<unsigned> &Ops,
2042 int FrameIndex) const {
2043 // This is a bit of a hack. Consider this instruction:
2044 //
2045 // %vreg0<def> = COPY %SP; GPR64all:%vreg0
2046 //
2047 // We explicitly chose GPR64all for the virtual register so such a copy might
2048 // be eliminated by RegisterCoalescer. However, that may not be possible, and
2049 // %vreg0 may even spill. We can't spill %SP, and since it is in the GPR64all
2050 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
2051 //
2052 // To prevent that, we are going to constrain the %vreg0 register class here.
2053 //
2054 // <rdar://problem/11522048>
2055 //
2056 if (MI->isCopy()) {
2057 unsigned DstReg = MI->getOperand(0).getReg();
2058 unsigned SrcReg = MI->getOperand(1).getReg();
2059 if (SrcReg == AArch64::SP &&
2060 TargetRegisterInfo::isVirtualRegister(DstReg)) {
2061 MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
2062 return nullptr;
2063 }
2064 if (DstReg == AArch64::SP &&
2065 TargetRegisterInfo::isVirtualRegister(SrcReg)) {
2066 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
2067 return nullptr;
2068 }
2069 }
2071 // Cannot fold.
2072 return nullptr;
2073 }
2075 int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI, int &Offset,
2076 bool *OutUseUnscaledOp,
2077 unsigned *OutUnscaledOp,
2078 int *EmittableOffset) {
2079 int Scale = 1;
2080 bool IsSigned = false;
2081 // The ImmIdx should be changed case by case if it is not 2.
2082 unsigned ImmIdx = 2;
2083 unsigned UnscaledOp = 0;
2084 // Set output values in case of early exit.
2085 if (EmittableOffset)
2086 *EmittableOffset = 0;
2087 if (OutUseUnscaledOp)
2088 *OutUseUnscaledOp = false;
2089 if (OutUnscaledOp)
2090 *OutUnscaledOp = 0;
2091 switch (MI.getOpcode()) {
2092 default:
2093 llvm_unreachable("unhandled opcode in rewriteAArch64FrameIndex");
2094 // Vector spills/fills can't take an immediate offset.
2095 case AArch64::LD1Twov2d:
2096 case AArch64::LD1Threev2d:
2097 case AArch64::LD1Fourv2d:
2098 case AArch64::LD1Twov1d:
2099 case AArch64::LD1Threev1d:
2100 case AArch64::LD1Fourv1d:
2101 case AArch64::ST1Twov2d:
2102 case AArch64::ST1Threev2d:
2103 case AArch64::ST1Fourv2d:
2104 case AArch64::ST1Twov1d:
2105 case AArch64::ST1Threev1d:
2106 case AArch64::ST1Fourv1d:
2107 return AArch64FrameOffsetCannotUpdate;
2108 case AArch64::PRFMui:
2109 Scale = 8;
2110 UnscaledOp = AArch64::PRFUMi;
2111 break;
2112 case AArch64::LDRXui:
2113 Scale = 8;
2114 UnscaledOp = AArch64::LDURXi;
2115 break;
2116 case AArch64::LDRWui:
2117 Scale = 4;
2118 UnscaledOp = AArch64::LDURWi;
2119 break;
2120 case AArch64::LDRBui:
2121 Scale = 1;
2122 UnscaledOp = AArch64::LDURBi;
2123 break;
2124 case AArch64::LDRHui:
2125 Scale = 2;
2126 UnscaledOp = AArch64::LDURHi;
2127 break;
2128 case AArch64::LDRSui:
2129 Scale = 4;
2130 UnscaledOp = AArch64::LDURSi;
2131 break;
2132 case AArch64::LDRDui:
2133 Scale = 8;
2134 UnscaledOp = AArch64::LDURDi;
2135 break;
2136 case AArch64::LDRQui:
2137 Scale = 16;
2138 UnscaledOp = AArch64::LDURQi;
2139 break;
2140 case AArch64::LDRBBui:
2141 Scale = 1;
2142 UnscaledOp = AArch64::LDURBBi;
2143 break;
2144 case AArch64::LDRHHui:
2145 Scale = 2;
2146 UnscaledOp = AArch64::LDURHHi;
2147 break;
2148 case AArch64::LDRSBXui:
2149 Scale = 1;
2150 UnscaledOp = AArch64::LDURSBXi;
2151 break;
2152 case AArch64::LDRSBWui:
2153 Scale = 1;
2154 UnscaledOp = AArch64::LDURSBWi;
2155 break;
2156 case AArch64::LDRSHXui:
2157 Scale = 2;
2158 UnscaledOp = AArch64::LDURSHXi;
2159 break;
2160 case AArch64::LDRSHWui:
2161 Scale = 2;
2162 UnscaledOp = AArch64::LDURSHWi;
2163 break;
2164 case AArch64::LDRSWui:
2165 Scale = 4;
2166 UnscaledOp = AArch64::LDURSWi;
2167 break;
2169 case AArch64::STRXui:
2170 Scale = 8;
2171 UnscaledOp = AArch64::STURXi;
2172 break;
2173 case AArch64::STRWui:
2174 Scale = 4;
2175 UnscaledOp = AArch64::STURWi;
2176 break;
2177 case AArch64::STRBui:
2178 Scale = 1;
2179 UnscaledOp = AArch64::STURBi;
2180 break;
2181 case AArch64::STRHui:
2182 Scale = 2;
2183 UnscaledOp = AArch64::STURHi;
2184 break;
2185 case AArch64::STRSui:
2186 Scale = 4;
2187 UnscaledOp = AArch64::STURSi;
2188 break;
2189 case AArch64::STRDui:
2190 Scale = 8;
2191 UnscaledOp = AArch64::STURDi;
2192 break;
2193 case AArch64::STRQui:
2194 Scale = 16;
2195 UnscaledOp = AArch64::STURQi;
2196 break;
2197 case AArch64::STRBBui:
2198 Scale = 1;
2199 UnscaledOp = AArch64::STURBBi;
2200 break;
2201 case AArch64::STRHHui:
2202 Scale = 2;
2203 UnscaledOp = AArch64::STURHHi;
2204 break;
2206 case AArch64::LDPXi:
2207 case AArch64::LDPDi:
2208 case AArch64::STPXi:
2209 case AArch64::STPDi:
2210 IsSigned = true;
2211 Scale = 8;
2212 break;
2213 case AArch64::LDPQi:
2214 case AArch64::STPQi:
2215 IsSigned = true;
2216 Scale = 16;
2217 break;
2218 case AArch64::LDPWi:
2219 case AArch64::LDPSi:
2220 case AArch64::STPWi:
2221 case AArch64::STPSi:
2222 IsSigned = true;
2223 Scale = 4;
2224 break;
2226 case AArch64::LDURXi:
2227 case AArch64::LDURWi:
2228 case AArch64::LDURBi:
2229 case AArch64::LDURHi:
2230 case AArch64::LDURSi:
2231 case AArch64::LDURDi:
2232 case AArch64::LDURQi:
2233 case AArch64::LDURHHi:
2234 case AArch64::LDURBBi:
2235 case AArch64::LDURSBXi:
2236 case AArch64::LDURSBWi:
2237 case AArch64::LDURSHXi:
2238 case AArch64::LDURSHWi:
2239 case AArch64::LDURSWi:
2240 case AArch64::STURXi:
2241 case AArch64::STURWi:
2242 case AArch64::STURBi:
2243 case AArch64::STURHi:
2244 case AArch64::STURSi:
2245 case AArch64::STURDi:
2246 case AArch64::STURQi:
2247 case AArch64::STURBBi:
2248 case AArch64::STURHHi:
2249 Scale = 1;
2250 break;
2251 }
2253 Offset += MI.getOperand(ImmIdx).getImm() * Scale;
2255 bool useUnscaledOp = false;
2256 // If the offset doesn't match the scale, we rewrite the instruction to
2257 // use the unscaled instruction instead. Likewise, if we have a negative
2258 // offset (and have an unscaled op to use).
2259 if ((Offset & (Scale - 1)) != 0 || (Offset < 0 && UnscaledOp != 0))
2260 useUnscaledOp = true;
2262 // Use an unscaled addressing mode if the instruction has a negative offset
2263 // (or if the instruction is already using an unscaled addressing mode).
2264 unsigned MaskBits;
2265 if (IsSigned) {
2266 // ldp/stp instructions.
2267 MaskBits = 7;
2268 Offset /= Scale;
2269 } else if (UnscaledOp == 0 || useUnscaledOp) {
2270 MaskBits = 9;
2271 IsSigned = true;
2272 Scale = 1;
2273 } else {
2274 MaskBits = 12;
2275 IsSigned = false;
2276 Offset /= Scale;
2277 }
2279 // Attempt to fold address computation.
2280 int MaxOff = (1 << (MaskBits - IsSigned)) - 1;
2281 int MinOff = (IsSigned ? (-MaxOff - 1) : 0);
2282 if (Offset >= MinOff && Offset <= MaxOff) {
2283 if (EmittableOffset)
2284 *EmittableOffset = Offset;
2285 Offset = 0;
2286 } else {
2287 int NewOff = Offset < 0 ? MinOff : MaxOff;
2288 if (EmittableOffset)
2289 *EmittableOffset = NewOff;
2290 Offset = (Offset - NewOff) * Scale;
2291 }
2292 if (OutUseUnscaledOp)
2293 *OutUseUnscaledOp = useUnscaledOp;
2294 if (OutUnscaledOp)
2295 *OutUnscaledOp = UnscaledOp;
2296 return AArch64FrameOffsetCanUpdate |
2297 (Offset == 0 ? AArch64FrameOffsetIsLegal : 0);
2298 }
2300 bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
2301 unsigned FrameReg, int &Offset,
2302 const AArch64InstrInfo *TII) {
2303 unsigned Opcode = MI.getOpcode();
2304 unsigned ImmIdx = FrameRegIdx + 1;
2306 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
2307 Offset += MI.getOperand(ImmIdx).getImm();
2308 emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
2309 MI.getOperand(0).getReg(), FrameReg, Offset, TII,
2310 MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
2311 MI.eraseFromParent();
2312 Offset = 0;
2313 return true;
2314 }
2316 int NewOffset;
2317 unsigned UnscaledOp;
2318 bool UseUnscaledOp;
2319 int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
2320 &UnscaledOp, &NewOffset);
2321 if (Status & AArch64FrameOffsetCanUpdate) {
2322 if (Status & AArch64FrameOffsetIsLegal)
2323 // Replace the FrameIndex with FrameReg.
2324 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
2325 if (UseUnscaledOp)
2326 MI.setDesc(TII->get(UnscaledOp));
2328 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
2329 return Offset == 0;
2330 }
2332 return false;
2333 }
2335 void AArch64InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
2336 NopInst.setOpcode(AArch64::HINT);
2337 NopInst.addOperand(MCOperand::CreateImm(0));
2338 }
2339 /// useMachineCombiner - return true when a target supports MachineCombiner
2340 bool AArch64InstrInfo::useMachineCombiner() const {
2341 // AArch64 supports the combiner
2342 return true;
2343 }
2344 //
2345 // True when Opc sets flag
2346 static bool isCombineInstrSettingFlag(unsigned Opc) {
2347 switch (Opc) {
2348 case AArch64::ADDSWrr:
2349 case AArch64::ADDSWri:
2350 case AArch64::ADDSXrr:
2351 case AArch64::ADDSXri:
2352 case AArch64::SUBSWrr:
2353 case AArch64::SUBSXrr:
2354 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2355 case AArch64::SUBSWri:
2356 case AArch64::SUBSXri:
2357 return true;
2358 default:
2359 break;
2360 }
2361 return false;
2362 }
2363 //
2364 // 32b Opcodes that can be combined with a MUL
2365 static bool isCombineInstrCandidate32(unsigned Opc) {
2366 switch (Opc) {
2367 case AArch64::ADDWrr:
2368 case AArch64::ADDWri:
2369 case AArch64::SUBWrr:
2370 case AArch64::ADDSWrr:
2371 case AArch64::ADDSWri:
2372 case AArch64::SUBSWrr:
2373 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2374 case AArch64::SUBWri:
2375 case AArch64::SUBSWri:
2376 return true;
2377 default:
2378 break;
2379 }
2380 return false;
2381 }
2382 //
2383 // 64b Opcodes that can be combined with a MUL
2384 static bool isCombineInstrCandidate64(unsigned Opc) {
2385 switch (Opc) {
2386 case AArch64::ADDXrr:
2387 case AArch64::ADDXri:
2388 case AArch64::SUBXrr:
2389 case AArch64::ADDSXrr:
2390 case AArch64::ADDSXri:
2391 case AArch64::SUBSXrr:
2392 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
2393 case AArch64::SUBXri:
2394 case AArch64::SUBSXri:
2395 return true;
2396 default:
2397 break;
2398 }
2399 return false;
2400 }
2401 //
2402 // Opcodes that can be combined with a MUL
2403 static bool isCombineInstrCandidate(unsigned Opc) {
2404 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
2405 }
2407 static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
2408 unsigned MulOpc, unsigned ZeroReg) {
2409 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2410 MachineInstr *MI = nullptr;
2411 // We need a virtual register definition.
2412 if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg()))
2413 MI = MRI.getUniqueVRegDef(MO.getReg());
2414 // And it needs to be in the trace (otherwise, it won't have a depth).
2415 if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != MulOpc)
2416 return false;
2418 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
2419 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
2420 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
2422 // The third input reg must be zero.
2423 if (MI->getOperand(3).getReg() != ZeroReg)
2424 return false;
2426 // Must only used by the user we combine with.
2427 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2428 return false;
2430 return true;
2431 }
2433 /// hasPattern - return true when there is potentially a faster code sequence
2434 /// for an instruction chain ending in \p Root. All potential patterns are
2435 /// listed
2436 /// in the \p Pattern vector. Pattern should be sorted in priority order since
2437 /// the pattern evaluator stops checking as soon as it finds a faster sequence.
2439 bool AArch64InstrInfo::hasPattern(
2440 MachineInstr &Root,
2441 SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const {
2442 unsigned Opc = Root.getOpcode();
2443 MachineBasicBlock &MBB = *Root.getParent();
2444 bool Found = false;
2446 if (!isCombineInstrCandidate(Opc))
2447 return 0;
2448 if (isCombineInstrSettingFlag(Opc)) {
2449 int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
2450 // When NZCV is live bail out.
2451 if (Cmp_NZCV == -1)
2452 return 0;
2453 unsigned NewOpc = convertFlagSettingOpcode(&Root);
2454 // When opcode can't change bail out.
2455 // CHECKME: do we miss any cases for opcode conversion?
2456 if (NewOpc == Opc)
2457 return 0;
2458 Opc = NewOpc;
2459 }
2461 switch (Opc) {
2462 default:
2463 break;
2464 case AArch64::ADDWrr:
2465 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
2466 "ADDWrr does not have register operands");
2467 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2468 AArch64::WZR)) {
2469 Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP1);
2470 Found = true;
2471 }
2472 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
2473 AArch64::WZR)) {
2474 Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP2);
2475 Found = true;
2476 }
2477 break;
2478 case AArch64::ADDXrr:
2479 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2480 AArch64::XZR)) {
2481 Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP1);
2482 Found = true;
2483 }
2484 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
2485 AArch64::XZR)) {
2486 Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP2);
2487 Found = true;
2488 }
2489 break;
2490 case AArch64::SUBWrr:
2491 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2492 AArch64::WZR)) {
2493 Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP1);
2494 Found = true;
2495 }
2496 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
2497 AArch64::WZR)) {
2498 Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP2);
2499 Found = true;
2500 }
2501 break;
2502 case AArch64::SUBXrr:
2503 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2504 AArch64::XZR)) {
2505 Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP1);
2506 Found = true;
2507 }
2508 if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
2509 AArch64::XZR)) {
2510 Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP2);
2511 Found = true;
2512 }
2513 break;
2514 case AArch64::ADDWri:
2515 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2516 AArch64::WZR)) {
2517 Pattern.push_back(MachineCombinerPattern::MC_MULADDWI_OP1);
2518 Found = true;
2519 }
2520 break;
2521 case AArch64::ADDXri:
2522 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2523 AArch64::XZR)) {
2524 Pattern.push_back(MachineCombinerPattern::MC_MULADDXI_OP1);
2525 Found = true;
2526 }
2527 break;
2528 case AArch64::SUBWri:
2529 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
2530 AArch64::WZR)) {
2531 Pattern.push_back(MachineCombinerPattern::MC_MULSUBWI_OP1);
2532 Found = true;
2533 }
2534 break;
2535 case AArch64::SUBXri:
2536 if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
2537 AArch64::XZR)) {
2538 Pattern.push_back(MachineCombinerPattern::MC_MULSUBXI_OP1);
2539 Found = true;
2540 }
2541 break;
2542 }
2543 return Found;
2544 }
2546 /// genMadd - Generate madd instruction and combine mul and add.
2547 /// Example:
2548 /// MUL I=A,B,0
2549 /// ADD R,I,C
2550 /// ==> MADD R,A,B,C
2551 /// \param Root is the ADD instruction
2552 /// \param [out] InsInstrs is a vector of machine instructions and will
2553 /// contain the generated madd instruction
2554 /// \param IdxMulOpd is index of operand in Root that is the result of
2555 /// the MUL. In the example above IdxMulOpd is 1.
2556 /// \param MaddOpc the opcode fo the madd instruction
2557 static MachineInstr *genMadd(MachineFunction &MF, MachineRegisterInfo &MRI,
2558 const TargetInstrInfo *TII, MachineInstr &Root,
2559 SmallVectorImpl<MachineInstr *> &InsInstrs,
2560 unsigned IdxMulOpd, unsigned MaddOpc,
2561 const TargetRegisterClass *RC) {
2562 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
2564 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
2565 MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
2566 unsigned ResultReg = Root.getOperand(0).getReg();
2567 unsigned SrcReg0 = MUL->getOperand(1).getReg();
2568 bool Src0IsKill = MUL->getOperand(1).isKill();
2569 unsigned SrcReg1 = MUL->getOperand(2).getReg();
2570 bool Src1IsKill = MUL->getOperand(2).isKill();
2571 unsigned SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
2572 bool Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
2574 if (TargetRegisterInfo::isVirtualRegister(ResultReg))
2575 MRI.constrainRegClass(ResultReg, RC);
2576 if (TargetRegisterInfo::isVirtualRegister(SrcReg0))
2577 MRI.constrainRegClass(SrcReg0, RC);
2578 if (TargetRegisterInfo::isVirtualRegister(SrcReg1))
2579 MRI.constrainRegClass(SrcReg1, RC);
2580 if (TargetRegisterInfo::isVirtualRegister(SrcReg2))
2581 MRI.constrainRegClass(SrcReg2, RC);
2583 MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc),
2584 ResultReg)
2585 .addReg(SrcReg0, getKillRegState(Src0IsKill))
2586 .addReg(SrcReg1, getKillRegState(Src1IsKill))
2587 .addReg(SrcReg2, getKillRegState(Src2IsKill));
2588 // Insert the MADD
2589 InsInstrs.push_back(MIB);
2590 return MUL;
2591 }
2593 /// genMaddR - Generate madd instruction and combine mul and add using
2594 /// an extra virtual register
2595 /// Example - an ADD intermediate needs to be stored in a register:
2596 /// MUL I=A,B,0
2597 /// ADD R,I,Imm
2598 /// ==> ORR V, ZR, Imm
2599 /// ==> MADD R,A,B,V
2600 /// \param Root is the ADD instruction
2601 /// \param [out] InsInstrs is a vector of machine instructions and will
2602 /// contain the generated madd instruction
2603 /// \param IdxMulOpd is index of operand in Root that is the result of
2604 /// the MUL. In the example above IdxMulOpd is 1.
2605 /// \param MaddOpc the opcode fo the madd instruction
2606 /// \param VR is a virtual register that holds the value of an ADD operand
2607 /// (V in the example above).
2608 static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
2609 const TargetInstrInfo *TII, MachineInstr &Root,
2610 SmallVectorImpl<MachineInstr *> &InsInstrs,
2611 unsigned IdxMulOpd, unsigned MaddOpc,
2612 unsigned VR, const TargetRegisterClass *RC) {
2613 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
2615 MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
2616 unsigned ResultReg = Root.getOperand(0).getReg();
2617 unsigned SrcReg0 = MUL->getOperand(1).getReg();
2618 bool Src0IsKill = MUL->getOperand(1).isKill();
2619 unsigned SrcReg1 = MUL->getOperand(2).getReg();
2620 bool Src1IsKill = MUL->getOperand(2).isKill();
2622 if (TargetRegisterInfo::isVirtualRegister(ResultReg))
2623 MRI.constrainRegClass(ResultReg, RC);
2624 if (TargetRegisterInfo::isVirtualRegister(SrcReg0))
2625 MRI.constrainRegClass(SrcReg0, RC);
2626 if (TargetRegisterInfo::isVirtualRegister(SrcReg1))
2627 MRI.constrainRegClass(SrcReg1, RC);
2628 if (TargetRegisterInfo::isVirtualRegister(VR))
2629 MRI.constrainRegClass(VR, RC);
2631 MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc),
2632 ResultReg)
2633 .addReg(SrcReg0, getKillRegState(Src0IsKill))
2634 .addReg(SrcReg1, getKillRegState(Src1IsKill))
2635 .addReg(VR);
2636 // Insert the MADD
2637 InsInstrs.push_back(MIB);
2638 return MUL;
2639 }
2641 /// genAlternativeCodeSequence - when hasPattern() finds a pattern
2642 /// this function generates the instructions that could replace the
2643 /// original code sequence
2644 void AArch64InstrInfo::genAlternativeCodeSequence(
2645 MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
2646 SmallVectorImpl<MachineInstr *> &InsInstrs,
2647 SmallVectorImpl<MachineInstr *> &DelInstrs,
2648 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
2649 MachineBasicBlock &MBB = *Root.getParent();
2650 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2651 MachineFunction &MF = *MBB.getParent();
2652 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
2654 MachineInstr *MUL;
2655 const TargetRegisterClass *RC;
2656 unsigned Opc;
2657 switch (Pattern) {
2658 default:
2659 // signal error.
2660 break;
2661 case MachineCombinerPattern::MC_MULADDW_OP1:
2662 case MachineCombinerPattern::MC_MULADDX_OP1:
2663 // MUL I=A,B,0
2664 // ADD R,I,C
2665 // ==> MADD R,A,B,C
2666 // --- Create(MADD);
2667 if (Pattern == MachineCombinerPattern::MC_MULADDW_OP1) {
2668 Opc = AArch64::MADDWrrr;
2669 RC = &AArch64::GPR32RegClass;
2670 } else {
2671 Opc = AArch64::MADDXrrr;
2672 RC = &AArch64::GPR64RegClass;
2673 }
2674 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
2675 break;
2676 case MachineCombinerPattern::MC_MULADDW_OP2:
2677 case MachineCombinerPattern::MC_MULADDX_OP2:
2678 // MUL I=A,B,0
2679 // ADD R,C,I
2680 // ==> MADD R,A,B,C
2681 // --- Create(MADD);
2682 if (Pattern == MachineCombinerPattern::MC_MULADDW_OP2) {
2683 Opc = AArch64::MADDWrrr;
2684 RC = &AArch64::GPR32RegClass;
2685 } else {
2686 Opc = AArch64::MADDXrrr;
2687 RC = &AArch64::GPR64RegClass;
2688 }
2689 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
2690 break;
2691 case MachineCombinerPattern::MC_MULADDWI_OP1:
2692 case MachineCombinerPattern::MC_MULADDXI_OP1: {
2693 // MUL I=A,B,0
2694 // ADD R,I,Imm
2695 // ==> ORR V, ZR, Imm
2696 // ==> MADD R,A,B,V
2697 // --- Create(MADD);
2698 const TargetRegisterClass *OrrRC;
2699 unsigned BitSize, OrrOpc, ZeroReg;
2700 if (Pattern == MachineCombinerPattern::MC_MULADDWI_OP1) {
2701 OrrOpc = AArch64::ORRWri;
2702 OrrRC = &AArch64::GPR32spRegClass;
2703 BitSize = 32;
2704 ZeroReg = AArch64::WZR;
2705 Opc = AArch64::MADDWrrr;
2706 RC = &AArch64::GPR32RegClass;
2707 } else {
2708 OrrOpc = AArch64::ORRXri;
2709 OrrRC = &AArch64::GPR64spRegClass;
2710 BitSize = 64;
2711 ZeroReg = AArch64::XZR;
2712 Opc = AArch64::MADDXrrr;
2713 RC = &AArch64::GPR64RegClass;
2714 }
2715 unsigned NewVR = MRI.createVirtualRegister(OrrRC);
2716 uint64_t Imm = Root.getOperand(2).getImm();
2718 if (Root.getOperand(3).isImm()) {
2719 unsigned Val = Root.getOperand(3).getImm();
2720 Imm = Imm << Val;
2721 }
2722 uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
2723 uint64_t Encoding;
2724 if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
2725 MachineInstrBuilder MIB1 =
2726 BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
2727 .addReg(ZeroReg)
2728 .addImm(Encoding);
2729 InsInstrs.push_back(MIB1);
2730 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2731 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
2732 }
2733 break;
2734 }
2735 case MachineCombinerPattern::MC_MULSUBW_OP1:
2736 case MachineCombinerPattern::MC_MULSUBX_OP1: {
2737 // MUL I=A,B,0
2738 // SUB R,I, C
2739 // ==> SUB V, 0, C
2740 // ==> MADD R,A,B,V // = -C + A*B
2741 // --- Create(MADD);
2742 const TargetRegisterClass *SubRC;
2743 unsigned SubOpc, ZeroReg;
2744 if (Pattern == MachineCombinerPattern::MC_MULSUBW_OP1) {
2745 SubOpc = AArch64::SUBWrr;
2746 SubRC = &AArch64::GPR32spRegClass;
2747 ZeroReg = AArch64::WZR;
2748 Opc = AArch64::MADDWrrr;
2749 RC = &AArch64::GPR32RegClass;
2750 } else {
2751 SubOpc = AArch64::SUBXrr;
2752 SubRC = &AArch64::GPR64spRegClass;
2753 ZeroReg = AArch64::XZR;
2754 Opc = AArch64::MADDXrrr;
2755 RC = &AArch64::GPR64RegClass;
2756 }
2757 unsigned NewVR = MRI.createVirtualRegister(SubRC);
2758 // SUB NewVR, 0, C
2759 MachineInstrBuilder MIB1 =
2760 BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc), NewVR)
2761 .addReg(ZeroReg)
2762 .addOperand(Root.getOperand(2));
2763 InsInstrs.push_back(MIB1);
2764 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2765 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
2766 break;
2767 }
2768 case MachineCombinerPattern::MC_MULSUBW_OP2:
2769 case MachineCombinerPattern::MC_MULSUBX_OP2:
2770 // MUL I=A,B,0
2771 // SUB R,C,I
2772 // ==> MSUB R,A,B,C (computes C - A*B)
2773 // --- Create(MSUB);
2774 if (Pattern == MachineCombinerPattern::MC_MULSUBW_OP2) {
2775 Opc = AArch64::MSUBWrrr;
2776 RC = &AArch64::GPR32RegClass;
2777 } else {
2778 Opc = AArch64::MSUBXrrr;
2779 RC = &AArch64::GPR64RegClass;
2780 }
2781 MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
2782 break;
2783 case MachineCombinerPattern::MC_MULSUBWI_OP1:
2784 case MachineCombinerPattern::MC_MULSUBXI_OP1: {
2785 // MUL I=A,B,0
2786 // SUB R,I, Imm
2787 // ==> ORR V, ZR, -Imm
2788 // ==> MADD R,A,B,V // = -Imm + A*B
2789 // --- Create(MADD);
2790 const TargetRegisterClass *OrrRC;
2791 unsigned BitSize, OrrOpc, ZeroReg;
2792 if (Pattern == MachineCombinerPattern::MC_MULSUBWI_OP1) {
2793 OrrOpc = AArch64::ORRWri;
2794 OrrRC = &AArch64::GPR32spRegClass;
2795 BitSize = 32;
2796 ZeroReg = AArch64::WZR;
2797 Opc = AArch64::MADDWrrr;
2798 RC = &AArch64::GPR32RegClass;
2799 } else {
2800 OrrOpc = AArch64::ORRXri;
2801 OrrRC = &AArch64::GPR64RegClass;
2802 BitSize = 64;
2803 ZeroReg = AArch64::XZR;
2804 Opc = AArch64::MADDXrrr;
2805 RC = &AArch64::GPR64RegClass;
2806 }
2807 unsigned NewVR = MRI.createVirtualRegister(OrrRC);
2808 int Imm = Root.getOperand(2).getImm();
2809 if (Root.getOperand(3).isImm()) {
2810 unsigned Val = Root.getOperand(3).getImm();
2811 Imm = Imm << Val;
2812 }
2813 uint64_t UImm = -Imm << (64 - BitSize) >> (64 - BitSize);
2814 uint64_t Encoding;
2815 if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
2816 MachineInstrBuilder MIB1 =
2817 BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc), NewVR)
2818 .addReg(ZeroReg)
2819 .addImm(Encoding);
2820 InsInstrs.push_back(MIB1);
2821 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2822 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
2823 }
2824 break;
2825 }
2826 } // end switch (Pattern)
2827 // Record MUL and ADD/SUB for deletion
2828 DelInstrs.push_back(MUL);
2829 DelInstrs.push_back(&Root);
2831 return;
2832 }