1 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the SelectionDAG class.
11 //
12 //===----------------------------------------------------------------------===//
14 #include "llvm/CodeGen/SelectionDAG.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/Analysis/ValueTracking.h"
22 #include "llvm/CodeGen/MachineBasicBlock.h"
23 #include "llvm/CodeGen/MachineConstantPool.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineModuleInfo.h"
26 #include "llvm/IR/CallingConv.h"
27 #include "llvm/IR/Constants.h"
28 #include "llvm/IR/DataLayout.h"
29 #include "llvm/IR/DebugInfo.h"
30 #include "llvm/IR/DerivedTypes.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalAlias.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/Intrinsics.h"
35 #include "llvm/Support/CommandLine.h"
36 #include "llvm/Support/Debug.h"
37 #include "llvm/Support/ErrorHandling.h"
38 #include "llvm/Support/ManagedStatic.h"
39 #include "llvm/Support/MathExtras.h"
40 #include "llvm/Support/Mutex.h"
41 #include "llvm/Support/raw_ostream.h"
42 #include "llvm/Target/TargetInstrInfo.h"
43 #include "llvm/Target/TargetIntrinsicInfo.h"
44 #include "llvm/Target/TargetLowering.h"
45 #include "llvm/Target/TargetMachine.h"
46 #include "llvm/Target/TargetOptions.h"
47 #include "llvm/Target/TargetRegisterInfo.h"
48 #include "llvm/Target/TargetSelectionDAGInfo.h"
49 #include <algorithm>
50 #include <cmath>
51 using namespace llvm;
53 /// makeVTList - Return an instance of the SDVTList struct initialized with the
54 /// specified members.
55 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
56 SDVTList Res = {VTs, NumVTs};
57 return Res;
58 }
60 // Default null implementations of the callbacks.
61 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
62 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
64 //===----------------------------------------------------------------------===//
65 // ConstantFPSDNode Class
66 //===----------------------------------------------------------------------===//
68 /// isExactlyValue - We don't rely on operator== working on double values, as
69 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
70 /// As such, this method can be used to do an exact bit-for-bit comparison of
71 /// two floating point values.
72 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
73 return getValueAPF().bitwiseIsEqual(V);
74 }
76 bool ConstantFPSDNode::isValueValidForType(EVT VT,
77 const APFloat& Val) {
78 assert(VT.isFloatingPoint() && "Can only convert between FP types");
80 // convert modifies in place, so make a copy.
81 APFloat Val2 = APFloat(Val);
82 bool losesInfo;
83 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
84 APFloat::rmNearestTiesToEven,
85 &losesInfo);
86 return !losesInfo;
87 }
89 //===----------------------------------------------------------------------===//
90 // ISD Namespace
91 //===----------------------------------------------------------------------===//
93 /// isBuildVectorAllOnes - Return true if the specified node is a
94 /// BUILD_VECTOR where all of the elements are ~0 or undef.
95 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
96 // Look through a bit convert.
97 if (N->getOpcode() == ISD::BITCAST)
98 N = N->getOperand(0).getNode();
100 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
102 unsigned i = 0, e = N->getNumOperands();
104 // Skip over all of the undef values.
105 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
106 ++i;
108 // Do not accept an all-undef vector.
109 if (i == e) return false;
111 // Do not accept build_vectors that aren't all constants or which have non-~0
112 // elements. We have to be a bit careful here, as the type of the constant
113 // may not be the same as the type of the vector elements due to type
114 // legalization (the elements are promoted to a legal type for the target and
115 // a vector of a type may be legal when the base element type is not).
116 // We only want to check enough bits to cover the vector elements, because
117 // we care if the resultant vector is all ones, not whether the individual
118 // constants are.
119 SDValue NotZero = N->getOperand(i);
120 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
121 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
122 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
123 return false;
124 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
125 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
126 return false;
127 } else
128 return false;
130 // Okay, we have at least one ~0 value, check to see if the rest match or are
131 // undefs. Even with the above element type twiddling, this should be OK, as
132 // the same type legalization should have applied to all the elements.
133 for (++i; i != e; ++i)
134 if (N->getOperand(i) != NotZero &&
135 N->getOperand(i).getOpcode() != ISD::UNDEF)
136 return false;
137 return true;
138 }
141 /// isBuildVectorAllZeros - Return true if the specified node is a
142 /// BUILD_VECTOR where all of the elements are 0 or undef.
143 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
144 // Look through a bit convert.
145 if (N->getOpcode() == ISD::BITCAST)
146 N = N->getOperand(0).getNode();
148 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
150 unsigned i = 0, e = N->getNumOperands();
152 // Skip over all of the undef values.
153 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
154 ++i;
156 // Do not accept an all-undef vector.
157 if (i == e) return false;
159 // Do not accept build_vectors that aren't all constants or which have non-0
160 // elements.
161 SDValue Zero = N->getOperand(i);
162 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
163 if (!CN->isNullValue())
164 return false;
165 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
166 if (!CFPN->getValueAPF().isPosZero())
167 return false;
168 } else
169 return false;
171 // Okay, we have at least one 0 value, check to see if the rest match or are
172 // undefs.
173 for (++i; i != e; ++i)
174 if (N->getOperand(i) != Zero &&
175 N->getOperand(i).getOpcode() != ISD::UNDEF)
176 return false;
177 return true;
178 }
180 /// \brief Return true if the specified node is a BUILD_VECTOR node of
181 /// all ConstantSDNode or undef.
182 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
183 if (N->getOpcode() != ISD::BUILD_VECTOR)
184 return false;
186 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
187 SDValue Op = N->getOperand(i);
188 if (Op.getOpcode() == ISD::UNDEF)
189 continue;
190 if (!isa<ConstantSDNode>(Op))
191 return false;
192 }
193 return true;
194 }
196 /// isScalarToVector - Return true if the specified node is a
197 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
198 /// element is not an undef.
199 bool ISD::isScalarToVector(const SDNode *N) {
200 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
201 return true;
203 if (N->getOpcode() != ISD::BUILD_VECTOR)
204 return false;
205 if (N->getOperand(0).getOpcode() == ISD::UNDEF)
206 return false;
207 unsigned NumElems = N->getNumOperands();
208 if (NumElems == 1)
209 return false;
210 for (unsigned i = 1; i < NumElems; ++i) {
211 SDValue V = N->getOperand(i);
212 if (V.getOpcode() != ISD::UNDEF)
213 return false;
214 }
215 return true;
216 }
218 /// allOperandsUndef - Return true if the node has at least one operand
219 /// and all operands of the specified node are ISD::UNDEF.
220 bool ISD::allOperandsUndef(const SDNode *N) {
221 // Return false if the node has no operands.
222 // This is "logically inconsistent" with the definition of "all" but
223 // is probably the desired behavior.
224 if (N->getNumOperands() == 0)
225 return false;
227 for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
228 if (N->getOperand(i).getOpcode() != ISD::UNDEF)
229 return false;
231 return true;
232 }
234 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
235 switch (ExtType) {
236 case ISD::EXTLOAD:
237 return ISD::ANY_EXTEND;
238 case ISD::SEXTLOAD:
239 return ISD::SIGN_EXTEND;
240 case ISD::ZEXTLOAD:
241 return ISD::ZERO_EXTEND;
242 default:
243 break;
244 }
246 llvm_unreachable("Invalid LoadExtType");
247 }
249 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
250 /// when given the operation for (X op Y).
251 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
252 // To perform this operation, we just need to swap the L and G bits of the
253 // operation.
254 unsigned OldL = (Operation >> 2) & 1;
255 unsigned OldG = (Operation >> 1) & 1;
256 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
257 (OldL << 1) | // New G bit
258 (OldG << 2)); // New L bit.
259 }
261 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
262 /// 'op' is a valid SetCC operation.
263 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
264 unsigned Operation = Op;
265 if (isInteger)
266 Operation ^= 7; // Flip L, G, E bits, but not U.
267 else
268 Operation ^= 15; // Flip all of the condition bits.
270 if (Operation > ISD::SETTRUE2)
271 Operation &= ~8; // Don't let N and U bits get set.
273 return ISD::CondCode(Operation);
274 }
277 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
278 /// signed operation and 2 if the result is an unsigned comparison. Return zero
279 /// if the operation does not depend on the sign of the input (setne and seteq).
280 static int isSignedOp(ISD::CondCode Opcode) {
281 switch (Opcode) {
282 default: llvm_unreachable("Illegal integer setcc operation!");
283 case ISD::SETEQ:
284 case ISD::SETNE: return 0;
285 case ISD::SETLT:
286 case ISD::SETLE:
287 case ISD::SETGT:
288 case ISD::SETGE: return 1;
289 case ISD::SETULT:
290 case ISD::SETULE:
291 case ISD::SETUGT:
292 case ISD::SETUGE: return 2;
293 }
294 }
296 /// getSetCCOrOperation - Return the result of a logical OR between different
297 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function
298 /// returns SETCC_INVALID if it is not possible to represent the resultant
299 /// comparison.
300 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
301 bool isInteger) {
302 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
303 // Cannot fold a signed integer setcc with an unsigned integer setcc.
304 return ISD::SETCC_INVALID;
306 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
308 // If the N and U bits get set then the resultant comparison DOES suddenly
309 // care about orderedness, and is true when ordered.
310 if (Op > ISD::SETTRUE2)
311 Op &= ~16; // Clear the U bit if the N bit is set.
313 // Canonicalize illegal integer setcc's.
314 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
315 Op = ISD::SETNE;
317 return ISD::CondCode(Op);
318 }
320 /// getSetCCAndOperation - Return the result of a logical AND between different
321 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This
322 /// function returns zero if it is not possible to represent the resultant
323 /// comparison.
324 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
325 bool isInteger) {
326 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
327 // Cannot fold a signed setcc with an unsigned setcc.
328 return ISD::SETCC_INVALID;
330 // Combine all of the condition bits.
331 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
333 // Canonicalize illegal integer setcc's.
334 if (isInteger) {
335 switch (Result) {
336 default: break;
337 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
338 case ISD::SETOEQ: // SETEQ & SETU[LG]E
339 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
340 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
341 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
342 }
343 }
345 return Result;
346 }
348 //===----------------------------------------------------------------------===//
349 // SDNode Profile Support
350 //===----------------------------------------------------------------------===//
352 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
353 ///
354 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
355 ID.AddInteger(OpC);
356 }
358 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
359 /// solely with their pointer.
360 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
361 ID.AddPointer(VTList.VTs);
362 }
364 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
365 ///
366 static void AddNodeIDOperands(FoldingSetNodeID &ID,
367 const SDValue *Ops, unsigned NumOps) {
368 for (; NumOps; --NumOps, ++Ops) {
369 ID.AddPointer(Ops->getNode());
370 ID.AddInteger(Ops->getResNo());
371 }
372 }
374 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
375 ///
376 static void AddNodeIDOperands(FoldingSetNodeID &ID,
377 const SDUse *Ops, unsigned NumOps) {
378 for (; NumOps; --NumOps, ++Ops) {
379 ID.AddPointer(Ops->getNode());
380 ID.AddInteger(Ops->getResNo());
381 }
382 }
384 static void AddNodeIDNode(FoldingSetNodeID &ID,
385 unsigned short OpC, SDVTList VTList,
386 const SDValue *OpList, unsigned N) {
387 AddNodeIDOpcode(ID, OpC);
388 AddNodeIDValueTypes(ID, VTList);
389 AddNodeIDOperands(ID, OpList, N);
390 }
392 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
393 /// the NodeID data.
394 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
395 switch (N->getOpcode()) {
396 case ISD::TargetExternalSymbol:
397 case ISD::ExternalSymbol:
398 llvm_unreachable("Should only be used on nodes with operands");
399 default: break; // Normal nodes don't need extra info.
400 case ISD::TargetConstant:
401 case ISD::Constant: {
402 const ConstantSDNode *C = cast<ConstantSDNode>(N);
403 ID.AddPointer(C->getConstantIntValue());
404 ID.AddBoolean(C->isOpaque());
405 break;
406 }
407 case ISD::TargetConstantFP:
408 case ISD::ConstantFP: {
409 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
410 break;
411 }
412 case ISD::TargetGlobalAddress:
413 case ISD::GlobalAddress:
414 case ISD::TargetGlobalTLSAddress:
415 case ISD::GlobalTLSAddress: {
416 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
417 ID.AddPointer(GA->getGlobal());
418 ID.AddInteger(GA->getOffset());
419 ID.AddInteger(GA->getTargetFlags());
420 ID.AddInteger(GA->getAddressSpace());
421 break;
422 }
423 case ISD::BasicBlock:
424 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
425 break;
426 case ISD::Register:
427 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
428 break;
429 case ISD::RegisterMask:
430 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
431 break;
432 case ISD::SRCVALUE:
433 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
434 break;
435 case ISD::FrameIndex:
436 case ISD::TargetFrameIndex:
437 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
438 break;
439 case ISD::JumpTable:
440 case ISD::TargetJumpTable:
441 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
442 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
443 break;
444 case ISD::ConstantPool:
445 case ISD::TargetConstantPool: {
446 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
447 ID.AddInteger(CP->getAlignment());
448 ID.AddInteger(CP->getOffset());
449 if (CP->isMachineConstantPoolEntry())
450 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
451 else
452 ID.AddPointer(CP->getConstVal());
453 ID.AddInteger(CP->getTargetFlags());
454 break;
455 }
456 case ISD::TargetIndex: {
457 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
458 ID.AddInteger(TI->getIndex());
459 ID.AddInteger(TI->getOffset());
460 ID.AddInteger(TI->getTargetFlags());
461 break;
462 }
463 case ISD::LOAD: {
464 const LoadSDNode *LD = cast<LoadSDNode>(N);
465 ID.AddInteger(LD->getMemoryVT().getRawBits());
466 ID.AddInteger(LD->getRawSubclassData());
467 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
468 break;
469 }
470 case ISD::STORE: {
471 const StoreSDNode *ST = cast<StoreSDNode>(N);
472 ID.AddInteger(ST->getMemoryVT().getRawBits());
473 ID.AddInteger(ST->getRawSubclassData());
474 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
475 break;
476 }
477 case ISD::ATOMIC_CMP_SWAP:
478 case ISD::ATOMIC_SWAP:
479 case ISD::ATOMIC_LOAD_ADD:
480 case ISD::ATOMIC_LOAD_SUB:
481 case ISD::ATOMIC_LOAD_AND:
482 case ISD::ATOMIC_LOAD_OR:
483 case ISD::ATOMIC_LOAD_XOR:
484 case ISD::ATOMIC_LOAD_NAND:
485 case ISD::ATOMIC_LOAD_MIN:
486 case ISD::ATOMIC_LOAD_MAX:
487 case ISD::ATOMIC_LOAD_UMIN:
488 case ISD::ATOMIC_LOAD_UMAX:
489 case ISD::ATOMIC_LOAD:
490 case ISD::ATOMIC_STORE: {
491 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
492 ID.AddInteger(AT->getMemoryVT().getRawBits());
493 ID.AddInteger(AT->getRawSubclassData());
494 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
495 break;
496 }
497 case ISD::PREFETCH: {
498 const MemSDNode *PF = cast<MemSDNode>(N);
499 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
500 break;
501 }
502 case ISD::VECTOR_SHUFFLE: {
503 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
504 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
505 i != e; ++i)
506 ID.AddInteger(SVN->getMaskElt(i));
507 break;
508 }
509 case ISD::TargetBlockAddress:
510 case ISD::BlockAddress: {
511 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
512 ID.AddPointer(BA->getBlockAddress());
513 ID.AddInteger(BA->getOffset());
514 ID.AddInteger(BA->getTargetFlags());
515 break;
516 }
517 } // end switch (N->getOpcode())
519 // Target specific memory nodes could also have address spaces to check.
520 if (N->isTargetMemoryOpcode())
521 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
522 }
524 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
525 /// data.
526 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
527 AddNodeIDOpcode(ID, N->getOpcode());
528 // Add the return value info.
529 AddNodeIDValueTypes(ID, N->getVTList());
530 // Add the operand info.
531 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands());
533 // Handle SDNode leafs with special info.
534 AddNodeIDCustom(ID, N);
535 }
537 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
538 /// the CSE map that carries volatility, temporalness, indexing mode, and
539 /// extension/truncation information.
540 ///
541 static inline unsigned
542 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
543 bool isNonTemporal, bool isInvariant) {
544 assert((ConvType & 3) == ConvType &&
545 "ConvType may not require more than 2 bits!");
546 assert((AM & 7) == AM &&
547 "AM may not require more than 3 bits!");
548 return ConvType |
549 (AM << 2) |
550 (isVolatile << 5) |
551 (isNonTemporal << 6) |
552 (isInvariant << 7);
553 }
555 //===----------------------------------------------------------------------===//
556 // SelectionDAG Class
557 //===----------------------------------------------------------------------===//
559 /// doNotCSE - Return true if CSE should not be performed for this node.
560 static bool doNotCSE(SDNode *N) {
561 if (N->getValueType(0) == MVT::Glue)
562 return true; // Never CSE anything that produces a flag.
564 switch (N->getOpcode()) {
565 default: break;
566 case ISD::HANDLENODE:
567 case ISD::EH_LABEL:
568 return true; // Never CSE these nodes.
569 }
571 // Check that remaining values produced are not flags.
572 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
573 if (N->getValueType(i) == MVT::Glue)
574 return true; // Never CSE anything that produces a flag.
576 return false;
577 }
579 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
580 /// SelectionDAG.
581 void SelectionDAG::RemoveDeadNodes() {
582 // Create a dummy node (which is not added to allnodes), that adds a reference
583 // to the root node, preventing it from being deleted.
584 HandleSDNode Dummy(getRoot());
586 SmallVector<SDNode*, 128> DeadNodes;
588 // Add all obviously-dead nodes to the DeadNodes worklist.
589 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
590 if (I->use_empty())
591 DeadNodes.push_back(I);
593 RemoveDeadNodes(DeadNodes);
595 // If the root changed (e.g. it was a dead load, update the root).
596 setRoot(Dummy.getValue());
597 }
599 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
600 /// given list, and any nodes that become unreachable as a result.
601 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
603 // Process the worklist, deleting the nodes and adding their uses to the
604 // worklist.
605 while (!DeadNodes.empty()) {
606 SDNode *N = DeadNodes.pop_back_val();
608 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
609 DUL->NodeDeleted(N, nullptr);
611 // Take the node out of the appropriate CSE map.
612 RemoveNodeFromCSEMaps(N);
614 // Next, brutally remove the operand list. This is safe to do, as there are
615 // no cycles in the graph.
616 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
617 SDUse &Use = *I++;
618 SDNode *Operand = Use.getNode();
619 Use.set(SDValue());
621 // Now that we removed this operand, see if there are no uses of it left.
622 if (Operand->use_empty())
623 DeadNodes.push_back(Operand);
624 }
626 DeallocateNode(N);
627 }
628 }
630 void SelectionDAG::RemoveDeadNode(SDNode *N){
631 SmallVector<SDNode*, 16> DeadNodes(1, N);
633 // Create a dummy node that adds a reference to the root node, preventing
634 // it from being deleted. (This matters if the root is an operand of the
635 // dead node.)
636 HandleSDNode Dummy(getRoot());
638 RemoveDeadNodes(DeadNodes);
639 }
641 void SelectionDAG::DeleteNode(SDNode *N) {
642 // First take this out of the appropriate CSE map.
643 RemoveNodeFromCSEMaps(N);
645 // Finally, remove uses due to operands of this node, remove from the
646 // AllNodes list, and delete the node.
647 DeleteNodeNotInCSEMaps(N);
648 }
650 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
651 assert(N != AllNodes.begin() && "Cannot delete the entry node!");
652 assert(N->use_empty() && "Cannot delete a node that is not dead!");
654 // Drop all of the operands and decrement used node's use counts.
655 N->DropOperands();
657 DeallocateNode(N);
658 }
660 void SelectionDAG::DeallocateNode(SDNode *N) {
661 if (N->OperandsNeedDelete)
662 delete[] N->OperandList;
664 // Set the opcode to DELETED_NODE to help catch bugs when node
665 // memory is reallocated.
666 N->NodeType = ISD::DELETED_NODE;
668 NodeAllocator.Deallocate(AllNodes.remove(N));
670 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
671 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N);
672 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
673 DbgVals[i]->setIsInvalidated();
674 }
676 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
677 /// correspond to it. This is useful when we're about to delete or repurpose
678 /// the node. We don't want future request for structurally identical nodes
679 /// to return N anymore.
680 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
681 bool Erased = false;
682 switch (N->getOpcode()) {
683 case ISD::HANDLENODE: return false; // noop.
684 case ISD::CONDCODE:
685 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
686 "Cond code doesn't exist!");
687 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
688 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
689 break;
690 case ISD::ExternalSymbol:
691 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
692 break;
693 case ISD::TargetExternalSymbol: {
694 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
695 Erased = TargetExternalSymbols.erase(
696 std::pair<std::string,unsigned char>(ESN->getSymbol(),
697 ESN->getTargetFlags()));
698 break;
699 }
700 case ISD::VALUETYPE: {
701 EVT VT = cast<VTSDNode>(N)->getVT();
702 if (VT.isExtended()) {
703 Erased = ExtendedValueTypeNodes.erase(VT);
704 } else {
705 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
706 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
707 }
708 break;
709 }
710 default:
711 // Remove it from the CSE Map.
712 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
713 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
714 Erased = CSEMap.RemoveNode(N);
715 break;
716 }
717 #ifndef NDEBUG
718 // Verify that the node was actually in one of the CSE maps, unless it has a
719 // flag result (which cannot be CSE'd) or is one of the special cases that are
720 // not subject to CSE.
721 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
722 !N->isMachineOpcode() && !doNotCSE(N)) {
723 N->dump(this);
724 dbgs() << "\n";
725 llvm_unreachable("Node is not in map!");
726 }
727 #endif
728 return Erased;
729 }
731 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
732 /// maps and modified in place. Add it back to the CSE maps, unless an identical
733 /// node already exists, in which case transfer all its users to the existing
734 /// node. This transfer can potentially trigger recursive merging.
735 ///
736 void
737 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
738 // For node types that aren't CSE'd, just act as if no identical node
739 // already exists.
740 if (!doNotCSE(N)) {
741 SDNode *Existing = CSEMap.GetOrInsertNode(N);
742 if (Existing != N) {
743 // If there was already an existing matching node, use ReplaceAllUsesWith
744 // to replace the dead one with the existing one. This can cause
745 // recursive merging of other unrelated nodes down the line.
746 ReplaceAllUsesWith(N, Existing);
748 // N is now dead. Inform the listeners and delete it.
749 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
750 DUL->NodeDeleted(N, Existing);
751 DeleteNodeNotInCSEMaps(N);
752 return;
753 }
754 }
756 // If the node doesn't already exist, we updated it. Inform listeners.
757 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
758 DUL->NodeUpdated(N);
759 }
761 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
762 /// were replaced with those specified. If this node is never memoized,
763 /// return null, otherwise return a pointer to the slot it would take. If a
764 /// node already exists with these operands, the slot will be non-null.
765 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
766 void *&InsertPos) {
767 if (doNotCSE(N))
768 return nullptr;
770 SDValue Ops[] = { Op };
771 FoldingSetNodeID ID;
772 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1);
773 AddNodeIDCustom(ID, N);
774 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
775 return Node;
776 }
778 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
779 /// were replaced with those specified. If this node is never memoized,
780 /// return null, otherwise return a pointer to the slot it would take. If a
781 /// node already exists with these operands, the slot will be non-null.
782 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
783 SDValue Op1, SDValue Op2,
784 void *&InsertPos) {
785 if (doNotCSE(N))
786 return nullptr;
788 SDValue Ops[] = { Op1, Op2 };
789 FoldingSetNodeID ID;
790 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2);
791 AddNodeIDCustom(ID, N);
792 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
793 return Node;
794 }
797 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
798 /// were replaced with those specified. If this node is never memoized,
799 /// return null, otherwise return a pointer to the slot it would take. If a
800 /// node already exists with these operands, the slot will be non-null.
801 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
802 const SDValue *Ops,unsigned NumOps,
803 void *&InsertPos) {
804 if (doNotCSE(N))
805 return nullptr;
807 FoldingSetNodeID ID;
808 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps);
809 AddNodeIDCustom(ID, N);
810 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
811 return Node;
812 }
814 #ifndef NDEBUG
815 /// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid.
816 static void VerifyNodeCommon(SDNode *N) {
817 switch (N->getOpcode()) {
818 default:
819 break;
820 case ISD::BUILD_PAIR: {
821 EVT VT = N->getValueType(0);
822 assert(N->getNumValues() == 1 && "Too many results!");
823 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
824 "Wrong return type!");
825 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
826 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
827 "Mismatched operand types!");
828 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
829 "Wrong operand type!");
830 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
831 "Wrong return type size");
832 break;
833 }
834 case ISD::BUILD_VECTOR: {
835 assert(N->getNumValues() == 1 && "Too many results!");
836 assert(N->getValueType(0).isVector() && "Wrong return type!");
837 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
838 "Wrong number of operands!");
839 EVT EltVT = N->getValueType(0).getVectorElementType();
840 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
841 assert((I->getValueType() == EltVT ||
842 (EltVT.isInteger() && I->getValueType().isInteger() &&
843 EltVT.bitsLE(I->getValueType()))) &&
844 "Wrong operand type!");
845 assert(I->getValueType() == N->getOperand(0).getValueType() &&
846 "Operands must all have the same type");
847 }
848 break;
849 }
850 }
851 }
853 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
854 static void VerifySDNode(SDNode *N) {
855 // The SDNode allocators cannot be used to allocate nodes with fields that are
856 // not present in an SDNode!
857 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!");
858 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!");
859 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!");
860 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!");
861 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!");
862 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!");
863 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!");
864 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!");
865 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!");
866 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!");
867 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!");
868 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!");
869 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!");
870 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!");
871 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!");
872 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!");
873 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!");
874 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!");
875 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!");
877 VerifyNodeCommon(N);
878 }
880 /// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is
881 /// invalid.
882 static void VerifyMachineNode(SDNode *N) {
883 // The MachineNode allocators cannot be used to allocate nodes with fields
884 // that are not present in a MachineNode!
885 // Currently there are no such nodes.
887 VerifyNodeCommon(N);
888 }
889 #endif // NDEBUG
891 /// getEVTAlignment - Compute the default alignment value for the
892 /// given type.
893 ///
894 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
895 Type *Ty = VT == MVT::iPTR ?
896 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
897 VT.getTypeForEVT(*getContext());
899 return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty);
900 }
902 // EntryNode could meaningfully have debug info if we can find it...
903 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
904 : TM(tm), TSI(*tm.getSelectionDAGInfo()), TLI(nullptr), OptLevel(OL),
905 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
906 Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
907 UpdateListeners(nullptr) {
908 AllNodes.push_back(&EntryNode);
909 DbgInfo = new SDDbgInfo();
910 }
912 void SelectionDAG::init(MachineFunction &mf, const TargetLowering *tli) {
913 MF = &mf;
914 TLI = tli;
915 Context = &mf.getFunction()->getContext();
916 }
918 SelectionDAG::~SelectionDAG() {
919 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
920 allnodes_clear();
921 delete DbgInfo;
922 }
924 void SelectionDAG::allnodes_clear() {
925 assert(&*AllNodes.begin() == &EntryNode);
926 AllNodes.remove(AllNodes.begin());
927 while (!AllNodes.empty())
928 DeallocateNode(AllNodes.begin());
929 }
931 void SelectionDAG::clear() {
932 allnodes_clear();
933 OperandAllocator.Reset();
934 CSEMap.clear();
936 ExtendedValueTypeNodes.clear();
937 ExternalSymbols.clear();
938 TargetExternalSymbols.clear();
939 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
940 static_cast<CondCodeSDNode*>(nullptr));
941 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
942 static_cast<SDNode*>(nullptr));
944 EntryNode.UseList = nullptr;
945 AllNodes.push_back(&EntryNode);
946 Root = getEntryNode();
947 DbgInfo->clear();
948 }
950 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
951 return VT.bitsGT(Op.getValueType()) ?
952 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
953 getNode(ISD::TRUNCATE, DL, VT, Op);
954 }
956 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
957 return VT.bitsGT(Op.getValueType()) ?
958 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
959 getNode(ISD::TRUNCATE, DL, VT, Op);
960 }
962 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
963 return VT.bitsGT(Op.getValueType()) ?
964 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
965 getNode(ISD::TRUNCATE, DL, VT, Op);
966 }
968 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
969 assert(!VT.isVector() &&
970 "getZeroExtendInReg should use the vector element type instead of "
971 "the vector type!");
972 if (Op.getValueType() == VT) return Op;
973 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
974 APInt Imm = APInt::getLowBitsSet(BitWidth,
975 VT.getSizeInBits());
976 return getNode(ISD::AND, DL, Op.getValueType(), Op,
977 getConstant(Imm, Op.getValueType()));
978 }
980 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
981 ///
982 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
983 EVT EltVT = VT.getScalarType();
984 SDValue NegOne =
985 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
986 return getNode(ISD::XOR, DL, VT, Val, NegOne);
987 }
989 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
990 EVT EltVT = VT.getScalarType();
991 assert((EltVT.getSizeInBits() >= 64 ||
992 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
993 "getConstant with a uint64_t value that doesn't fit in the type!");
994 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
995 }
997 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
998 {
999 return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
1000 }
1002 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
1003 bool isO) {
1004 assert(VT.isInteger() && "Cannot create FP integer constant!");
1006 EVT EltVT = VT.getScalarType();
1007 const ConstantInt *Elt = &Val;
1009 const TargetLowering *TLI = TM.getTargetLowering();
1011 // In some cases the vector type is legal but the element type is illegal and
1012 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1013 // inserted value (the type does not need to match the vector element type).
1014 // Any extra bits introduced will be truncated away.
1015 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1016 TargetLowering::TypePromoteInteger) {
1017 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1018 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
1019 Elt = ConstantInt::get(*getContext(), NewVal);
1020 }
1021 // In other cases the element type is illegal and needs to be expanded, for
1022 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1023 // the value into n parts and use a vector type with n-times the elements.
1024 // Then bitcast to the type requested.
1025 // Legalizing constants too early makes the DAGCombiner's job harder so we
1026 // only legalize if the DAG tells us we must produce legal types.
1027 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1028 TLI->getTypeAction(*getContext(), EltVT) ==
1029 TargetLowering::TypeExpandInteger) {
1030 APInt NewVal = Elt->getValue();
1031 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1032 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1033 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1034 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1036 // Check the temporary vector is the correct size. If this fails then
1037 // getTypeToTransformTo() probably returned a type whose size (in bits)
1038 // isn't a power-of-2 factor of the requested type size.
1039 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1041 SmallVector<SDValue, 2> EltParts;
1042 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1043 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1044 .trunc(ViaEltSizeInBits),
1045 ViaEltVT, isT, isO));
1046 }
1048 // EltParts is currently in little endian order. If we actually want
1049 // big-endian order then reverse it now.
1050 if (TLI->isBigEndian())
1051 std::reverse(EltParts.begin(), EltParts.end());
1053 // The elements must be reversed when the element order is different
1054 // to the endianness of the elements (because the BITCAST is itself a
1055 // vector shuffle in this situation). However, we do not need any code to
1056 // perform this reversal because getConstant() is producing a vector
1057 // splat.
1058 // This situation occurs in MIPS MSA.
1060 SmallVector<SDValue, 8> Ops;
1061 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
1062 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1064 SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
1065 getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
1066 &Ops[0], Ops.size()));
1067 return Result;
1068 }
1070 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1071 "APInt size does not match type size!");
1072 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1073 FoldingSetNodeID ID;
1074 AddNodeIDNode(ID, Opc, getVTList(EltVT), nullptr, 0);
1075 ID.AddPointer(Elt);
1076 ID.AddBoolean(isO);
1077 void *IP = nullptr;
1078 SDNode *N = nullptr;
1079 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
1080 if (!VT.isVector())
1081 return SDValue(N, 0);
1083 if (!N) {
1084 N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
1085 CSEMap.InsertNode(N, IP);
1086 AllNodes.push_back(N);
1087 }
1089 SDValue Result(N, 0);
1090 if (VT.isVector()) {
1091 SmallVector<SDValue, 8> Ops;
1092 Ops.assign(VT.getVectorNumElements(), Result);
1093 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size());
1094 }
1095 return Result;
1096 }
1098 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
1099 return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget);
1100 }
1103 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
1104 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
1105 }
1107 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
1108 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1110 EVT EltVT = VT.getScalarType();
1112 // Do the map lookup using the actual bit pattern for the floating point
1113 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1114 // we don't have issues with SNANs.
1115 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1116 FoldingSetNodeID ID;
1117 AddNodeIDNode(ID, Opc, getVTList(EltVT), nullptr, 0);
1118 ID.AddPointer(&V);
1119 void *IP = nullptr;
1120 SDNode *N = nullptr;
1121 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
1122 if (!VT.isVector())
1123 return SDValue(N, 0);
1125 if (!N) {
1126 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
1127 CSEMap.InsertNode(N, IP);
1128 AllNodes.push_back(N);
1129 }
1131 SDValue Result(N, 0);
1132 if (VT.isVector()) {
1133 SmallVector<SDValue, 8> Ops;
1134 Ops.assign(VT.getVectorNumElements(), Result);
1135 // FIXME SDLoc info might be appropriate here
1136 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size());
1137 }
1138 return Result;
1139 }
1141 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
1142 EVT EltVT = VT.getScalarType();
1143 if (EltVT==MVT::f32)
1144 return getConstantFP(APFloat((float)Val), VT, isTarget);
1145 else if (EltVT==MVT::f64)
1146 return getConstantFP(APFloat(Val), VT, isTarget);
1147 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
1148 EltVT==MVT::f16) {
1149 bool ignored;
1150 APFloat apf = APFloat(Val);
1151 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1152 &ignored);
1153 return getConstantFP(apf, VT, isTarget);
1154 } else
1155 llvm_unreachable("Unsupported type in getConstantFP");
1156 }
1158 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
1159 EVT VT, int64_t Offset,
1160 bool isTargetGA,
1161 unsigned char TargetFlags) {
1162 assert((TargetFlags == 0 || isTargetGA) &&
1163 "Cannot set target flags on target-independent globals");
1164 const TargetLowering *TLI = TM.getTargetLowering();
1166 // Truncate (with sign-extension) the offset value to the pointer size.
1167 unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
1168 if (BitWidth < 64)
1169 Offset = SignExtend64(Offset, BitWidth);
1171 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
1172 if (!GVar) {
1173 // If GV is an alias then use the aliasee for determining thread-localness.
1174 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
1175 GVar = dyn_cast_or_null<GlobalVariable>(GA->getAliasedGlobal());
1176 }
1178 unsigned Opc;
1179 if (GVar && GVar->isThreadLocal())
1180 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1181 else
1182 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1184 FoldingSetNodeID ID;
1185 AddNodeIDNode(ID, Opc, getVTList(VT), nullptr, 0);
1186 ID.AddPointer(GV);
1187 ID.AddInteger(Offset);
1188 ID.AddInteger(TargetFlags);
1189 ID.AddInteger(GV->getType()->getAddressSpace());
1190 void *IP = nullptr;
1191 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1192 return SDValue(E, 0);
1194 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
1195 DL.getDebugLoc(), GV, VT,
1196 Offset, TargetFlags);
1197 CSEMap.InsertNode(N, IP);
1198 AllNodes.push_back(N);
1199 return SDValue(N, 0);
1200 }
1202 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1203 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1204 FoldingSetNodeID ID;
1205 AddNodeIDNode(ID, Opc, getVTList(VT), nullptr, 0);
1206 ID.AddInteger(FI);
1207 void *IP = nullptr;
1208 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1209 return SDValue(E, 0);
1211 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
1212 CSEMap.InsertNode(N, IP);
1213 AllNodes.push_back(N);
1214 return SDValue(N, 0);
1215 }
1217 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1218 unsigned char TargetFlags) {
1219 assert((TargetFlags == 0 || isTarget) &&
1220 "Cannot set target flags on target-independent jump tables");
1221 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1222 FoldingSetNodeID ID;
1223 AddNodeIDNode(ID, Opc, getVTList(VT), nullptr, 0);
1224 ID.AddInteger(JTI);
1225 ID.AddInteger(TargetFlags);
1226 void *IP = nullptr;
1227 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1228 return SDValue(E, 0);
1230 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
1231 TargetFlags);
1232 CSEMap.InsertNode(N, IP);
1233 AllNodes.push_back(N);
1234 return SDValue(N, 0);
1235 }
1237 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1238 unsigned Alignment, int Offset,
1239 bool isTarget,
1240 unsigned char TargetFlags) {
1241 assert((TargetFlags == 0 || isTarget) &&
1242 "Cannot set target flags on target-independent globals");
1243 if (Alignment == 0)
1244 Alignment =
1245 TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType());
1246 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1247 FoldingSetNodeID ID;
1248 AddNodeIDNode(ID, Opc, getVTList(VT), nullptr, 0);
1249 ID.AddInteger(Alignment);
1250 ID.AddInteger(Offset);
1251 ID.AddPointer(C);
1252 ID.AddInteger(TargetFlags);
1253 void *IP = nullptr;
1254 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1255 return SDValue(E, 0);
1257 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1258 Alignment, TargetFlags);
1259 CSEMap.InsertNode(N, IP);
1260 AllNodes.push_back(N);
1261 return SDValue(N, 0);
1262 }
1265 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1266 unsigned Alignment, int Offset,
1267 bool isTarget,
1268 unsigned char TargetFlags) {
1269 assert((TargetFlags == 0 || isTarget) &&
1270 "Cannot set target flags on target-independent globals");
1271 if (Alignment == 0)
1272 Alignment =
1273 TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType());
1274 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1275 FoldingSetNodeID ID;
1276 AddNodeIDNode(ID, Opc, getVTList(VT), nullptr, 0);
1277 ID.AddInteger(Alignment);
1278 ID.AddInteger(Offset);
1279 C->addSelectionDAGCSEId(ID);
1280 ID.AddInteger(TargetFlags);
1281 void *IP = nullptr;
1282 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1283 return SDValue(E, 0);
1285 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1286 Alignment, TargetFlags);
1287 CSEMap.InsertNode(N, IP);
1288 AllNodes.push_back(N);
1289 return SDValue(N, 0);
1290 }
1292 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1293 unsigned char TargetFlags) {
1294 FoldingSetNodeID ID;
1295 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), nullptr, 0);
1296 ID.AddInteger(Index);
1297 ID.AddInteger(Offset);
1298 ID.AddInteger(TargetFlags);
1299 void *IP = nullptr;
1300 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1301 return SDValue(E, 0);
1303 SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
1304 TargetFlags);
1305 CSEMap.InsertNode(N, IP);
1306 AllNodes.push_back(N);
1307 return SDValue(N, 0);
1308 }
1310 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1311 FoldingSetNodeID ID;
1312 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), nullptr, 0);
1313 ID.AddPointer(MBB);
1314 void *IP = nullptr;
1315 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1316 return SDValue(E, 0);
1318 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
1319 CSEMap.InsertNode(N, IP);
1320 AllNodes.push_back(N);
1321 return SDValue(N, 0);
1322 }
1324 SDValue SelectionDAG::getValueType(EVT VT) {
1325 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1326 ValueTypeNodes.size())
1327 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1329 SDNode *&N = VT.isExtended() ?
1330 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1332 if (N) return SDValue(N, 0);
1333 N = new (NodeAllocator) VTSDNode(VT);
1334 AllNodes.push_back(N);
1335 return SDValue(N, 0);
1336 }
1338 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1339 SDNode *&N = ExternalSymbols[Sym];
1340 if (N) return SDValue(N, 0);
1341 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
1342 AllNodes.push_back(N);
1343 return SDValue(N, 0);
1344 }
1346 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1347 unsigned char TargetFlags) {
1348 SDNode *&N =
1349 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1350 TargetFlags)];
1351 if (N) return SDValue(N, 0);
1352 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
1353 AllNodes.push_back(N);
1354 return SDValue(N, 0);
1355 }
1357 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1358 if ((unsigned)Cond >= CondCodeNodes.size())
1359 CondCodeNodes.resize(Cond+1);
1361 if (!CondCodeNodes[Cond]) {
1362 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
1363 CondCodeNodes[Cond] = N;
1364 AllNodes.push_back(N);
1365 }
1367 return SDValue(CondCodeNodes[Cond], 0);
1368 }
1370 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
1371 // the shuffle mask M that point at N1 to point at N2, and indices that point
1372 // N2 to point at N1.
1373 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
1374 std::swap(N1, N2);
1375 int NElts = M.size();
1376 for (int i = 0; i != NElts; ++i) {
1377 if (M[i] >= NElts)
1378 M[i] -= NElts;
1379 else if (M[i] >= 0)
1380 M[i] += NElts;
1381 }
1382 }
1384 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
1385 SDValue N2, const int *Mask) {
1386 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1387 "Invalid VECTOR_SHUFFLE");
1389 // Canonicalize shuffle undef, undef -> undef
1390 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
1391 return getUNDEF(VT);
1393 // Validate that all indices in Mask are within the range of the elements
1394 // input to the shuffle.
1395 unsigned NElts = VT.getVectorNumElements();
1396 SmallVector<int, 8> MaskVec;
1397 for (unsigned i = 0; i != NElts; ++i) {
1398 assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
1399 MaskVec.push_back(Mask[i]);
1400 }
1402 // Canonicalize shuffle v, v -> v, undef
1403 if (N1 == N2) {
1404 N2 = getUNDEF(VT);
1405 for (unsigned i = 0; i != NElts; ++i)
1406 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
1407 }
1409 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1410 if (N1.getOpcode() == ISD::UNDEF)
1411 commuteShuffle(N1, N2, MaskVec);
1413 // Canonicalize all index into lhs, -> shuffle lhs, undef
1414 // Canonicalize all index into rhs, -> shuffle rhs, undef
1415 bool AllLHS = true, AllRHS = true;
1416 bool N2Undef = N2.getOpcode() == ISD::UNDEF;
1417 for (unsigned i = 0; i != NElts; ++i) {
1418 if (MaskVec[i] >= (int)NElts) {
1419 if (N2Undef)
1420 MaskVec[i] = -1;
1421 else
1422 AllLHS = false;
1423 } else if (MaskVec[i] >= 0) {
1424 AllRHS = false;
1425 }
1426 }
1427 if (AllLHS && AllRHS)
1428 return getUNDEF(VT);
1429 if (AllLHS && !N2Undef)
1430 N2 = getUNDEF(VT);
1431 if (AllRHS) {
1432 N1 = getUNDEF(VT);
1433 commuteShuffle(N1, N2, MaskVec);
1434 }
1436 // If Identity shuffle return that node.
1437 bool Identity = true;
1438 for (unsigned i = 0; i != NElts; ++i) {
1439 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
1440 }
1441 if (Identity && NElts)
1442 return N1;
1444 FoldingSetNodeID ID;
1445 SDValue Ops[2] = { N1, N2 };
1446 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2);
1447 for (unsigned i = 0; i != NElts; ++i)
1448 ID.AddInteger(MaskVec[i]);
1450 void* IP = nullptr;
1451 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1452 return SDValue(E, 0);
1454 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1455 // SDNode doesn't have access to it. This memory will be "leaked" when
1456 // the node is deallocated, but recovered when the NodeAllocator is released.
1457 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1458 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
1460 ShuffleVectorSDNode *N =
1461 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
1462 dl.getDebugLoc(), N1, N2,
1463 MaskAlloc);
1464 CSEMap.InsertNode(N, IP);
1465 AllNodes.push_back(N);
1466 return SDValue(N, 0);
1467 }
1469 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
1470 SDValue Val, SDValue DTy,
1471 SDValue STy, SDValue Rnd, SDValue Sat,
1472 ISD::CvtCode Code) {
1473 // If the src and dest types are the same and the conversion is between
1474 // integer types of the same sign or two floats, no conversion is necessary.
1475 if (DTy == STy &&
1476 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
1477 return Val;
1479 FoldingSetNodeID ID;
1480 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
1481 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5);
1482 void* IP = nullptr;
1483 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1484 return SDValue(E, 0);
1486 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
1487 dl.getDebugLoc(),
1488 Ops, 5, Code);
1489 CSEMap.InsertNode(N, IP);
1490 AllNodes.push_back(N);
1491 return SDValue(N, 0);
1492 }
1494 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1495 FoldingSetNodeID ID;
1496 AddNodeIDNode(ID, ISD::Register, getVTList(VT), nullptr, 0);
1497 ID.AddInteger(RegNo);
1498 void *IP = nullptr;
1499 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1500 return SDValue(E, 0);
1502 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
1503 CSEMap.InsertNode(N, IP);
1504 AllNodes.push_back(N);
1505 return SDValue(N, 0);
1506 }
1508 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1509 FoldingSetNodeID ID;
1510 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), nullptr, 0);
1511 ID.AddPointer(RegMask);
1512 void *IP = nullptr;
1513 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1514 return SDValue(E, 0);
1516 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
1517 CSEMap.InsertNode(N, IP);
1518 AllNodes.push_back(N);
1519 return SDValue(N, 0);
1520 }
1522 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
1523 FoldingSetNodeID ID;
1524 SDValue Ops[] = { Root };
1525 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1);
1526 ID.AddPointer(Label);
1527 void *IP = nullptr;
1528 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1529 return SDValue(E, 0);
1531 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
1532 dl.getDebugLoc(), Root, Label);
1533 CSEMap.InsertNode(N, IP);
1534 AllNodes.push_back(N);
1535 return SDValue(N, 0);
1536 }
1539 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1540 int64_t Offset,
1541 bool isTarget,
1542 unsigned char TargetFlags) {
1543 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1545 FoldingSetNodeID ID;
1546 AddNodeIDNode(ID, Opc, getVTList(VT), nullptr, 0);
1547 ID.AddPointer(BA);
1548 ID.AddInteger(Offset);
1549 ID.AddInteger(TargetFlags);
1550 void *IP = nullptr;
1551 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1552 return SDValue(E, 0);
1554 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
1555 TargetFlags);
1556 CSEMap.InsertNode(N, IP);
1557 AllNodes.push_back(N);
1558 return SDValue(N, 0);
1559 }
1561 SDValue SelectionDAG::getSrcValue(const Value *V) {
1562 assert((!V || V->getType()->isPointerTy()) &&
1563 "SrcValue is not a pointer?");
1565 FoldingSetNodeID ID;
1566 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), nullptr, 0);
1567 ID.AddPointer(V);
1569 void *IP = nullptr;
1570 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1571 return SDValue(E, 0);
1573 SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
1574 CSEMap.InsertNode(N, IP);
1575 AllNodes.push_back(N);
1576 return SDValue(N, 0);
1577 }
1579 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
1580 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1581 FoldingSetNodeID ID;
1582 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), nullptr, 0);
1583 ID.AddPointer(MD);
1585 void *IP = nullptr;
1586 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1587 return SDValue(E, 0);
1589 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
1590 CSEMap.InsertNode(N, IP);
1591 AllNodes.push_back(N);
1592 return SDValue(N, 0);
1593 }
1595 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
1596 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
1597 unsigned SrcAS, unsigned DestAS) {
1598 SDValue Ops[] = {Ptr};
1599 FoldingSetNodeID ID;
1600 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), &Ops[0], 1);
1601 ID.AddInteger(SrcAS);
1602 ID.AddInteger(DestAS);
1604 void *IP = nullptr;
1605 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
1606 return SDValue(E, 0);
1608 SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
1609 dl.getDebugLoc(),
1610 VT, Ptr, SrcAS, DestAS);
1611 CSEMap.InsertNode(N, IP);
1612 AllNodes.push_back(N);
1613 return SDValue(N, 0);
1614 }
1616 /// getShiftAmountOperand - Return the specified value casted to
1617 /// the target's desired shift amount type.
1618 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1619 EVT OpTy = Op.getValueType();
1620 EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy);
1621 if (OpTy == ShTy || OpTy.isVector()) return Op;
1623 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
1624 return getNode(Opcode, SDLoc(Op), ShTy, Op);
1625 }
1627 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
1628 /// specified value type.
1629 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1630 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
1631 unsigned ByteSize = VT.getStoreSize();
1632 Type *Ty = VT.getTypeForEVT(*getContext());
1633 const TargetLowering *TLI = TM.getTargetLowering();
1634 unsigned StackAlign =
1635 std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
1637 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
1638 return getFrameIndex(FrameIdx, TLI->getPointerTy());
1639 }
1641 /// CreateStackTemporary - Create a stack temporary suitable for holding
1642 /// either of the specified value types.
1643 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1644 unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
1645 VT2.getStoreSizeInBits())/8;
1646 Type *Ty1 = VT1.getTypeForEVT(*getContext());
1647 Type *Ty2 = VT2.getTypeForEVT(*getContext());
1648 const TargetLowering *TLI = TM.getTargetLowering();
1649 const DataLayout *TD = TLI->getDataLayout();
1650 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
1651 TD->getPrefTypeAlignment(Ty2));
1653 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
1654 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
1655 return getFrameIndex(FrameIdx, TLI->getPointerTy());
1656 }
1658 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
1659 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
1660 // These setcc operations always fold.
1661 switch (Cond) {
1662 default: break;
1663 case ISD::SETFALSE:
1664 case ISD::SETFALSE2: return getConstant(0, VT);
1665 case ISD::SETTRUE:
1666 case ISD::SETTRUE2: {
1667 const TargetLowering *TLI = TM.getTargetLowering();
1668 TargetLowering::BooleanContent Cnt = TLI->getBooleanContents(VT.isVector());
1669 return getConstant(
1670 Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
1671 }
1673 case ISD::SETOEQ:
1674 case ISD::SETOGT:
1675 case ISD::SETOGE:
1676 case ISD::SETOLT:
1677 case ISD::SETOLE:
1678 case ISD::SETONE:
1679 case ISD::SETO:
1680 case ISD::SETUO:
1681 case ISD::SETUEQ:
1682 case ISD::SETUNE:
1683 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
1684 break;
1685 }
1687 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
1688 const APInt &C2 = N2C->getAPIntValue();
1689 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
1690 const APInt &C1 = N1C->getAPIntValue();
1692 switch (Cond) {
1693 default: llvm_unreachable("Unknown integer setcc!");
1694 case ISD::SETEQ: return getConstant(C1 == C2, VT);
1695 case ISD::SETNE: return getConstant(C1 != C2, VT);
1696 case ISD::SETULT: return getConstant(C1.ult(C2), VT);
1697 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
1698 case ISD::SETULE: return getConstant(C1.ule(C2), VT);
1699 case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
1700 case ISD::SETLT: return getConstant(C1.slt(C2), VT);
1701 case ISD::SETGT: return getConstant(C1.sgt(C2), VT);
1702 case ISD::SETLE: return getConstant(C1.sle(C2), VT);
1703 case ISD::SETGE: return getConstant(C1.sge(C2), VT);
1704 }
1705 }
1706 }
1707 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
1708 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
1709 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
1710 switch (Cond) {
1711 default: break;
1712 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
1713 return getUNDEF(VT);
1714 // fall through
1715 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
1716 case ISD::SETNE: if (R==APFloat::cmpUnordered)
1717 return getUNDEF(VT);
1718 // fall through
1719 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
1720 R==APFloat::cmpLessThan, VT);
1721 case ISD::SETLT: if (R==APFloat::cmpUnordered)
1722 return getUNDEF(VT);
1723 // fall through
1724 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
1725 case ISD::SETGT: if (R==APFloat::cmpUnordered)
1726 return getUNDEF(VT);
1727 // fall through
1728 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
1729 case ISD::SETLE: if (R==APFloat::cmpUnordered)
1730 return getUNDEF(VT);
1731 // fall through
1732 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
1733 R==APFloat::cmpEqual, VT);
1734 case ISD::SETGE: if (R==APFloat::cmpUnordered)
1735 return getUNDEF(VT);
1736 // fall through
1737 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
1738 R==APFloat::cmpEqual, VT);
1739 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT);
1740 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT);
1741 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
1742 R==APFloat::cmpEqual, VT);
1743 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
1744 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
1745 R==APFloat::cmpLessThan, VT);
1746 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
1747 R==APFloat::cmpUnordered, VT);
1748 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
1749 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
1750 }
1751 } else {
1752 // Ensure that the constant occurs on the RHS.
1753 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
1754 MVT CompVT = N1.getValueType().getSimpleVT();
1755 if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT))
1756 return SDValue();
1758 return getSetCC(dl, VT, N2, N1, SwappedCond);
1759 }
1760 }
1762 // Could not fold it.
1763 return SDValue();
1764 }
1766 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
1767 /// use this predicate to simplify operations downstream.
1768 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
1769 // This predicate is not safe for vector operations.
1770 if (Op.getValueType().isVector())
1771 return false;
1773 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1774 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
1775 }
1777 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
1778 /// this predicate to simplify operations downstream. Mask is known to be zero
1779 /// for bits that V cannot have.
1780 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
1781 unsigned Depth) const {
1782 APInt KnownZero, KnownOne;
1783 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth);
1784 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1785 return (KnownZero & Mask) == Mask;
1786 }
1788 /// ComputeMaskedBits - Determine which of the bits specified in Mask are
1789 /// known to be either zero or one and return them in the KnownZero/KnownOne
1790 /// bitsets. This code only analyzes bits in Mask, in order to short-circuit
1791 /// processing.
1792 void SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero,
1793 APInt &KnownOne, unsigned Depth) const {
1794 const TargetLowering *TLI = TM.getTargetLowering();
1795 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1797 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
1798 if (Depth == 6)
1799 return; // Limit search depth.
1801 APInt KnownZero2, KnownOne2;
1803 switch (Op.getOpcode()) {
1804 case ISD::Constant:
1805 // We know all of the bits for a constant!
1806 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
1807 KnownZero = ~KnownOne;
1808 return;
1809 case ISD::AND:
1810 // If either the LHS or the RHS are Zero, the result is zero.
1811 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1812 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1813 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1814 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1816 // Output known-1 bits are only known if set in both the LHS & RHS.
1817 KnownOne &= KnownOne2;
1818 // Output known-0 are known to be clear if zero in either the LHS | RHS.
1819 KnownZero |= KnownZero2;
1820 return;
1821 case ISD::OR:
1822 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1823 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1824 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1825 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1827 // Output known-0 bits are only known if clear in both the LHS & RHS.
1828 KnownZero &= KnownZero2;
1829 // Output known-1 are known to be set if set in either the LHS | RHS.
1830 KnownOne |= KnownOne2;
1831 return;
1832 case ISD::XOR: {
1833 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1834 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1835 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1836 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1838 // Output known-0 bits are known if clear or set in both the LHS & RHS.
1839 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
1840 // Output known-1 are known to be set if set in only one of the LHS, RHS.
1841 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
1842 KnownZero = KnownZeroOut;
1843 return;
1844 }
1845 case ISD::MUL: {
1846 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
1847 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1848 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1849 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1851 // If low bits are zero in either operand, output low known-0 bits.
1852 // Also compute a conserative estimate for high known-0 bits.
1853 // More trickiness is possible, but this is sufficient for the
1854 // interesting case of alignment computation.
1855 KnownOne.clearAllBits();
1856 unsigned TrailZ = KnownZero.countTrailingOnes() +
1857 KnownZero2.countTrailingOnes();
1858 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() +
1859 KnownZero2.countLeadingOnes(),
1860 BitWidth) - BitWidth;
1862 TrailZ = std::min(TrailZ, BitWidth);
1863 LeadZ = std::min(LeadZ, BitWidth);
1864 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
1865 APInt::getHighBitsSet(BitWidth, LeadZ);
1866 return;
1867 }
1868 case ISD::UDIV: {
1869 // For the purposes of computing leading zeros we can conservatively
1870 // treat a udiv as a logical right shift by the power of 2 known to
1871 // be less than the denominator.
1872 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
1873 unsigned LeadZ = KnownZero2.countLeadingOnes();
1875 KnownOne2.clearAllBits();
1876 KnownZero2.clearAllBits();
1877 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
1878 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
1879 if (RHSUnknownLeadingOnes != BitWidth)
1880 LeadZ = std::min(BitWidth,
1881 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
1883 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
1884 return;
1885 }
1886 case ISD::SELECT:
1887 ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
1888 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
1889 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1890 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1892 // Only known if known in both the LHS and RHS.
1893 KnownOne &= KnownOne2;
1894 KnownZero &= KnownZero2;
1895 return;
1896 case ISD::SELECT_CC:
1897 ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
1898 ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
1899 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1900 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
1902 // Only known if known in both the LHS and RHS.
1903 KnownOne &= KnownOne2;
1904 KnownZero &= KnownZero2;
1905 return;
1906 case ISD::SADDO:
1907 case ISD::UADDO:
1908 case ISD::SSUBO:
1909 case ISD::USUBO:
1910 case ISD::SMULO:
1911 case ISD::UMULO:
1912 if (Op.getResNo() != 1)
1913 return;
1914 // The boolean result conforms to getBooleanContents. Fall through.
1915 case ISD::SETCC:
1916 // If we know the result of a setcc has the top bits zero, use this info.
1917 if (TLI->getBooleanContents(Op.getValueType().isVector()) ==
1918 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1)
1919 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
1920 return;
1921 case ISD::SHL:
1922 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
1923 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1924 unsigned ShAmt = SA->getZExtValue();
1926 // If the shift count is an invalid immediate, don't do anything.
1927 if (ShAmt >= BitWidth)
1928 return;
1930 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
1931 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1932 KnownZero <<= ShAmt;
1933 KnownOne <<= ShAmt;
1934 // low bits known zero.
1935 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
1936 }
1937 return;
1938 case ISD::SRL:
1939 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
1940 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1941 unsigned ShAmt = SA->getZExtValue();
1943 // If the shift count is an invalid immediate, don't do anything.
1944 if (ShAmt >= BitWidth)
1945 return;
1947 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
1948 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1949 KnownZero = KnownZero.lshr(ShAmt);
1950 KnownOne = KnownOne.lshr(ShAmt);
1952 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
1953 KnownZero |= HighBits; // High bits known zero.
1954 }
1955 return;
1956 case ISD::SRA:
1957 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1958 unsigned ShAmt = SA->getZExtValue();
1960 // If the shift count is an invalid immediate, don't do anything.
1961 if (ShAmt >= BitWidth)
1962 return;
1964 // If any of the demanded bits are produced by the sign extension, we also
1965 // demand the input sign bit.
1966 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
1968 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
1969 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
1970 KnownZero = KnownZero.lshr(ShAmt);
1971 KnownOne = KnownOne.lshr(ShAmt);
1973 // Handle the sign bits.
1974 APInt SignBit = APInt::getSignBit(BitWidth);
1975 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask.
1977 if (KnownZero.intersects(SignBit)) {
1978 KnownZero |= HighBits; // New bits are known zero.
1979 } else if (KnownOne.intersects(SignBit)) {
1980 KnownOne |= HighBits; // New bits are known one.
1981 }
1982 }
1983 return;
1984 case ISD::SIGN_EXTEND_INREG: {
1985 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1986 unsigned EBits = EVT.getScalarType().getSizeInBits();
1988 // Sign extension. Compute the demanded bits in the result that are not
1989 // present in the input.
1990 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
1992 APInt InSignBit = APInt::getSignBit(EBits);
1993 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
1995 // If the sign extended bits are demanded, we know that the sign
1996 // bit is demanded.
1997 InSignBit = InSignBit.zext(BitWidth);
1998 if (NewBits.getBoolValue())
1999 InputDemandedBits |= InSignBit;
2001 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2002 KnownOne &= InputDemandedBits;
2003 KnownZero &= InputDemandedBits;
2004 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
2006 // If the sign bit of the input is known set or clear, then we know the
2007 // top bits of the result.
2008 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear
2009 KnownZero |= NewBits;
2010 KnownOne &= ~NewBits;
2011 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set
2012 KnownOne |= NewBits;
2013 KnownZero &= ~NewBits;
2014 } else { // Input sign bit unknown
2015 KnownZero &= ~NewBits;
2016 KnownOne &= ~NewBits;
2017 }
2018 return;
2019 }
2020 case ISD::CTTZ:
2021 case ISD::CTTZ_ZERO_UNDEF:
2022 case ISD::CTLZ:
2023 case ISD::CTLZ_ZERO_UNDEF:
2024 case ISD::CTPOP: {
2025 unsigned LowBits = Log2_32(BitWidth)+1;
2026 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
2027 KnownOne.clearAllBits();
2028 return;
2029 }
2030 case ISD::LOAD: {
2031 LoadSDNode *LD = cast<LoadSDNode>(Op);
2032 // If this is a ZEXTLoad and we are looking at the loaded value.
2033 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
2034 EVT VT = LD->getMemoryVT();
2035 unsigned MemBits = VT.getScalarType().getSizeInBits();
2036 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
2037 } else if (const MDNode *Ranges = LD->getRanges()) {
2038 computeMaskedBitsLoad(*Ranges, KnownZero);
2039 }
2040 return;
2041 }
2042 case ISD::ZERO_EXTEND: {
2043 EVT InVT = Op.getOperand(0).getValueType();
2044 unsigned InBits = InVT.getScalarType().getSizeInBits();
2045 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2046 KnownZero = KnownZero.trunc(InBits);
2047 KnownOne = KnownOne.trunc(InBits);
2048 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2049 KnownZero = KnownZero.zext(BitWidth);
2050 KnownOne = KnownOne.zext(BitWidth);
2051 KnownZero |= NewBits;
2052 return;
2053 }
2054 case ISD::SIGN_EXTEND: {
2055 EVT InVT = Op.getOperand(0).getValueType();
2056 unsigned InBits = InVT.getScalarType().getSizeInBits();
2057 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2059 KnownZero = KnownZero.trunc(InBits);
2060 KnownOne = KnownOne.trunc(InBits);
2061 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2063 // Note if the sign bit is known to be zero or one.
2064 bool SignBitKnownZero = KnownZero.isNegative();
2065 bool SignBitKnownOne = KnownOne.isNegative();
2066 assert(!(SignBitKnownZero && SignBitKnownOne) &&
2067 "Sign bit can't be known to be both zero and one!");
2069 KnownZero = KnownZero.zext(BitWidth);
2070 KnownOne = KnownOne.zext(BitWidth);
2072 // If the sign bit is known zero or one, the top bits match.
2073 if (SignBitKnownZero)
2074 KnownZero |= NewBits;
2075 else if (SignBitKnownOne)
2076 KnownOne |= NewBits;
2077 return;
2078 }
2079 case ISD::ANY_EXTEND: {
2080 EVT InVT = Op.getOperand(0).getValueType();
2081 unsigned InBits = InVT.getScalarType().getSizeInBits();
2082 KnownZero = KnownZero.trunc(InBits);
2083 KnownOne = KnownOne.trunc(InBits);
2084 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2085 KnownZero = KnownZero.zext(BitWidth);
2086 KnownOne = KnownOne.zext(BitWidth);
2087 return;
2088 }
2089 case ISD::TRUNCATE: {
2090 EVT InVT = Op.getOperand(0).getValueType();
2091 unsigned InBits = InVT.getScalarType().getSizeInBits();
2092 KnownZero = KnownZero.zext(InBits);
2093 KnownOne = KnownOne.zext(InBits);
2094 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2095 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
2096 KnownZero = KnownZero.trunc(BitWidth);
2097 KnownOne = KnownOne.trunc(BitWidth);
2098 break;
2099 }
2100 case ISD::AssertZext: {
2101 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2102 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
2103 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2104 KnownZero |= (~InMask);
2105 KnownOne &= (~KnownZero);
2106 return;
2107 }
2108 case ISD::FGETSIGN:
2109 // All bits are zero except the low bit.
2110 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2111 return;
2113 case ISD::SUB: {
2114 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
2115 // We know that the top bits of C-X are clear if X contains less bits
2116 // than C (i.e. no wrap-around can happen). For example, 20-X is
2117 // positive if we can prove that X is >= 0 and < 16.
2118 if (CLHS->getAPIntValue().isNonNegative()) {
2119 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2120 // NLZ can't be BitWidth with no sign bit
2121 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
2122 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2124 // If all of the MaskV bits are known to be zero, then we know the
2125 // output top bits are zero, because we now know that the output is
2126 // from [0-C].
2127 if ((KnownZero2 & MaskV) == MaskV) {
2128 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2129 // Top bits known zero.
2130 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
2131 }
2132 }
2133 }
2134 }
2135 // fall through
2136 case ISD::ADD:
2137 case ISD::ADDE: {
2138 // Output known-0 bits are known if clear or set in both the low clear bits
2139 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
2140 // low 3 bits clear.
2141 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2142 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
2143 unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
2145 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2146 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
2147 KnownZeroOut = std::min(KnownZeroOut,
2148 KnownZero2.countTrailingOnes());
2150 if (Op.getOpcode() == ISD::ADD) {
2151 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
2152 return;
2153 }
2155 // With ADDE, a carry bit may be added in, so we can only use this
2156 // information if we know (at least) that the low two bits are clear. We
2157 // then return to the caller that the low bit is unknown but that other bits
2158 // are known zero.
2159 if (KnownZeroOut >= 2) // ADDE
2160 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
2161 return;
2162 }
2163 case ISD::SREM:
2164 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2165 const APInt &RA = Rem->getAPIntValue().abs();
2166 if (RA.isPowerOf2()) {
2167 APInt LowBits = RA - 1;
2168 ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
2170 // The low bits of the first operand are unchanged by the srem.
2171 KnownZero = KnownZero2 & LowBits;
2172 KnownOne = KnownOne2 & LowBits;
2174 // If the first operand is non-negative or has all low bits zero, then
2175 // the upper bits are all zero.
2176 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
2177 KnownZero |= ~LowBits;
2179 // If the first operand is negative and not all low bits are zero, then
2180 // the upper bits are all one.
2181 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
2182 KnownOne |= ~LowBits;
2183 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
2184 }
2185 }
2186 return;
2187 case ISD::UREM: {
2188 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2189 const APInt &RA = Rem->getAPIntValue();
2190 if (RA.isPowerOf2()) {
2191 APInt LowBits = (RA - 1);
2192 KnownZero |= ~LowBits;
2193 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1);
2194 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
2195 break;
2196 }
2197 }
2199 // Since the result is less than or equal to either operand, any leading
2200 // zero bits in either operand must also exist in the result.
2201 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2202 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2204 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
2205 KnownZero2.countLeadingOnes());
2206 KnownOne.clearAllBits();
2207 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
2208 return;
2209 }
2210 case ISD::FrameIndex:
2211 case ISD::TargetFrameIndex:
2212 if (unsigned Align = InferPtrAlignment(Op)) {
2213 // The low bits are known zero if the pointer is aligned.
2214 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
2215 return;
2216 }
2217 break;
2219 default:
2220 if (Op.getOpcode() < ISD::BUILTIN_OP_END)
2221 break;
2222 // Fallthrough
2223 case ISD::INTRINSIC_WO_CHAIN:
2224 case ISD::INTRINSIC_W_CHAIN:
2225 case ISD::INTRINSIC_VOID:
2226 // Allow the target to implement this method for its nodes.
2227 TLI->computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
2228 return;
2229 }
2230 }
2232 /// ComputeNumSignBits - Return the number of times the sign bit of the
2233 /// register is replicated into the other bits. We know that at least 1 bit
2234 /// is always equal to the sign bit (itself), but other cases can give us
2235 /// information. For example, immediately after an "SRA X, 2", we know that
2236 /// the top 3 bits are all equal to each other, so we return 3.
2237 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
2238 const TargetLowering *TLI = TM.getTargetLowering();
2239 EVT VT = Op.getValueType();
2240 assert(VT.isInteger() && "Invalid VT!");
2241 unsigned VTBits = VT.getScalarType().getSizeInBits();
2242 unsigned Tmp, Tmp2;
2243 unsigned FirstAnswer = 1;
2245 if (Depth == 6)
2246 return 1; // Limit search depth.
2248 switch (Op.getOpcode()) {
2249 default: break;
2250 case ISD::AssertSext:
2251 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2252 return VTBits-Tmp+1;
2253 case ISD::AssertZext:
2254 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2255 return VTBits-Tmp;
2257 case ISD::Constant: {
2258 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
2259 return Val.getNumSignBits();
2260 }
2262 case ISD::SIGN_EXTEND:
2263 Tmp =
2264 VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
2265 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
2267 case ISD::SIGN_EXTEND_INREG:
2268 // Max of the input and what this extends.
2269 Tmp =
2270 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
2271 Tmp = VTBits-Tmp+1;
2273 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2274 return std::max(Tmp, Tmp2);
2276 case ISD::SRA:
2277 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2278 // SRA X, C -> adds C sign bits.
2279 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2280 Tmp += C->getZExtValue();
2281 if (Tmp > VTBits) Tmp = VTBits;
2282 }
2283 return Tmp;
2284 case ISD::SHL:
2285 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2286 // shl destroys sign bits.
2287 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2288 if (C->getZExtValue() >= VTBits || // Bad shift.
2289 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out.
2290 return Tmp - C->getZExtValue();
2291 }
2292 break;
2293 case ISD::AND:
2294 case ISD::OR:
2295 case ISD::XOR: // NOT is handled here.
2296 // Logical binary ops preserve the number of sign bits at the worst.
2297 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2298 if (Tmp != 1) {
2299 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2300 FirstAnswer = std::min(Tmp, Tmp2);
2301 // We computed what we know about the sign bits as our first
2302 // answer. Now proceed to the generic code that uses
2303 // ComputeMaskedBits, and pick whichever answer is better.
2304 }
2305 break;
2307 case ISD::SELECT:
2308 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2309 if (Tmp == 1) return 1; // Early out.
2310 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
2311 return std::min(Tmp, Tmp2);
2313 case ISD::SADDO:
2314 case ISD::UADDO:
2315 case ISD::SSUBO:
2316 case ISD::USUBO:
2317 case ISD::SMULO:
2318 case ISD::UMULO:
2319 if (Op.getResNo() != 1)
2320 break;
2321 // The boolean result conforms to getBooleanContents. Fall through.
2322 case ISD::SETCC:
2323 // If setcc returns 0/-1, all bits are sign bits.
2324 if (TLI->getBooleanContents(Op.getValueType().isVector()) ==
2325 TargetLowering::ZeroOrNegativeOneBooleanContent)
2326 return VTBits;
2327 break;
2328 case ISD::ROTL:
2329 case ISD::ROTR:
2330 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2331 unsigned RotAmt = C->getZExtValue() & (VTBits-1);
2333 // Handle rotate right by N like a rotate left by 32-N.
2334 if (Op.getOpcode() == ISD::ROTR)
2335 RotAmt = (VTBits-RotAmt) & (VTBits-1);
2337 // If we aren't rotating out all of the known-in sign bits, return the
2338 // number that are left. This handles rotl(sext(x), 1) for example.
2339 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2340 if (Tmp > RotAmt+1) return Tmp-RotAmt;
2341 }
2342 break;
2343 case ISD::ADD:
2344 // Add can have at most one carry bit. Thus we know that the output
2345 // is, at worst, one more bit than the inputs.
2346 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2347 if (Tmp == 1) return 1; // Early out.
2349 // Special case decrementing a value (ADD X, -1):
2350 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2351 if (CRHS->isAllOnesValue()) {
2352 APInt KnownZero, KnownOne;
2353 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2355 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2356 // sign bits set.
2357 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2358 return VTBits;
2360 // If we are subtracting one from a positive number, there is no carry
2361 // out of the result.
2362 if (KnownZero.isNegative())
2363 return Tmp;
2364 }
2366 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2367 if (Tmp2 == 1) return 1;
2368 return std::min(Tmp, Tmp2)-1;
2370 case ISD::SUB:
2371 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2372 if (Tmp2 == 1) return 1;
2374 // Handle NEG.
2375 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
2376 if (CLHS->isNullValue()) {
2377 APInt KnownZero, KnownOne;
2378 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2379 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2380 // sign bits set.
2381 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2382 return VTBits;
2384 // If the input is known to be positive (the sign bit is known clear),
2385 // the output of the NEG has the same number of sign bits as the input.
2386 if (KnownZero.isNegative())
2387 return Tmp2;
2389 // Otherwise, we treat this like a SUB.
2390 }
2392 // Sub can have at most one carry bit. Thus we know that the output
2393 // is, at worst, one more bit than the inputs.
2394 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2395 if (Tmp == 1) return 1; // Early out.
2396 return std::min(Tmp, Tmp2)-1;
2397 case ISD::TRUNCATE:
2398 // FIXME: it's tricky to do anything useful for this, but it is an important
2399 // case for targets like X86.
2400 break;
2401 }
2403 // If we are looking at the loaded value of the SDNode.
2404 if (Op.getResNo() == 0) {
2405 // Handle LOADX separately here. EXTLOAD case will fallthrough.
2406 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
2407 unsigned ExtType = LD->getExtensionType();
2408 switch (ExtType) {
2409 default: break;
2410 case ISD::SEXTLOAD: // '17' bits known
2411 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2412 return VTBits-Tmp+1;
2413 case ISD::ZEXTLOAD: // '16' bits known
2414 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2415 return VTBits-Tmp;
2416 }
2417 }
2418 }
2420 // Allow the target to implement this method for its nodes.
2421 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2422 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2423 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2424 Op.getOpcode() == ISD::INTRINSIC_VOID) {
2425 unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
2426 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
2427 }
2429 // Finally, if we can prove that the top bits of the result are 0's or 1's,
2430 // use this information.
2431 APInt KnownZero, KnownOne;
2432 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth);
2434 APInt Mask;
2435 if (KnownZero.isNegative()) { // sign bit is 0
2436 Mask = KnownZero;
2437 } else if (KnownOne.isNegative()) { // sign bit is 1;
2438 Mask = KnownOne;
2439 } else {
2440 // Nothing known.
2441 return FirstAnswer;
2442 }
2444 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
2445 // the number of identical bits in the top of the input value.
2446 Mask = ~Mask;
2447 Mask <<= Mask.getBitWidth()-VTBits;
2448 // Return # leading zeros. We use 'min' here in case Val was zero before
2449 // shifting. We don't want to return '64' as for an i32 "0".
2450 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
2451 }
2453 /// isBaseWithConstantOffset - Return true if the specified operand is an
2454 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
2455 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
2456 /// semantics as an ADD. This handles the equivalence:
2457 /// X|Cst == X+Cst iff X&Cst = 0.
2458 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
2459 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
2460 !isa<ConstantSDNode>(Op.getOperand(1)))
2461 return false;
2463 if (Op.getOpcode() == ISD::OR &&
2464 !MaskedValueIsZero(Op.getOperand(0),
2465 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
2466 return false;
2468 return true;
2469 }
2472 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
2473 // If we're told that NaNs won't happen, assume they won't.
2474 if (getTarget().Options.NoNaNsFPMath)
2475 return true;
2477 // If the value is a constant, we can obviously see if it is a NaN or not.
2478 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2479 return !C->getValueAPF().isNaN();
2481 // TODO: Recognize more cases here.
2483 return false;
2484 }
2486 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
2487 // If the value is a constant, we can obviously see if it is a zero or not.
2488 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2489 return !C->isZero();
2491 // TODO: Recognize more cases here.
2492 switch (Op.getOpcode()) {
2493 default: break;
2494 case ISD::OR:
2495 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2496 return !C->isNullValue();
2497 break;
2498 }
2500 return false;
2501 }
2503 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
2504 // Check the obvious case.
2505 if (A == B) return true;
2507 // For for negative and positive zero.
2508 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
2509 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
2510 if (CA->isZero() && CB->isZero()) return true;
2512 // Otherwise they may not be equal.
2513 return false;
2514 }
2516 /// getNode - Gets or creates the specified node.
2517 ///
2518 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
2519 FoldingSetNodeID ID;
2520 AddNodeIDNode(ID, Opcode, getVTList(VT), nullptr, 0);
2521 void *IP = nullptr;
2522 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
2523 return SDValue(E, 0);
2525 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
2526 DL.getDebugLoc(), getVTList(VT));
2527 CSEMap.InsertNode(N, IP);
2529 AllNodes.push_back(N);
2530 #ifndef NDEBUG
2531 VerifySDNode(N);
2532 #endif
2533 return SDValue(N, 0);
2534 }
2536 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
2537 EVT VT, SDValue Operand) {
2538 // Constant fold unary operations with an integer constant operand. Even
2539 // opaque constant will be folded, because the folding of unary operations
2540 // doesn't create new constants with different values. Nevertheless, the
2541 // opaque flag is preserved during folding to prevent future folding with
2542 // other constants.
2543 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
2544 const APInt &Val = C->getAPIntValue();
2545 switch (Opcode) {
2546 default: break;
2547 case ISD::SIGN_EXTEND:
2548 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
2549 C->isTargetOpcode(), C->isOpaque());
2550 case ISD::ANY_EXTEND:
2551 case ISD::ZERO_EXTEND:
2552 case ISD::TRUNCATE:
2553 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
2554 C->isTargetOpcode(), C->isOpaque());
2555 case ISD::UINT_TO_FP:
2556 case ISD::SINT_TO_FP: {
2557 APFloat apf(EVTToAPFloatSemantics(VT),
2558 APInt::getNullValue(VT.getSizeInBits()));
2559 (void)apf.convertFromAPInt(Val,
2560 Opcode==ISD::SINT_TO_FP,
2561 APFloat::rmNearestTiesToEven);
2562 return getConstantFP(apf, VT);
2563 }
2564 case ISD::BITCAST:
2565 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
2566 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
2567 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
2568 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
2569 break;
2570 case ISD::BSWAP:
2571 return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
2572 C->isOpaque());
2573 case ISD::CTPOP:
2574 return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
2575 C->isOpaque());
2576 case ISD::CTLZ:
2577 case ISD::CTLZ_ZERO_UNDEF:
2578 return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
2579 C->isOpaque());
2580 case ISD::CTTZ:
2581 case ISD::CTTZ_ZERO_UNDEF:
2582 return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
2583 C->isOpaque());
2584 }
2585 }
2587 // Constant fold unary operations with a floating point constant operand.
2588 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
2589 APFloat V = C->getValueAPF(); // make copy
2590 switch (Opcode) {
2591 case ISD::FNEG:
2592 V.changeSign();
2593 return getConstantFP(V, VT);
2594 case ISD::FABS:
2595 V.clearSign();
2596 return getConstantFP(V, VT);
2597 case ISD::FCEIL: {
2598 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
2599 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2600 return getConstantFP(V, VT);
2601 break;
2602 }
2603 case ISD::FTRUNC: {
2604 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
2605 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2606 return getConstantFP(V, VT);
2607 break;
2608 }
2609 case ISD::FFLOOR: {
2610 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
2611 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2612 return getConstantFP(V, VT);
2613 break;
2614 }
2615 case ISD::FP_EXTEND: {
2616 bool ignored;
2617 // This can return overflow, underflow, or inexact; we don't care.
2618 // FIXME need to be more flexible about rounding mode.
2619 (void)V.convert(EVTToAPFloatSemantics(VT),
2620 APFloat::rmNearestTiesToEven, &ignored);
2621 return getConstantFP(V, VT);
2622 }
2623 case ISD::FP_TO_SINT:
2624 case ISD::FP_TO_UINT: {
2625 integerPart x[2];
2626 bool ignored;
2627 assert(integerPartWidth >= 64);
2628 // FIXME need to be more flexible about rounding mode.
2629 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
2630 Opcode==ISD::FP_TO_SINT,
2631 APFloat::rmTowardZero, &ignored);
2632 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual
2633 break;
2634 APInt api(VT.getSizeInBits(), x);
2635 return getConstant(api, VT);
2636 }
2637 case ISD::BITCAST:
2638 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
2639 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
2640 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
2641 return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
2642 break;
2643 }
2644 }
2646 unsigned OpOpcode = Operand.getNode()->getOpcode();
2647 switch (Opcode) {
2648 case ISD::TokenFactor:
2649 case ISD::MERGE_VALUES:
2650 case ISD::CONCAT_VECTORS:
2651 return Operand; // Factor, merge or concat of one node? No need.
2652 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
2653 case ISD::FP_EXTEND:
2654 assert(VT.isFloatingPoint() &&
2655 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
2656 if (Operand.getValueType() == VT) return Operand; // noop conversion.
2657 assert((!VT.isVector() ||
2658 VT.getVectorNumElements() ==
2659 Operand.getValueType().getVectorNumElements()) &&
2660 "Vector element count mismatch!");
2661 if (Operand.getOpcode() == ISD::UNDEF)
2662 return getUNDEF(VT);
2663 break;
2664 case ISD::SIGN_EXTEND:
2665 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2666 "Invalid SIGN_EXTEND!");
2667 if (Operand.getValueType() == VT) return Operand; // noop extension
2668 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2669 "Invalid sext node, dst < src!");
2670 assert((!VT.isVector() ||
2671 VT.getVectorNumElements() ==
2672 Operand.getValueType().getVectorNumElements()) &&
2673 "Vector element count mismatch!");
2674 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
2675 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2676 else if (OpOpcode == ISD::UNDEF)
2677 // sext(undef) = 0, because the top bits will all be the same.
2678 return getConstant(0, VT);
2679 break;
2680 case ISD::ZERO_EXTEND:
2681 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2682 "Invalid ZERO_EXTEND!");
2683 if (Operand.getValueType() == VT) return Operand; // noop extension
2684 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2685 "Invalid zext node, dst < src!");
2686 assert((!VT.isVector() ||
2687 VT.getVectorNumElements() ==
2688 Operand.getValueType().getVectorNumElements()) &&
2689 "Vector element count mismatch!");
2690 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
2691 return getNode(ISD::ZERO_EXTEND, DL, VT,
2692 Operand.getNode()->getOperand(0));
2693 else if (OpOpcode == ISD::UNDEF)
2694 // zext(undef) = 0, because the top bits will be zero.
2695 return getConstant(0, VT);
2696 break;
2697 case ISD::ANY_EXTEND:
2698 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2699 "Invalid ANY_EXTEND!");
2700 if (Operand.getValueType() == VT) return Operand; // noop extension
2701 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
2702 "Invalid anyext node, dst < src!");
2703 assert((!VT.isVector() ||
2704 VT.getVectorNumElements() ==
2705 Operand.getValueType().getVectorNumElements()) &&
2706 "Vector element count mismatch!");
2708 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
2709 OpOpcode == ISD::ANY_EXTEND)
2710 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
2711 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2712 else if (OpOpcode == ISD::UNDEF)
2713 return getUNDEF(VT);
2715 // (ext (trunx x)) -> x
2716 if (OpOpcode == ISD::TRUNCATE) {
2717 SDValue OpOp = Operand.getNode()->getOperand(0);
2718 if (OpOp.getValueType() == VT)
2719 return OpOp;
2720 }
2721 break;
2722 case ISD::TRUNCATE:
2723 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
2724 "Invalid TRUNCATE!");
2725 if (Operand.getValueType() == VT) return Operand; // noop truncate
2726 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
2727 "Invalid truncate node, src < dst!");
2728 assert((!VT.isVector() ||
2729 VT.getVectorNumElements() ==
2730 Operand.getValueType().getVectorNumElements()) &&
2731 "Vector element count mismatch!");
2732 if (OpOpcode == ISD::TRUNCATE)
2733 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
2734 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
2735 OpOpcode == ISD::ANY_EXTEND) {
2736 // If the source is smaller than the dest, we still need an extend.
2737 if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
2738 .bitsLT(VT.getScalarType()))
2739 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
2740 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
2741 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
2742 return Operand.getNode()->getOperand(0);
2743 }
2744 if (OpOpcode == ISD::UNDEF)
2745 return getUNDEF(VT);
2746 break;
2747 case ISD::BITCAST:
2748 // Basic sanity checking.
2749 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
2750 && "Cannot BITCAST between types of different sizes!");
2751 if (VT == Operand.getValueType()) return Operand; // noop conversion.
2752 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
2753 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
2754 if (OpOpcode == ISD::UNDEF)
2755 return getUNDEF(VT);
2756 break;
2757 case ISD::SCALAR_TO_VECTOR:
2758 assert(VT.isVector() && !Operand.getValueType().isVector() &&
2759 (VT.getVectorElementType() == Operand.getValueType() ||
2760 (VT.getVectorElementType().isInteger() &&
2761 Operand.getValueType().isInteger() &&
2762 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
2763 "Illegal SCALAR_TO_VECTOR node!");
2764 if (OpOpcode == ISD::UNDEF)
2765 return getUNDEF(VT);
2766 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
2767 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
2768 isa<ConstantSDNode>(Operand.getOperand(1)) &&
2769 Operand.getConstantOperandVal(1) == 0 &&
2770 Operand.getOperand(0).getValueType() == VT)
2771 return Operand.getOperand(0);
2772 break;
2773 case ISD::FNEG:
2774 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
2775 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
2776 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
2777 Operand.getNode()->getOperand(0));
2778 if (OpOpcode == ISD::FNEG) // --X -> X
2779 return Operand.getNode()->getOperand(0);
2780 break;
2781 case ISD::FABS:
2782 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
2783 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
2784 break;
2785 }
2787 SDNode *N;
2788 SDVTList VTs = getVTList(VT);
2789 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
2790 FoldingSetNodeID ID;
2791 SDValue Ops[1] = { Operand };
2792 AddNodeIDNode(ID, Opcode, VTs, Ops, 1);
2793 void *IP = nullptr;
2794 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
2795 return SDValue(E, 0);
2797 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
2798 DL.getDebugLoc(), VTs, Operand);
2799 CSEMap.InsertNode(N, IP);
2800 } else {
2801 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
2802 DL.getDebugLoc(), VTs, Operand);
2803 }
2805 AllNodes.push_back(N);
2806 #ifndef NDEBUG
2807 VerifySDNode(N);
2808 #endif
2809 return SDValue(N, 0);
2810 }
2812 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
2813 SDNode *Cst1, SDNode *Cst2) {
2814 // If the opcode is a target-specific ISD node, there's nothing we can
2815 // do here and the operand rules may not line up with the below, so
2816 // bail early.
2817 if (Opcode >= ISD::BUILTIN_OP_END)
2818 return SDValue();
2820 SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
2821 SmallVector<SDValue, 4> Outputs;
2822 EVT SVT = VT.getScalarType();
2824 ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
2825 ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
2826 if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
2827 return SDValue();
2829 if (Scalar1 && Scalar2)
2830 // Scalar instruction.
2831 Inputs.push_back(std::make_pair(Scalar1, Scalar2));
2832 else {
2833 // For vectors extract each constant element into Inputs so we can constant
2834 // fold them individually.
2835 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
2836 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
2837 if (!BV1 || !BV2)
2838 return SDValue();
2840 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
2842 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
2843 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
2844 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
2845 if (!V1 || !V2) // Not a constant, bail.
2846 return SDValue();
2848 if (V1->isOpaque() || V2->isOpaque())
2849 return SDValue();
2851 // Avoid BUILD_VECTOR nodes that perform implicit truncation.
2852 // FIXME: This is valid and could be handled by truncating the APInts.
2853 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
2854 return SDValue();
2856 Inputs.push_back(std::make_pair(V1, V2));
2857 }
2858 }
2860 // We have a number of constant values, constant fold them element by element.
2861 for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
2862 const APInt &C1 = Inputs[I].first->getAPIntValue();
2863 const APInt &C2 = Inputs[I].second->getAPIntValue();
2865 switch (Opcode) {
2866 case ISD::ADD:
2867 Outputs.push_back(getConstant(C1 + C2, SVT));
2868 break;
2869 case ISD::SUB:
2870 Outputs.push_back(getConstant(C1 - C2, SVT));
2871 break;
2872 case ISD::MUL:
2873 Outputs.push_back(getConstant(C1 * C2, SVT));
2874 break;
2875 case ISD::UDIV:
2876 if (!C2.getBoolValue())
2877 return SDValue();
2878 Outputs.push_back(getConstant(C1.udiv(C2), SVT));
2879 break;
2880 case ISD::UREM:
2881 if (!C2.getBoolValue())
2882 return SDValue();
2883 Outputs.push_back(getConstant(C1.urem(C2), SVT));
2884 break;
2885 case ISD::SDIV:
2886 if (!C2.getBoolValue())
2887 return SDValue();
2888 Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
2889 break;
2890 case ISD::SREM:
2891 if (!C2.getBoolValue())
2892 return SDValue();
2893 Outputs.push_back(getConstant(C1.srem(C2), SVT));
2894 break;
2895 case ISD::AND:
2896 Outputs.push_back(getConstant(C1 & C2, SVT));
2897 break;
2898 case ISD::OR:
2899 Outputs.push_back(getConstant(C1 | C2, SVT));
2900 break;
2901 case ISD::XOR:
2902 Outputs.push_back(getConstant(C1 ^ C2, SVT));
2903 break;
2904 case ISD::SHL:
2905 Outputs.push_back(getConstant(C1 << C2, SVT));
2906 break;
2907 case ISD::SRL:
2908 Outputs.push_back(getConstant(C1.lshr(C2), SVT));
2909 break;
2910 case ISD::SRA:
2911 Outputs.push_back(getConstant(C1.ashr(C2), SVT));
2912 break;
2913 case ISD::ROTL:
2914 Outputs.push_back(getConstant(C1.rotl(C2), SVT));
2915 break;
2916 case ISD::ROTR:
2917 Outputs.push_back(getConstant(C1.rotr(C2), SVT));
2918 break;
2919 default:
2920 return SDValue();
2921 }
2922 }
2924 // Handle the scalar case first.
2925 if (Scalar1 && Scalar2)
2926 return Outputs.back();
2928 // Otherwise build a big vector out of the scalar elements we generated.
2929 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(),
2930 Outputs.size());
2931 }
2933 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
2934 SDValue N2) {
2935 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
2936 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
2937 switch (Opcode) {
2938 default: break;
2939 case ISD::TokenFactor:
2940 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
2941 N2.getValueType() == MVT::Other && "Invalid token factor!");
2942 // Fold trivial token factors.
2943 if (N1.getOpcode() == ISD::EntryToken) return N2;
2944 if (N2.getOpcode() == ISD::EntryToken) return N1;
2945 if (N1 == N2) return N1;
2946 break;
2947 case ISD::CONCAT_VECTORS:
2948 // Concat of UNDEFs is UNDEF.
2949 if (N1.getOpcode() == ISD::UNDEF &&
2950 N2.getOpcode() == ISD::UNDEF)
2951 return getUNDEF(VT);
2953 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
2954 // one big BUILD_VECTOR.
2955 if (N1.getOpcode() == ISD::BUILD_VECTOR &&
2956 N2.getOpcode() == ISD::BUILD_VECTOR) {
2957 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
2958 N1.getNode()->op_end());
2959 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
2960 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size());
2961 }
2962 break;
2963 case ISD::AND:
2964 assert(VT.isInteger() && "This operator does not apply to FP types!");
2965 assert(N1.getValueType() == N2.getValueType() &&
2966 N1.getValueType() == VT && "Binary operator types must match!");
2967 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
2968 // worth handling here.
2969 if (N2C && N2C->isNullValue())
2970 return N2;
2971 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
2972 return N1;
2973 break;
2974 case ISD::OR:
2975 case ISD::XOR:
2976 case ISD::ADD:
2977 case ISD::SUB:
2978 assert(VT.isInteger() && "This operator does not apply to FP types!");
2979 assert(N1.getValueType() == N2.getValueType() &&
2980 N1.getValueType() == VT && "Binary operator types must match!");
2981 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
2982 // it's worth handling here.
2983 if (N2C && N2C->isNullValue())
2984 return N1;
2985 break;
2986 case ISD::UDIV:
2987 case ISD::UREM:
2988 case ISD::MULHU:
2989 case ISD::MULHS:
2990 case ISD::MUL:
2991 case ISD::SDIV:
2992 case ISD::SREM:
2993 assert(VT.isInteger() && "This operator does not apply to FP types!");
2994 assert(N1.getValueType() == N2.getValueType() &&
2995 N1.getValueType() == VT && "Binary operator types must match!");
2996 break;
2997 case ISD::FADD:
2998 case ISD::FSUB:
2999 case ISD::FMUL:
3000 case ISD::FDIV:
3001 case ISD::FREM:
3002 if (getTarget().Options.UnsafeFPMath) {
3003 if (Opcode == ISD::FADD) {
3004 // 0+x --> x
3005 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
3006 if (CFP->getValueAPF().isZero())
3007 return N2;
3008 // x+0 --> x
3009 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
3010 if (CFP->getValueAPF().isZero())
3011 return N1;
3012 } else if (Opcode == ISD::FSUB) {
3013 // x-0 --> x
3014 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
3015 if (CFP->getValueAPF().isZero())
3016 return N1;
3017 } else if (Opcode == ISD::FMUL) {
3018 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
3019 SDValue V = N2;
3021 // If the first operand isn't the constant, try the second
3022 if (!CFP) {
3023 CFP = dyn_cast<ConstantFPSDNode>(N2);
3024 V = N1;
3025 }
3027 if (CFP) {
3028 // 0*x --> 0
3029 if (CFP->isZero())
3030 return SDValue(CFP,0);
3031 // 1*x --> x
3032 if (CFP->isExactlyValue(1.0))
3033 return V;
3034 }
3035 }
3036 }
3037 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
3038 assert(N1.getValueType() == N2.getValueType() &&
3039 N1.getValueType() == VT && "Binary operator types must match!");
3040 break;
3041 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
3042 assert(N1.getValueType() == VT &&
3043 N1.getValueType().isFloatingPoint() &&
3044 N2.getValueType().isFloatingPoint() &&
3045 "Invalid FCOPYSIGN!");
3046 break;
3047 case ISD::SHL:
3048 case ISD::SRA:
3049 case ISD::SRL:
3050 case ISD::ROTL:
3051 case ISD::ROTR:
3052 assert(VT == N1.getValueType() &&
3053 "Shift operators return type must be the same as their first arg");
3054 assert(VT.isInteger() && N2.getValueType().isInteger() &&
3055 "Shifts only work on integers");
3056 assert((!VT.isVector() || VT == N2.getValueType()) &&
3057 "Vector shift amounts must be in the same as their first arg");
3058 // Verify that the shift amount VT is bit enough to hold valid shift
3059 // amounts. This catches things like trying to shift an i1024 value by an
3060 // i8, which is easy to fall into in generic code that uses
3061 // TLI.getShiftAmount().
3062 assert(N2.getValueType().getSizeInBits() >=
3063 Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
3064 "Invalid use of small shift amount with oversized value!");
3066 // Always fold shifts of i1 values so the code generator doesn't need to
3067 // handle them. Since we know the size of the shift has to be less than the
3068 // size of the value, the shift/rotate count is guaranteed to be zero.
3069 if (VT == MVT::i1)
3070 return N1;
3071 if (N2C && N2C->isNullValue())
3072 return N1;
3073 break;
3074 case ISD::FP_ROUND_INREG: {
3075 EVT EVT = cast<VTSDNode>(N2)->getVT();
3076 assert(VT == N1.getValueType() && "Not an inreg round!");
3077 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
3078 "Cannot FP_ROUND_INREG integer types");
3079 assert(EVT.isVector() == VT.isVector() &&
3080 "FP_ROUND_INREG type should be vector iff the operand "
3081 "type is vector!");
3082 assert((!EVT.isVector() ||
3083 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3084 "Vector element counts must match in FP_ROUND_INREG");
3085 assert(EVT.bitsLE(VT) && "Not rounding down!");
3086 (void)EVT;
3087 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
3088 break;
3089 }
3090 case ISD::FP_ROUND:
3091 assert(VT.isFloatingPoint() &&
3092 N1.getValueType().isFloatingPoint() &&
3093 VT.bitsLE(N1.getValueType()) &&
3094 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
3095 if (N1.getValueType() == VT) return N1; // noop conversion.
3096 break;
3097 case ISD::AssertSext:
3098 case ISD::AssertZext: {
3099 EVT EVT = cast<VTSDNode>(N2)->getVT();
3100 assert(VT == N1.getValueType() && "Not an inreg extend!");
3101 assert(VT.isInteger() && EVT.isInteger() &&
3102 "Cannot *_EXTEND_INREG FP types");
3103 assert(!EVT.isVector() &&
3104 "AssertSExt/AssertZExt type should be the vector element type "
3105 "rather than the vector type!");
3106 assert(EVT.bitsLE(VT) && "Not extending!");
3107 if (VT == EVT) return N1; // noop assertion.
3108 break;
3109 }
3110 case ISD::SIGN_EXTEND_INREG: {
3111 EVT EVT = cast<VTSDNode>(N2)->getVT();
3112 assert(VT == N1.getValueType() && "Not an inreg extend!");
3113 assert(VT.isInteger() && EVT.isInteger() &&
3114 "Cannot *_EXTEND_INREG FP types");
3115 assert(EVT.isVector() == VT.isVector() &&
3116 "SIGN_EXTEND_INREG type should be vector iff the operand "
3117 "type is vector!");
3118 assert((!EVT.isVector() ||
3119 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3120 "Vector element counts must match in SIGN_EXTEND_INREG");
3121 assert(EVT.bitsLE(VT) && "Not extending!");
3122 if (EVT == VT) return N1; // Not actually extending
3124 if (N1C) {
3125 APInt Val = N1C->getAPIntValue();
3126 unsigned FromBits = EVT.getScalarType().getSizeInBits();
3127 Val <<= Val.getBitWidth()-FromBits;
3128 Val = Val.ashr(Val.getBitWidth()-FromBits);
3129 return getConstant(Val, VT);
3130 }
3131 break;
3132 }
3133 case ISD::EXTRACT_VECTOR_ELT:
3134 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
3135 if (N1.getOpcode() == ISD::UNDEF)
3136 return getUNDEF(VT);
3138 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
3139 // expanding copies of large vectors from registers.
3140 if (N2C &&
3141 N1.getOpcode() == ISD::CONCAT_VECTORS &&
3142 N1.getNumOperands() > 0) {
3143 unsigned Factor =
3144 N1.getOperand(0).getValueType().getVectorNumElements();
3145 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
3146 N1.getOperand(N2C->getZExtValue() / Factor),
3147 getConstant(N2C->getZExtValue() % Factor,
3148 N2.getValueType()));
3149 }
3151 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
3152 // expanding large vector constants.
3153 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
3154 SDValue Elt = N1.getOperand(N2C->getZExtValue());
3156 if (VT != Elt.getValueType())
3157 // If the vector element type is not legal, the BUILD_VECTOR operands
3158 // are promoted and implicitly truncated, and the result implicitly
3159 // extended. Make that explicit here.
3160 Elt = getAnyExtOrTrunc(Elt, DL, VT);
3162 return Elt;
3163 }
3165 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
3166 // operations are lowered to scalars.
3167 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
3168 // If the indices are the same, return the inserted element else
3169 // if the indices are known different, extract the element from
3170 // the original vector.
3171 SDValue N1Op2 = N1.getOperand(2);
3172 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
3174 if (N1Op2C && N2C) {
3175 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
3176 if (VT == N1.getOperand(1).getValueType())
3177 return N1.getOperand(1);
3178 else
3179 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
3180 }
3182 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
3183 }
3184 }
3185 break;
3186 case ISD::EXTRACT_ELEMENT:
3187 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
3188 assert(!N1.getValueType().isVector() && !VT.isVector() &&
3189 (N1.getValueType().isInteger() == VT.isInteger()) &&
3190 N1.getValueType() != VT &&
3191 "Wrong types for EXTRACT_ELEMENT!");
3193 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
3194 // 64-bit integers into 32-bit parts. Instead of building the extract of
3195 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
3196 if (N1.getOpcode() == ISD::BUILD_PAIR)
3197 return N1.getOperand(N2C->getZExtValue());
3199 // EXTRACT_ELEMENT of a constant int is also very common.
3200 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
3201 unsigned ElementSize = VT.getSizeInBits();
3202 unsigned Shift = ElementSize * N2C->getZExtValue();
3203 APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
3204 return getConstant(ShiftedVal.trunc(ElementSize), VT);
3205 }
3206 break;
3207 case ISD::EXTRACT_SUBVECTOR: {
3208 SDValue Index = N2;
3209 if (VT.isSimple() && N1.getValueType().isSimple()) {
3210 assert(VT.isVector() && N1.getValueType().isVector() &&
3211 "Extract subvector VTs must be a vectors!");
3212 assert(VT.getVectorElementType() ==
3213 N1.getValueType().getVectorElementType() &&
3214 "Extract subvector VTs must have the same element type!");
3215 assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
3216 "Extract subvector must be from larger vector to smaller vector!");
3218 if (isa<ConstantSDNode>(Index.getNode())) {
3219 assert((VT.getVectorNumElements() +
3220 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
3221 <= N1.getValueType().getVectorNumElements())
3222 && "Extract subvector overflow!");
3223 }
3225 // Trivial extraction.
3226 if (VT.getSimpleVT() == N1.getSimpleValueType())
3227 return N1;
3228 }
3229 break;
3230 }
3231 }
3233 // Perform trivial constant folding.
3234 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
3235 if (SV.getNode()) return SV;
3237 // Canonicalize constant to RHS if commutative.
3238 if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
3239 std::swap(N1C, N2C);
3240 std::swap(N1, N2);
3241 }
3243 // Constant fold FP operations.
3244 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
3245 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
3246 if (N1CFP) {
3247 if (!N2CFP && isCommutativeBinOp(Opcode)) {
3248 // Canonicalize constant to RHS if commutative.
3249 std::swap(N1CFP, N2CFP);
3250 std::swap(N1, N2);
3251 } else if (N2CFP) {
3252 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
3253 APFloat::opStatus s;
3254 switch (Opcode) {
3255 case ISD::FADD:
3256 s = V1.add(V2, APFloat::rmNearestTiesToEven);
3257 if (s != APFloat::opInvalidOp)
3258 return getConstantFP(V1, VT);
3259 break;
3260 case ISD::FSUB:
3261 s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
3262 if (s!=APFloat::opInvalidOp)
3263 return getConstantFP(V1, VT);
3264 break;
3265 case ISD::FMUL:
3266 s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
3267 if (s!=APFloat::opInvalidOp)
3268 return getConstantFP(V1, VT);
3269 break;
3270 case ISD::FDIV:
3271 s = V1.divide(V2, APFloat::rmNearestTiesToEven);
3272 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero)
3273 return getConstantFP(V1, VT);
3274 break;
3275 case ISD::FREM :
3276 s = V1.mod(V2, APFloat::rmNearestTiesToEven);
3277 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero)
3278 return getConstantFP(V1, VT);
3279 break;
3280 case ISD::FCOPYSIGN:
3281 V1.copySign(V2);
3282 return getConstantFP(V1, VT);
3283 default: break;
3284 }
3285 }
3287 if (Opcode == ISD::FP_ROUND) {
3288 APFloat V = N1CFP->getValueAPF(); // make copy
3289 bool ignored;
3290 // This can return overflow, underflow, or inexact; we don't care.
3291 // FIXME need to be more flexible about rounding mode.
3292 (void)V.convert(EVTToAPFloatSemantics(VT),
3293 APFloat::rmNearestTiesToEven, &ignored);
3294 return getConstantFP(V, VT);
3295 }
3296 }
3298 // Canonicalize an UNDEF to the RHS, even over a constant.
3299 if (N1.getOpcode() == ISD::UNDEF) {
3300 if (isCommutativeBinOp(Opcode)) {
3301 std::swap(N1, N2);
3302 } else {
3303 switch (Opcode) {
3304 case ISD::FP_ROUND_INREG:
3305 case ISD::SIGN_EXTEND_INREG:
3306 case ISD::SUB:
3307 case ISD::FSUB:
3308 case ISD::FDIV:
3309 case ISD::FREM:
3310 case ISD::SRA:
3311 return N1; // fold op(undef, arg2) -> undef
3312 case ISD::UDIV:
3313 case ISD::SDIV:
3314 case ISD::UREM:
3315 case ISD::SREM:
3316 case ISD::SRL:
3317 case ISD::SHL:
3318 if (!VT.isVector())
3319 return getConstant(0, VT); // fold op(undef, arg2) -> 0
3320 // For vectors, we can't easily build an all zero vector, just return
3321 // the LHS.
3322 return N2;
3323 }
3324 }
3325 }
3327 // Fold a bunch of operators when the RHS is undef.
3328 if (N2.getOpcode() == ISD::UNDEF) {
3329 switch (Opcode) {
3330 case ISD::XOR:
3331 if (N1.getOpcode() == ISD::UNDEF)
3332 // Handle undef ^ undef -> 0 special case. This is a common
3333 // idiom (misuse).
3334 return getConstant(0, VT);
3335 // fallthrough
3336 case ISD::ADD:
3337 case ISD::ADDC:
3338 case ISD::ADDE:
3339 case ISD::SUB:
3340 case ISD::UDIV:
3341 case ISD::SDIV:
3342 case ISD::UREM:
3343 case ISD::SREM:
3344 return N2; // fold op(arg1, undef) -> undef
3345 case ISD::FADD:
3346 case ISD::FSUB:
3347 case ISD::FMUL:
3348 case ISD::FDIV:
3349 case ISD::FREM:
3350 if (getTarget().Options.UnsafeFPMath)
3351 return N2;
3352 break;
3353 case ISD::MUL:
3354 case ISD::AND:
3355 case ISD::SRL:
3356 case ISD::SHL:
3357 if (!VT.isVector())
3358 return getConstant(0, VT); // fold op(arg1, undef) -> 0
3359 // For vectors, we can't easily build an all zero vector, just return
3360 // the LHS.
3361 return N1;
3362 case ISD::OR:
3363 if (!VT.isVector())
3364 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
3365 // For vectors, we can't easily build an all one vector, just return
3366 // the LHS.
3367 return N1;
3368 case ISD::SRA:
3369 return N1;
3370 }
3371 }
3373 // Memoize this node if possible.
3374 SDNode *N;
3375 SDVTList VTs = getVTList(VT);
3376 if (VT != MVT::Glue) {
3377 SDValue Ops[] = { N1, N2 };
3378 FoldingSetNodeID ID;
3379 AddNodeIDNode(ID, Opcode, VTs, Ops, 2);
3380 void *IP = nullptr;
3381 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3382 return SDValue(E, 0);
3384 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
3385 DL.getDebugLoc(), VTs, N1, N2);
3386 CSEMap.InsertNode(N, IP);
3387 } else {
3388 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
3389 DL.getDebugLoc(), VTs, N1, N2);
3390 }
3392 AllNodes.push_back(N);
3393 #ifndef NDEBUG
3394 VerifySDNode(N);
3395 #endif
3396 return SDValue(N, 0);
3397 }
3399 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3400 SDValue N1, SDValue N2, SDValue N3) {
3401 // Perform various simplifications.
3402 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
3403 switch (Opcode) {
3404 case ISD::FMA: {
3405 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
3406 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
3407 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
3408 if (N1CFP && N2CFP && N3CFP) {
3409 APFloat V1 = N1CFP->getValueAPF();
3410 const APFloat &V2 = N2CFP->getValueAPF();
3411 const APFloat &V3 = N3CFP->getValueAPF();
3412 APFloat::opStatus s =
3413 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
3414 if (s != APFloat::opInvalidOp)
3415 return getConstantFP(V1, VT);
3416 }
3417 break;
3418 }
3419 case ISD::CONCAT_VECTORS:
3420 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
3421 // one big BUILD_VECTOR.
3422 if (N1.getOpcode() == ISD::BUILD_VECTOR &&
3423 N2.getOpcode() == ISD::BUILD_VECTOR &&
3424 N3.getOpcode() == ISD::BUILD_VECTOR) {
3425 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
3426 N1.getNode()->op_end());
3427 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
3428 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
3429 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size());
3430 }
3431 break;
3432 case ISD::SETCC: {
3433 // Use FoldSetCC to simplify SETCC's.
3434 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
3435 if (Simp.getNode()) return Simp;
3436 break;
3437 }
3438 case ISD::SELECT:
3439 if (N1C) {
3440 if (N1C->getZExtValue())
3441 return N2; // select true, X, Y -> X
3442 return N3; // select false, X, Y -> Y
3443 }
3445 if (N2 == N3) return N2; // select C, X, X -> X
3446 break;
3447 case ISD::VECTOR_SHUFFLE:
3448 llvm_unreachable("should use getVectorShuffle constructor!");
3449 case ISD::INSERT_SUBVECTOR: {
3450 SDValue Index = N3;
3451 if (VT.isSimple() && N1.getValueType().isSimple()
3452 && N2.getValueType().isSimple()) {
3453 assert(VT.isVector() && N1.getValueType().isVector() &&
3454 N2.getValueType().isVector() &&
3455 "Insert subvector VTs must be a vectors");
3456 assert(VT == N1.getValueType() &&
3457 "Dest and insert subvector source types must match!");
3458 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
3459 "Insert subvector must be from smaller vector to larger vector!");
3460 if (isa<ConstantSDNode>(Index.getNode())) {
3461 assert((N2.getValueType().getVectorNumElements() +
3462 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
3463 <= VT.getVectorNumElements())
3464 && "Insert subvector overflow!");
3465 }
3467 // Trivial insertion.
3468 if (VT.getSimpleVT() == N2.getSimpleValueType())
3469 return N2;
3470 }
3471 break;
3472 }
3473 case ISD::BITCAST:
3474 // Fold bit_convert nodes from a type to themselves.
3475 if (N1.getValueType() == VT)
3476 return N1;
3477 break;
3478 }
3480 // Memoize node if it doesn't produce a flag.
3481 SDNode *N;
3482 SDVTList VTs = getVTList(VT);
3483 if (VT != MVT::Glue) {
3484 SDValue Ops[] = { N1, N2, N3 };
3485 FoldingSetNodeID ID;
3486 AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
3487 void *IP = nullptr;
3488 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
3489 return SDValue(E, 0);
3491 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3492 DL.getDebugLoc(), VTs, N1, N2, N3);
3493 CSEMap.InsertNode(N, IP);
3494 } else {
3495 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3496 DL.getDebugLoc(), VTs, N1, N2, N3);
3497 }
3499 AllNodes.push_back(N);
3500 #ifndef NDEBUG
3501 VerifySDNode(N);
3502 #endif
3503 return SDValue(N, 0);
3504 }
3506 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3507 SDValue N1, SDValue N2, SDValue N3,
3508 SDValue N4) {
3509 SDValue Ops[] = { N1, N2, N3, N4 };
3510 return getNode(Opcode, DL, VT, Ops, 4);
3511 }
3513 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3514 SDValue N1, SDValue N2, SDValue N3,
3515 SDValue N4, SDValue N5) {
3516 SDValue Ops[] = { N1, N2, N3, N4, N5 };
3517 return getNode(Opcode, DL, VT, Ops, 5);
3518 }
3520 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
3521 /// the incoming stack arguments to be loaded from the stack.
3522 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
3523 SmallVector<SDValue, 8> ArgChains;
3525 // Include the original chain at the beginning of the list. When this is
3526 // used by target LowerCall hooks, this helps legalize find the
3527 // CALLSEQ_BEGIN node.
3528 ArgChains.push_back(Chain);
3530 // Add a chain value for each stack argument.
3531 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
3532 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
3533 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3534 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3535 if (FI->getIndex() < 0)
3536 ArgChains.push_back(SDValue(L, 1));
3538 // Build a tokenfactor for all the chains.
3539 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other,
3540 &ArgChains[0], ArgChains.size());
3541 }
3543 /// getMemsetValue - Vectorized representation of the memset value
3544 /// operand.
3545 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
3546 SDLoc dl) {
3547 assert(Value.getOpcode() != ISD::UNDEF);
3549 unsigned NumBits = VT.getScalarType().getSizeInBits();
3550 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
3551 assert(C->getAPIntValue().getBitWidth() == 8);
3552 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
3553 if (VT.isInteger())
3554 return DAG.getConstant(Val, VT);
3555 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
3556 }
3558 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
3559 if (NumBits > 8) {
3560 // Use a multiplication with 0x010101... to extend the input to the
3561 // required length.
3562 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
3563 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
3564 }
3566 return Value;
3567 }
3569 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
3570 /// used when a memcpy is turned into a memset when the source is a constant
3571 /// string ptr.
3572 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
3573 const TargetLowering &TLI, StringRef Str) {
3574 // Handle vector with all elements zero.
3575 if (Str.empty()) {
3576 if (VT.isInteger())
3577 return DAG.getConstant(0, VT);
3578 else if (VT == MVT::f32 || VT == MVT::f64)
3579 return DAG.getConstantFP(0.0, VT);
3580 else if (VT.isVector()) {
3581 unsigned NumElts = VT.getVectorNumElements();
3582 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
3583 return DAG.getNode(ISD::BITCAST, dl, VT,
3584 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
3585 EltVT, NumElts)));
3586 } else
3587 llvm_unreachable("Expected type!");
3588 }
3590 assert(!VT.isVector() && "Can't handle vector type here!");
3591 unsigned NumVTBits = VT.getSizeInBits();
3592 unsigned NumVTBytes = NumVTBits / 8;
3593 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
3595 APInt Val(NumVTBits, 0);
3596 if (TLI.isLittleEndian()) {
3597 for (unsigned i = 0; i != NumBytes; ++i)
3598 Val |= (uint64_t)(unsigned char)Str[i] << i*8;
3599 } else {
3600 for (unsigned i = 0; i != NumBytes; ++i)
3601 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
3602 }
3604 // If the "cost" of materializing the integer immediate is less than the cost
3605 // of a load, then it is cost effective to turn the load into the immediate.
3606 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
3607 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
3608 return DAG.getConstant(Val, VT);
3609 return SDValue(nullptr, 0);
3610 }
3612 /// getMemBasePlusOffset - Returns base and offset node for the
3613 ///
3614 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
3615 SelectionDAG &DAG) {
3616 EVT VT = Base.getValueType();
3617 return DAG.getNode(ISD::ADD, dl,
3618 VT, Base, DAG.getConstant(Offset, VT));
3619 }
3621 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
3622 ///
3623 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
3624 unsigned SrcDelta = 0;
3625 GlobalAddressSDNode *G = nullptr;
3626 if (Src.getOpcode() == ISD::GlobalAddress)
3627 G = cast<GlobalAddressSDNode>(Src);
3628 else if (Src.getOpcode() == ISD::ADD &&
3629 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
3630 Src.getOperand(1).getOpcode() == ISD::Constant) {
3631 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
3632 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
3633 }
3634 if (!G)
3635 return false;
3637 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
3638 }
3640 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
3641 /// to replace the memset / memcpy. Return true if the number of memory ops
3642 /// is below the threshold. It returns the types of the sequence of
3643 /// memory ops to perform memset / memcpy by reference.
3644 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
3645 unsigned Limit, uint64_t Size,
3646 unsigned DstAlign, unsigned SrcAlign,
3647 bool IsMemset,
3648 bool ZeroMemset,
3649 bool MemcpyStrSrc,
3650 bool AllowOverlap,
3651 SelectionDAG &DAG,
3652 const TargetLowering &TLI) {
3653 assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
3654 "Expecting memcpy / memset source to meet alignment requirement!");
3655 // If 'SrcAlign' is zero, that means the memory operation does not need to
3656 // load the value, i.e. memset or memcpy from constant string. Otherwise,
3657 // it's the inferred alignment of the source. 'DstAlign', on the other hand,
3658 // is the specified alignment of the memory operation. If it is zero, that
3659 // means it's possible to change the alignment of the destination.
3660 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
3661 // not need to be loaded.
3662 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
3663 IsMemset, ZeroMemset, MemcpyStrSrc,
3664 DAG.getMachineFunction());
3666 if (VT == MVT::Other) {
3667 unsigned AS = 0;
3668 if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
3669 TLI.allowsUnalignedMemoryAccesses(VT, AS)) {
3670 VT = TLI.getPointerTy();
3671 } else {
3672 switch (DstAlign & 7) {
3673 case 0: VT = MVT::i64; break;
3674 case 4: VT = MVT::i32; break;
3675 case 2: VT = MVT::i16; break;
3676 default: VT = MVT::i8; break;
3677 }
3678 }
3680 MVT LVT = MVT::i64;
3681 while (!TLI.isTypeLegal(LVT))
3682 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
3683 assert(LVT.isInteger());
3685 if (VT.bitsGT(LVT))
3686 VT = LVT;
3687 }
3689 unsigned NumMemOps = 0;
3690 while (Size != 0) {
3691 unsigned VTSize = VT.getSizeInBits() / 8;
3692 while (VTSize > Size) {
3693 // For now, only use non-vector load / store's for the left-over pieces.
3694 EVT NewVT = VT;
3695 unsigned NewVTSize;
3697 bool Found = false;
3698 if (VT.isVector() || VT.isFloatingPoint()) {
3699 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
3700 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
3701 TLI.isSafeMemOpType(NewVT.getSimpleVT()))
3702 Found = true;
3703 else if (NewVT == MVT::i64 &&
3704 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
3705 TLI.isSafeMemOpType(MVT::f64)) {
3706 // i64 is usually not legal on 32-bit targets, but f64 may be.
3707 NewVT = MVT::f64;
3708 Found = true;
3709 }
3710 }
3712 if (!Found) {
3713 do {
3714 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
3715 if (NewVT == MVT::i8)
3716 break;
3717 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
3718 }
3719 NewVTSize = NewVT.getSizeInBits() / 8;
3721 // If the new VT cannot cover all of the remaining bits, then consider
3722 // issuing a (or a pair of) unaligned and overlapping load / store.
3723 // FIXME: Only does this for 64-bit or more since we don't have proper
3724 // cost model for unaligned load / store.
3725 bool Fast;
3726 unsigned AS = 0;
3727 if (NumMemOps && AllowOverlap &&
3728 VTSize >= 8 && NewVTSize < Size &&
3729 TLI.allowsUnalignedMemoryAccesses(VT, AS, &Fast) && Fast)
3730 VTSize = Size;
3731 else {
3732 VT = NewVT;
3733 VTSize = NewVTSize;
3734 }
3735 }
3737 if (++NumMemOps > Limit)
3738 return false;
3740 MemOps.push_back(VT);
3741 Size -= VTSize;
3742 }
3744 return true;
3745 }
3747 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
3748 SDValue Chain, SDValue Dst,
3749 SDValue Src, uint64_t Size,
3750 unsigned Align, bool isVol,
3751 bool AlwaysInline,
3752 MachinePointerInfo DstPtrInfo,
3753 MachinePointerInfo SrcPtrInfo) {
3754 // Turn a memcpy of undef to nop.
3755 if (Src.getOpcode() == ISD::UNDEF)
3756 return Chain;
3758 // Expand memcpy to a series of load and store ops if the size operand falls
3759 // below a certain threshold.
3760 // TODO: In the AlwaysInline case, if the size is big then generate a loop
3761 // rather than maybe a humongous number of loads and stores.
3762 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3763 std::vector<EVT> MemOps;
3764 bool DstAlignCanChange = false;
3765 MachineFunction &MF = DAG.getMachineFunction();
3766 MachineFrameInfo *MFI = MF.getFrameInfo();
3767 bool OptSize =
3768 MF.getFunction()->getAttributes().
3769 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
3770 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3771 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3772 DstAlignCanChange = true;
3773 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
3774 if (Align > SrcAlign)
3775 SrcAlign = Align;
3776 StringRef Str;
3777 bool CopyFromStr = isMemSrcFromString(Src, Str);
3778 bool isZeroStr = CopyFromStr && Str.empty();
3779 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
3781 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
3782 (DstAlignCanChange ? 0 : Align),
3783 (isZeroStr ? 0 : SrcAlign),
3784 false, false, CopyFromStr, true, DAG, TLI))
3785 return SDValue();
3787 if (DstAlignCanChange) {
3788 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
3789 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
3791 // Don't promote to an alignment that would require dynamic stack
3792 // realignment.
3793 const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo();
3794 if (!TRI->needsStackRealignment(MF))
3795 while (NewAlign > Align &&
3796 TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
3797 NewAlign /= 2;
3799 if (NewAlign > Align) {
3800 // Give the stack frame object a larger alignment if needed.
3801 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3802 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3803 Align = NewAlign;
3804 }
3805 }
3807 SmallVector<SDValue, 8> OutChains;
3808 unsigned NumMemOps = MemOps.size();
3809 uint64_t SrcOff = 0, DstOff = 0;
3810 for (unsigned i = 0; i != NumMemOps; ++i) {
3811 EVT VT = MemOps[i];
3812 unsigned VTSize = VT.getSizeInBits() / 8;
3813 SDValue Value, Store;
3815 if (VTSize > Size) {
3816 // Issuing an unaligned load / store pair that overlaps with the previous
3817 // pair. Adjust the offset accordingly.
3818 assert(i == NumMemOps-1 && i != 0);
3819 SrcOff -= VTSize - Size;
3820 DstOff -= VTSize - Size;
3821 }
3823 if (CopyFromStr &&
3824 (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
3825 // It's unlikely a store of a vector immediate can be done in a single
3826 // instruction. It would require a load from a constantpool first.
3827 // We only handle zero vectors here.
3828 // FIXME: Handle other cases where store of vector immediate is done in
3829 // a single instruction.
3830 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
3831 if (Value.getNode())
3832 Store = DAG.getStore(Chain, dl, Value,
3833 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
3834 DstPtrInfo.getWithOffset(DstOff), isVol,
3835 false, Align);
3836 }
3838 if (!Store.getNode()) {
3839 // The type might not be legal for the target. This should only happen
3840 // if the type is smaller than a legal type, as on PPC, so the right
3841 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
3842 // to Load/Store if NVT==VT.
3843 // FIXME does the case above also need this?
3844 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3845 assert(NVT.bitsGE(VT));
3846 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
3847 getMemBasePlusOffset(Src, SrcOff, dl, DAG),
3848 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
3849 MinAlign(SrcAlign, SrcOff));
3850 Store = DAG.getTruncStore(Chain, dl, Value,
3851 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
3852 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
3853 false, Align);
3854 }
3855 OutChains.push_back(Store);
3856 SrcOff += VTSize;
3857 DstOff += VTSize;
3858 Size -= VTSize;
3859 }
3861 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3862 &OutChains[0], OutChains.size());
3863 }
3865 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
3866 SDValue Chain, SDValue Dst,
3867 SDValue Src, uint64_t Size,
3868 unsigned Align, bool isVol,
3869 bool AlwaysInline,
3870 MachinePointerInfo DstPtrInfo,
3871 MachinePointerInfo SrcPtrInfo) {
3872 // Turn a memmove of undef to nop.
3873 if (Src.getOpcode() == ISD::UNDEF)
3874 return Chain;
3876 // Expand memmove to a series of load and store ops if the size operand falls
3877 // below a certain threshold.
3878 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3879 std::vector<EVT> MemOps;
3880 bool DstAlignCanChange = false;
3881 MachineFunction &MF = DAG.getMachineFunction();
3882 MachineFrameInfo *MFI = MF.getFrameInfo();
3883 bool OptSize = MF.getFunction()->getAttributes().
3884 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
3885 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3886 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3887 DstAlignCanChange = true;
3888 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
3889 if (Align > SrcAlign)
3890 SrcAlign = Align;
3891 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
3893 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
3894 (DstAlignCanChange ? 0 : Align), SrcAlign,
3895 false, false, false, false, DAG, TLI))
3896 return SDValue();
3898 if (DstAlignCanChange) {
3899 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
3900 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
3901 if (NewAlign > Align) {
3902 // Give the stack frame object a larger alignment if needed.
3903 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3904 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3905 Align = NewAlign;
3906 }
3907 }
3909 uint64_t SrcOff = 0, DstOff = 0;
3910 SmallVector<SDValue, 8> LoadValues;
3911 SmallVector<SDValue, 8> LoadChains;
3912 SmallVector<SDValue, 8> OutChains;
3913 unsigned NumMemOps = MemOps.size();
3914 for (unsigned i = 0; i < NumMemOps; i++) {
3915 EVT VT = MemOps[i];
3916 unsigned VTSize = VT.getSizeInBits() / 8;
3917 SDValue Value;
3919 Value = DAG.getLoad(VT, dl, Chain,
3920 getMemBasePlusOffset(Src, SrcOff, dl, DAG),
3921 SrcPtrInfo.getWithOffset(SrcOff), isVol,
3922 false, false, SrcAlign);
3923 LoadValues.push_back(Value);
3924 LoadChains.push_back(Value.getValue(1));
3925 SrcOff += VTSize;
3926 }
3927 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3928 &LoadChains[0], LoadChains.size());
3929 OutChains.clear();
3930 for (unsigned i = 0; i < NumMemOps; i++) {
3931 EVT VT = MemOps[i];
3932 unsigned VTSize = VT.getSizeInBits() / 8;
3933 SDValue Store;
3935 Store = DAG.getStore(Chain, dl, LoadValues[i],
3936 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
3937 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
3938 OutChains.push_back(Store);
3939 DstOff += VTSize;
3940 }
3942 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3943 &OutChains[0], OutChains.size());
3944 }
3946 /// \brief Lower the call to 'memset' intrinsic function into a series of store
3947 /// operations.
3948 ///
3949 /// \param DAG Selection DAG where lowered code is placed.
3950 /// \param dl Link to corresponding IR location.
3951 /// \param Chain Control flow dependency.
3952 /// \param Dst Pointer to destination memory location.
3953 /// \param Src Value of byte to write into the memory.
3954 /// \param Size Number of bytes to write.
3955 /// \param Align Alignment of the destination in bytes.
3956 /// \param isVol True if destination is volatile.
3957 /// \param DstPtrInfo IR information on the memory pointer.
3958 /// \returns New head in the control flow, if lowering was successful, empty
3959 /// SDValue otherwise.
3960 ///
3961 /// The function tries to replace 'llvm.memset' intrinsic with several store
3962 /// operations and value calculation code. This is usually profitable for small
3963 /// memory size.
3964 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
3965 SDValue Chain, SDValue Dst,
3966 SDValue Src, uint64_t Size,
3967 unsigned Align, bool isVol,
3968 MachinePointerInfo DstPtrInfo) {
3969 // Turn a memset of undef to nop.
3970 if (Src.getOpcode() == ISD::UNDEF)
3971 return Chain;
3973 // Expand memset to a series of load/store ops if the size operand
3974 // falls below a certain threshold.
3975 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3976 std::vector<EVT> MemOps;
3977 bool DstAlignCanChange = false;
3978 MachineFunction &MF = DAG.getMachineFunction();
3979 MachineFrameInfo *MFI = MF.getFrameInfo();
3980 bool OptSize = MF.getFunction()->getAttributes().
3981 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
3982 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
3983 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
3984 DstAlignCanChange = true;
3985 bool IsZeroVal =
3986 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
3987 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
3988 Size, (DstAlignCanChange ? 0 : Align), 0,
3989 true, IsZeroVal, false, true, DAG, TLI))
3990 return SDValue();
3992 if (DstAlignCanChange) {
3993 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
3994 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
3995 if (NewAlign > Align) {
3996 // Give the stack frame object a larger alignment if needed.
3997 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
3998 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
3999 Align = NewAlign;
4000 }
4001 }
4003 SmallVector<SDValue, 8> OutChains;
4004 uint64_t DstOff = 0;
4005 unsigned NumMemOps = MemOps.size();
4007 // Find the largest store and generate the bit pattern for it.
4008 EVT LargestVT = MemOps[0];
4009 for (unsigned i = 1; i < NumMemOps; i++)
4010 if (MemOps[i].bitsGT(LargestVT))
4011 LargestVT = MemOps[i];
4012 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
4014 for (unsigned i = 0; i < NumMemOps; i++) {
4015 EVT VT = MemOps[i];
4016 unsigned VTSize = VT.getSizeInBits() / 8;
4017 if (VTSize > Size) {
4018 // Issuing an unaligned load / store pair that overlaps with the previous
4019 // pair. Adjust the offset accordingly.
4020 assert(i == NumMemOps-1 && i != 0);
4021 DstOff -= VTSize - Size;
4022 }
4024 // If this store is smaller than the largest store see whether we can get
4025 // the smaller value for free with a truncate.
4026 SDValue Value = MemSetValue;
4027 if (VT.bitsLT(LargestVT)) {
4028 if (!LargestVT.isVector() && !VT.isVector() &&
4029 TLI.isTruncateFree(LargestVT, VT))
4030 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
4031 else
4032 Value = getMemsetValue(Src, VT, DAG, dl);
4033 }
4034 assert(Value.getValueType() == VT && "Value with wrong type.");
4035 SDValue Store = DAG.getStore(Chain, dl, Value,
4036 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4037 DstPtrInfo.getWithOffset(DstOff),
4038 isVol, false, Align);
4039 OutChains.push_back(Store);
4040 DstOff += VT.getSizeInBits() / 8;
4041 Size -= VTSize;
4042 }
4044 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4045 &OutChains[0], OutChains.size());
4046 }
4048 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
4049 SDValue Src, SDValue Size,
4050 unsigned Align, bool isVol, bool AlwaysInline,
4051 MachinePointerInfo DstPtrInfo,
4052 MachinePointerInfo SrcPtrInfo) {
4053 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4055 // Check to see if we should lower the memcpy to loads and stores first.
4056 // For cases within the target-specified limits, this is the best choice.
4057 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4058 if (ConstantSize) {
4059 // Memcpy with size zero? Just return the original chain.
4060 if (ConstantSize->isNullValue())
4061 return Chain;
4063 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4064 ConstantSize->getZExtValue(),Align,
4065 isVol, false, DstPtrInfo, SrcPtrInfo);
4066 if (Result.getNode())
4067 return Result;
4068 }
4070 // Then check to see if we should lower the memcpy with target-specific
4071 // code. If the target chooses to do this, this is the next best.
4072 SDValue Result =
4073 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
4074 isVol, AlwaysInline,
4075 DstPtrInfo, SrcPtrInfo);
4076 if (Result.getNode())
4077 return Result;
4079 // If we really need inline code and the target declined to provide it,
4080 // use a (potentially long) sequence of loads and stores.
4081 if (AlwaysInline) {
4082 assert(ConstantSize && "AlwaysInline requires a constant size!");
4083 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4084 ConstantSize->getZExtValue(), Align, isVol,
4085 true, DstPtrInfo, SrcPtrInfo);
4086 }
4088 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
4089 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
4090 // respect volatile, so they may do things like read or write memory
4091 // beyond the given memory regions. But fixing this isn't easy, and most
4092 // people don't care.
4094 const TargetLowering *TLI = TM.getTargetLowering();
4096 // Emit a library call.
4097 TargetLowering::ArgListTy Args;
4098 TargetLowering::ArgListEntry Entry;
4099 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
4100 Entry.Node = Dst; Args.push_back(Entry);
4101 Entry.Node = Src; Args.push_back(Entry);
4102 Entry.Node = Size; Args.push_back(Entry);
4103 // FIXME: pass in SDLoc
4104 TargetLowering::
4105 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()),
4106 false, false, false, false, 0,
4107 TLI->getLibcallCallingConv(RTLIB::MEMCPY),
4108 /*isTailCall=*/false,
4109 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false,
4110 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
4111 TLI->getPointerTy()),
4112 Args, *this, dl);
4113 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4115 return CallResult.second;
4116 }
4118 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
4119 SDValue Src, SDValue Size,
4120 unsigned Align, bool isVol,
4121 MachinePointerInfo DstPtrInfo,
4122 MachinePointerInfo SrcPtrInfo) {
4123 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4125 // Check to see if we should lower the memmove to loads and stores first.
4126 // For cases within the target-specified limits, this is the best choice.
4127 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4128 if (ConstantSize) {
4129 // Memmove with size zero? Just return the original chain.
4130 if (ConstantSize->isNullValue())
4131 return Chain;
4133 SDValue Result =
4134 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
4135 ConstantSize->getZExtValue(), Align, isVol,
4136 false, DstPtrInfo, SrcPtrInfo);
4137 if (Result.getNode())
4138 return Result;
4139 }
4141 // Then check to see if we should lower the memmove with target-specific
4142 // code. If the target chooses to do this, this is the next best.
4143 SDValue Result =
4144 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol,
4145 DstPtrInfo, SrcPtrInfo);
4146 if (Result.getNode())
4147 return Result;
4149 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
4150 // not be safe. See memcpy above for more details.
4152 const TargetLowering *TLI = TM.getTargetLowering();
4154 // Emit a library call.
4155 TargetLowering::ArgListTy Args;
4156 TargetLowering::ArgListEntry Entry;
4157 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
4158 Entry.Node = Dst; Args.push_back(Entry);
4159 Entry.Node = Src; Args.push_back(Entry);
4160 Entry.Node = Size; Args.push_back(Entry);
4161 // FIXME: pass in SDLoc
4162 TargetLowering::
4163 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()),
4164 false, false, false, false, 0,
4165 TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
4166 /*isTailCall=*/false,
4167 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false,
4168 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
4169 TLI->getPointerTy()),
4170 Args, *this, dl);
4171 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4173 return CallResult.second;
4174 }
4176 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
4177 SDValue Src, SDValue Size,
4178 unsigned Align, bool isVol,
4179 MachinePointerInfo DstPtrInfo) {
4180 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4182 // Check to see if we should lower the memset to stores first.
4183 // For cases within the target-specified limits, this is the best choice.
4184 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4185 if (ConstantSize) {
4186 // Memset with size zero? Just return the original chain.
4187 if (ConstantSize->isNullValue())
4188 return Chain;
4190 SDValue Result =
4191 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
4192 Align, isVol, DstPtrInfo);
4194 if (Result.getNode())
4195 return Result;
4196 }
4198 // Then check to see if we should lower the memset with target-specific
4199 // code. If the target chooses to do this, this is the next best.
4200 SDValue Result =
4201 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol,
4202 DstPtrInfo);
4203 if (Result.getNode())
4204 return Result;
4206 // Emit a library call.
4207 const TargetLowering *TLI = TM.getTargetLowering();
4208 Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
4209 TargetLowering::ArgListTy Args;
4210 TargetLowering::ArgListEntry Entry;
4211 Entry.Node = Dst; Entry.Ty = IntPtrTy;
4212 Args.push_back(Entry);
4213 // Extend or truncate the argument to be an i32 value for the call.
4214 if (Src.getValueType().bitsGT(MVT::i32))
4215 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
4216 else
4217 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
4218 Entry.Node = Src;
4219 Entry.Ty = Type::getInt32Ty(*getContext());
4220 Entry.isSExt = true;
4221 Args.push_back(Entry);
4222 Entry.Node = Size;
4223 Entry.Ty = IntPtrTy;
4224 Entry.isSExt = false;
4225 Args.push_back(Entry);
4226 // FIXME: pass in SDLoc
4227 TargetLowering::
4228 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()),
4229 false, false, false, false, 0,
4230 TLI->getLibcallCallingConv(RTLIB::MEMSET),
4231 /*isTailCall=*/false,
4232 /*doesNotReturn*/false, /*isReturnValueUsed=*/false,
4233 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
4234 TLI->getPointerTy()),
4235 Args, *this, dl);
4236 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4238 return CallResult.second;
4239 }
4241 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4242 SDVTList VTList, SDValue *Ops, unsigned NumOps,
4243 MachineMemOperand *MMO,
4244 AtomicOrdering SuccessOrdering,
4245 AtomicOrdering FailureOrdering,
4246 SynchronizationScope SynchScope) {
4247 FoldingSetNodeID ID;
4248 ID.AddInteger(MemVT.getRawBits());
4249 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
4250 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4251 void* IP = nullptr;
4252 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4253 cast<AtomicSDNode>(E)->refineAlignment(MMO);
4254 return SDValue(E, 0);
4255 }
4257 // Allocate the operands array for the node out of the BumpPtrAllocator, since
4258 // SDNode doesn't have access to it. This memory will be "leaked" when
4259 // the node is deallocated, but recovered when the allocator is released.
4260 // If the number of operands is less than 5 we use AtomicSDNode's internal
4261 // storage.
4262 SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) : nullptr;
4264 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
4265 dl.getDebugLoc(), VTList, MemVT,
4266 Ops, DynOps, NumOps, MMO,
4267 SuccessOrdering, FailureOrdering,
4268 SynchScope);
4269 CSEMap.InsertNode(N, IP);
4270 AllNodes.push_back(N);
4271 return SDValue(N, 0);
4272 }
4274 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4275 SDVTList VTList, SDValue *Ops, unsigned NumOps,
4276 MachineMemOperand *MMO,
4277 AtomicOrdering Ordering,
4278 SynchronizationScope SynchScope) {
4279 return getAtomic(Opcode, dl, MemVT, VTList, Ops, NumOps, MMO, Ordering,
4280 Ordering, SynchScope);
4281 }
4283 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4284 SDValue Chain, SDValue Ptr, SDValue Cmp,
4285 SDValue Swp, MachinePointerInfo PtrInfo,
4286 unsigned Alignment,
4287 AtomicOrdering SuccessOrdering,
4288 AtomicOrdering FailureOrdering,
4289 SynchronizationScope SynchScope) {
4290 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4291 Alignment = getEVTAlignment(MemVT);
4293 MachineFunction &MF = getMachineFunction();
4295 // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE.
4296 // For now, atomics are considered to be volatile always.
4297 // FIXME: Volatile isn't really correct; we should keep track of atomic
4298 // orderings in the memoperand.
4299 unsigned Flags = MachineMemOperand::MOVolatile;
4300 if (Opcode != ISD::ATOMIC_STORE)
4301 Flags |= MachineMemOperand::MOLoad;
4302 if (Opcode != ISD::ATOMIC_LOAD)
4303 Flags |= MachineMemOperand::MOStore;
4305 MachineMemOperand *MMO =
4306 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
4308 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
4309 SuccessOrdering, FailureOrdering, SynchScope);
4310 }
4312 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4313 SDValue Chain,
4314 SDValue Ptr, SDValue Cmp,
4315 SDValue Swp, MachineMemOperand *MMO,
4316 AtomicOrdering SuccessOrdering,
4317 AtomicOrdering FailureOrdering,
4318 SynchronizationScope SynchScope) {
4319 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
4320 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4322 EVT VT = Cmp.getValueType();
4324 SDVTList VTs = getVTList(VT, MVT::Other);
4325 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4326 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, SuccessOrdering,
4327 FailureOrdering, SynchScope);
4328 }
4330 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4331 SDValue Chain,
4332 SDValue Ptr, SDValue Val,
4333 const Value* PtrVal,
4334 unsigned Alignment,
4335 AtomicOrdering Ordering,
4336 SynchronizationScope SynchScope) {
4337 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4338 Alignment = getEVTAlignment(MemVT);
4340 MachineFunction &MF = getMachineFunction();
4341 // An atomic store does not load. An atomic load does not store.
4342 // (An atomicrmw obviously both loads and stores.)
4343 // For now, atomics are considered to be volatile always, and they are
4344 // chained as such.
4345 // FIXME: Volatile isn't really correct; we should keep track of atomic
4346 // orderings in the memoperand.
4347 unsigned Flags = MachineMemOperand::MOVolatile;
4348 if (Opcode != ISD::ATOMIC_STORE)
4349 Flags |= MachineMemOperand::MOLoad;
4350 if (Opcode != ISD::ATOMIC_LOAD)
4351 Flags |= MachineMemOperand::MOStore;
4353 MachineMemOperand *MMO =
4354 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4355 MemVT.getStoreSize(), Alignment);
4357 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
4358 Ordering, SynchScope);
4359 }
4361 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4362 SDValue Chain,
4363 SDValue Ptr, SDValue Val,
4364 MachineMemOperand *MMO,
4365 AtomicOrdering Ordering,
4366 SynchronizationScope SynchScope) {
4367 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
4368 Opcode == ISD::ATOMIC_LOAD_SUB ||
4369 Opcode == ISD::ATOMIC_LOAD_AND ||
4370 Opcode == ISD::ATOMIC_LOAD_OR ||
4371 Opcode == ISD::ATOMIC_LOAD_XOR ||
4372 Opcode == ISD::ATOMIC_LOAD_NAND ||
4373 Opcode == ISD::ATOMIC_LOAD_MIN ||
4374 Opcode == ISD::ATOMIC_LOAD_MAX ||
4375 Opcode == ISD::ATOMIC_LOAD_UMIN ||
4376 Opcode == ISD::ATOMIC_LOAD_UMAX ||
4377 Opcode == ISD::ATOMIC_SWAP ||
4378 Opcode == ISD::ATOMIC_STORE) &&
4379 "Invalid Atomic Op");
4381 EVT VT = Val.getValueType();
4383 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
4384 getVTList(VT, MVT::Other);
4385 SDValue Ops[] = {Chain, Ptr, Val};
4386 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope);
4387 }
4389 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4390 EVT VT, SDValue Chain,
4391 SDValue Ptr,
4392 MachineMemOperand *MMO,
4393 AtomicOrdering Ordering,
4394 SynchronizationScope SynchScope) {
4395 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
4397 SDVTList VTs = getVTList(VT, MVT::Other);
4398 SDValue Ops[] = {Chain, Ptr};
4399 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 2, MMO, Ordering, SynchScope);
4400 }
4402 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
4403 SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps,
4404 SDLoc dl) {
4405 if (NumOps == 1)
4406 return Ops[0];
4408 SmallVector<EVT, 4> VTs;
4409 VTs.reserve(NumOps);
4410 for (unsigned i = 0; i < NumOps; ++i)
4411 VTs.push_back(Ops[i].getValueType());
4412 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs),
4413 Ops, NumOps);
4414 }
4416 SDValue
4417 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl,
4418 const EVT *VTs, unsigned NumVTs,
4419 const SDValue *Ops, unsigned NumOps,
4420 EVT MemVT, MachinePointerInfo PtrInfo,
4421 unsigned Align, bool Vol,
4422 bool ReadMem, bool WriteMem) {
4423 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps,
4424 MemVT, PtrInfo, Align, Vol,
4425 ReadMem, WriteMem);
4426 }
4428 SDValue
4429 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
4430 const SDValue *Ops, unsigned NumOps,
4431 EVT MemVT, MachinePointerInfo PtrInfo,
4432 unsigned Align, bool Vol,
4433 bool ReadMem, bool WriteMem) {
4434 if (Align == 0) // Ensure that codegen never sees alignment 0
4435 Align = getEVTAlignment(MemVT);
4437 MachineFunction &MF = getMachineFunction();
4438 unsigned Flags = 0;
4439 if (WriteMem)
4440 Flags |= MachineMemOperand::MOStore;
4441 if (ReadMem)
4442 Flags |= MachineMemOperand::MOLoad;
4443 if (Vol)
4444 Flags |= MachineMemOperand::MOVolatile;
4445 MachineMemOperand *MMO =
4446 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align);
4448 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO);
4449 }
4451 SDValue
4452 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
4453 const SDValue *Ops, unsigned NumOps,
4454 EVT MemVT, MachineMemOperand *MMO) {
4455 assert((Opcode == ISD::INTRINSIC_VOID ||
4456 Opcode == ISD::INTRINSIC_W_CHAIN ||
4457 Opcode == ISD::PREFETCH ||
4458 Opcode == ISD::LIFETIME_START ||
4459 Opcode == ISD::LIFETIME_END ||
4460 (Opcode <= INT_MAX &&
4461 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
4462 "Opcode is not a memory-accessing opcode!");
4464 // Memoize the node unless it returns a flag.
4465 MemIntrinsicSDNode *N;
4466 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
4467 FoldingSetNodeID ID;
4468 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
4469 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4470 void *IP = nullptr;
4471 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4472 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
4473 return SDValue(E, 0);
4474 }
4476 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4477 dl.getDebugLoc(), VTList, Ops,
4478 NumOps, MemVT, MMO);
4479 CSEMap.InsertNode(N, IP);
4480 } else {
4481 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4482 dl.getDebugLoc(), VTList, Ops,
4483 NumOps, MemVT, MMO);
4484 }
4485 AllNodes.push_back(N);
4486 return SDValue(N, 0);
4487 }
4489 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4490 /// MachinePointerInfo record from it. This is particularly useful because the
4491 /// code generator has many cases where it doesn't bother passing in a
4492 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
4493 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
4494 // If this is FI+Offset, we can model it.
4495 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
4496 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
4498 // If this is (FI+Offset1)+Offset2, we can model it.
4499 if (Ptr.getOpcode() != ISD::ADD ||
4500 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
4501 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
4502 return MachinePointerInfo();
4504 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
4505 return MachinePointerInfo::getFixedStack(FI, Offset+
4506 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
4507 }
4509 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4510 /// MachinePointerInfo record from it. This is particularly useful because the
4511 /// code generator has many cases where it doesn't bother passing in a
4512 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
4513 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
4514 // If the 'Offset' value isn't a constant, we can't handle this.
4515 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
4516 return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
4517 if (OffsetOp.getOpcode() == ISD::UNDEF)
4518 return InferPointerInfo(Ptr);
4519 return MachinePointerInfo();
4520 }
4523 SDValue
4524 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
4525 EVT VT, SDLoc dl, SDValue Chain,
4526 SDValue Ptr, SDValue Offset,
4527 MachinePointerInfo PtrInfo, EVT MemVT,
4528 bool isVolatile, bool isNonTemporal, bool isInvariant,
4529 unsigned Alignment, const MDNode *TBAAInfo,
4530 const MDNode *Ranges) {
4531 assert(Chain.getValueType() == MVT::Other &&
4532 "Invalid chain type");
4533 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4534 Alignment = getEVTAlignment(VT);
4536 unsigned Flags = MachineMemOperand::MOLoad;
4537 if (isVolatile)
4538 Flags |= MachineMemOperand::MOVolatile;
4539 if (isNonTemporal)
4540 Flags |= MachineMemOperand::MONonTemporal;
4541 if (isInvariant)
4542 Flags |= MachineMemOperand::MOInvariant;
4544 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
4545 // clients.
4546 if (PtrInfo.V.isNull())
4547 PtrInfo = InferPointerInfo(Ptr, Offset);
4549 MachineFunction &MF = getMachineFunction();
4550 MachineMemOperand *MMO =
4551 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
4552 TBAAInfo, Ranges);
4553 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
4554 }
4556 SDValue
4557 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
4558 EVT VT, SDLoc dl, SDValue Chain,
4559 SDValue Ptr, SDValue Offset, EVT MemVT,
4560 MachineMemOperand *MMO) {
4561 if (VT == MemVT) {
4562 ExtType = ISD::NON_EXTLOAD;
4563 } else if (ExtType == ISD::NON_EXTLOAD) {
4564 assert(VT == MemVT && "Non-extending load from different memory type!");
4565 } else {
4566 // Extending load.
4567 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
4568 "Should only be an extending load, not truncating!");
4569 assert(VT.isInteger() == MemVT.isInteger() &&
4570 "Cannot convert from FP to Int or Int -> FP!");
4571 assert(VT.isVector() == MemVT.isVector() &&
4572 "Cannot use trunc store to convert to or from a vector!");
4573 assert((!VT.isVector() ||
4574 VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
4575 "Cannot use trunc store to change the number of vector elements!");
4576 }
4578 bool Indexed = AM != ISD::UNINDEXED;
4579 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
4580 "Unindexed load with an offset!");
4582 SDVTList VTs = Indexed ?
4583 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
4584 SDValue Ops[] = { Chain, Ptr, Offset };
4585 FoldingSetNodeID ID;
4586 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
4587 ID.AddInteger(MemVT.getRawBits());
4588 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
4589 MMO->isNonTemporal(),
4590 MMO->isInvariant()));
4591 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4592 void *IP = nullptr;
4593 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4594 cast<LoadSDNode>(E)->refineAlignment(MMO);
4595 return SDValue(E, 0);
4596 }
4597 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
4598 dl.getDebugLoc(), VTs, AM, ExtType,
4599 MemVT, MMO);
4600 CSEMap.InsertNode(N, IP);
4601 AllNodes.push_back(N);
4602 return SDValue(N, 0);
4603 }
4605 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
4606 SDValue Chain, SDValue Ptr,
4607 MachinePointerInfo PtrInfo,
4608 bool isVolatile, bool isNonTemporal,
4609 bool isInvariant, unsigned Alignment,
4610 const MDNode *TBAAInfo,
4611 const MDNode *Ranges) {
4612 SDValue Undef = getUNDEF(Ptr.getValueType());
4613 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
4614 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
4615 TBAAInfo, Ranges);
4616 }
4618 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
4619 SDValue Chain, SDValue Ptr,
4620 MachineMemOperand *MMO) {
4621 SDValue Undef = getUNDEF(Ptr.getValueType());
4622 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
4623 VT, MMO);
4624 }
4626 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
4627 SDValue Chain, SDValue Ptr,
4628 MachinePointerInfo PtrInfo, EVT MemVT,
4629 bool isVolatile, bool isNonTemporal,
4630 unsigned Alignment, const MDNode *TBAAInfo) {
4631 SDValue Undef = getUNDEF(Ptr.getValueType());
4632 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
4633 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment,
4634 TBAAInfo);
4635 }
4638 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
4639 SDValue Chain, SDValue Ptr, EVT MemVT,
4640 MachineMemOperand *MMO) {
4641 SDValue Undef = getUNDEF(Ptr.getValueType());
4642 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
4643 MemVT, MMO);
4644 }
4646 SDValue
4647 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
4648 SDValue Offset, ISD::MemIndexedMode AM) {
4649 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
4650 assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
4651 "Load is already a indexed load!");
4652 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
4653 LD->getChain(), Base, Offset, LD->getPointerInfo(),
4654 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
4655 false, LD->getAlignment());
4656 }
4658 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
4659 SDValue Ptr, MachinePointerInfo PtrInfo,
4660 bool isVolatile, bool isNonTemporal,
4661 unsigned Alignment, const MDNode *TBAAInfo) {
4662 assert(Chain.getValueType() == MVT::Other &&
4663 "Invalid chain type");
4664 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4665 Alignment = getEVTAlignment(Val.getValueType());
4667 unsigned Flags = MachineMemOperand::MOStore;
4668 if (isVolatile)
4669 Flags |= MachineMemOperand::MOVolatile;
4670 if (isNonTemporal)
4671 Flags |= MachineMemOperand::MONonTemporal;
4673 if (PtrInfo.V.isNull())
4674 PtrInfo = InferPointerInfo(Ptr);
4676 MachineFunction &MF = getMachineFunction();
4677 MachineMemOperand *MMO =
4678 MF.getMachineMemOperand(PtrInfo, Flags,
4679 Val.getValueType().getStoreSize(), Alignment,
4680 TBAAInfo);
4682 return getStore(Chain, dl, Val, Ptr, MMO);
4683 }
4685 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
4686 SDValue Ptr, MachineMemOperand *MMO) {
4687 assert(Chain.getValueType() == MVT::Other &&
4688 "Invalid chain type");
4689 EVT VT = Val.getValueType();
4690 SDVTList VTs = getVTList(MVT::Other);
4691 SDValue Undef = getUNDEF(Ptr.getValueType());
4692 SDValue Ops[] = { Chain, Val, Ptr, Undef };
4693 FoldingSetNodeID ID;
4694 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
4695 ID.AddInteger(VT.getRawBits());
4696 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
4697 MMO->isNonTemporal(), MMO->isInvariant()));
4698 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4699 void *IP = nullptr;
4700 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4701 cast<StoreSDNode>(E)->refineAlignment(MMO);
4702 return SDValue(E, 0);
4703 }
4704 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
4705 dl.getDebugLoc(), VTs,
4706 ISD::UNINDEXED, false, VT, MMO);
4707 CSEMap.InsertNode(N, IP);
4708 AllNodes.push_back(N);
4709 return SDValue(N, 0);
4710 }
4712 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
4713 SDValue Ptr, MachinePointerInfo PtrInfo,
4714 EVT SVT,bool isVolatile, bool isNonTemporal,
4715 unsigned Alignment,
4716 const MDNode *TBAAInfo) {
4717 assert(Chain.getValueType() == MVT::Other &&
4718 "Invalid chain type");
4719 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4720 Alignment = getEVTAlignment(SVT);
4722 unsigned Flags = MachineMemOperand::MOStore;
4723 if (isVolatile)
4724 Flags |= MachineMemOperand::MOVolatile;
4725 if (isNonTemporal)
4726 Flags |= MachineMemOperand::MONonTemporal;
4728 if (PtrInfo.V.isNull())
4729 PtrInfo = InferPointerInfo(Ptr);
4731 MachineFunction &MF = getMachineFunction();
4732 MachineMemOperand *MMO =
4733 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
4734 TBAAInfo);
4736 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
4737 }
4739 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
4740 SDValue Ptr, EVT SVT,
4741 MachineMemOperand *MMO) {
4742 EVT VT = Val.getValueType();
4744 assert(Chain.getValueType() == MVT::Other &&
4745 "Invalid chain type");
4746 if (VT == SVT)
4747 return getStore(Chain, dl, Val, Ptr, MMO);
4749 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
4750 "Should only be a truncating store, not extending!");
4751 assert(VT.isInteger() == SVT.isInteger() &&
4752 "Can't do FP-INT conversion!");
4753 assert(VT.isVector() == SVT.isVector() &&
4754 "Cannot use trunc store to convert to or from a vector!");
4755 assert((!VT.isVector() ||
4756 VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
4757 "Cannot use trunc store to change the number of vector elements!");
4759 SDVTList VTs = getVTList(MVT::Other);
4760 SDValue Undef = getUNDEF(Ptr.getValueType());
4761 SDValue Ops[] = { Chain, Val, Ptr, Undef };
4762 FoldingSetNodeID ID;
4763 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
4764 ID.AddInteger(SVT.getRawBits());
4765 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
4766 MMO->isNonTemporal(), MMO->isInvariant()));
4767 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4768 void *IP = nullptr;
4769 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
4770 cast<StoreSDNode>(E)->refineAlignment(MMO);
4771 return SDValue(E, 0);
4772 }
4773 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
4774 dl.getDebugLoc(), VTs,
4775 ISD::UNINDEXED, true, SVT, MMO);
4776 CSEMap.InsertNode(N, IP);
4777 AllNodes.push_back(N);
4778 return SDValue(N, 0);
4779 }
4781 SDValue
4782 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
4783 SDValue Offset, ISD::MemIndexedMode AM) {
4784 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
4785 assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
4786 "Store is already a indexed store!");
4787 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
4788 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
4789 FoldingSetNodeID ID;
4790 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
4791 ID.AddInteger(ST->getMemoryVT().getRawBits());
4792 ID.AddInteger(ST->getRawSubclassData());
4793 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
4794 void *IP = nullptr;
4795 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
4796 return SDValue(E, 0);
4798 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
4799 dl.getDebugLoc(), VTs, AM,
4800 ST->isTruncatingStore(),
4801 ST->getMemoryVT(),
4802 ST->getMemOperand());
4803 CSEMap.InsertNode(N, IP);
4804 AllNodes.push_back(N);
4805 return SDValue(N, 0);
4806 }
4808 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
4809 SDValue Chain, SDValue Ptr,
4810 SDValue SV,
4811 unsigned Align) {
4812 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
4813 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4);
4814 }
4816 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
4817 const SDUse *Ops, unsigned NumOps) {
4818 switch (NumOps) {
4819 case 0: return getNode(Opcode, DL, VT);
4820 case 1: return getNode(Opcode, DL, VT, Ops[0]);
4821 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
4822 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
4823 default: break;
4824 }
4826 // Copy from an SDUse array into an SDValue array for use with
4827 // the regular getNode logic.
4828 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps);
4829 return getNode(Opcode, DL, VT, &NewOps[0], NumOps);
4830 }
4832 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
4833 const SDValue *Ops, unsigned NumOps) {
4834 switch (NumOps) {
4835 case 0: return getNode(Opcode, DL, VT);
4836 case 1: return getNode(Opcode, DL, VT, Ops[0]);
4837 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
4838 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
4839 default: break;
4840 }
4842 switch (Opcode) {
4843 default: break;
4844 case ISD::SELECT_CC: {
4845 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
4846 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
4847 "LHS and RHS of condition must have same type!");
4848 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
4849 "True and False arms of SelectCC must have same type!");
4850 assert(Ops[2].getValueType() == VT &&
4851 "select_cc node must be of same type as true and false value!");
4852 break;
4853 }
4854 case ISD::BR_CC: {
4855 assert(NumOps == 5 && "BR_CC takes 5 operands!");
4856 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
4857 "LHS/RHS of comparison should match types!");
4858 break;
4859 }
4860 }
4862 // Memoize nodes.
4863 SDNode *N;
4864 SDVTList VTs = getVTList(VT);
4866 if (VT != MVT::Glue) {
4867 FoldingSetNodeID ID;
4868 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps);
4869 void *IP = nullptr;
4871 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
4872 return SDValue(E, 0);
4874 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4875 VTs, Ops, NumOps);
4876 CSEMap.InsertNode(N, IP);
4877 } else {
4878 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4879 VTs, Ops, NumOps);
4880 }
4882 AllNodes.push_back(N);
4883 #ifndef NDEBUG
4884 VerifySDNode(N);
4885 #endif
4886 return SDValue(N, 0);
4887 }
4889 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
4890 ArrayRef<EVT> ResultTys,
4891 const SDValue *Ops, unsigned NumOps) {
4892 return getNode(Opcode, DL, getVTList(ResultTys),
4893 Ops, NumOps);
4894 }
4896 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
4897 const EVT *VTs, unsigned NumVTs,
4898 const SDValue *Ops, unsigned NumOps) {
4899 if (NumVTs == 1)
4900 return getNode(Opcode, DL, VTs[0], Ops, NumOps);
4901 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps);
4902 }
4904 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
4905 const SDValue *Ops, unsigned NumOps) {
4906 if (VTList.NumVTs == 1)
4907 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps);
4909 #if 0
4910 switch (Opcode) {
4911 // FIXME: figure out how to safely handle things like
4912 // int foo(int x) { return 1 << (x & 255); }
4913 // int bar() { return foo(256); }
4914 case ISD::SRA_PARTS:
4915 case ISD::SRL_PARTS:
4916 case ISD::SHL_PARTS:
4917 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4918 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
4919 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
4920 else if (N3.getOpcode() == ISD::AND)
4921 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
4922 // If the and is only masking out bits that cannot effect the shift,
4923 // eliminate the and.
4924 unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
4925 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
4926 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
4927 }
4928 break;
4929 }
4930 #endif
4932 // Memoize the node unless it returns a flag.
4933 SDNode *N;
4934 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
4935 FoldingSetNodeID ID;
4936 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
4937 void *IP = nullptr;
4938 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
4939 return SDValue(E, 0);
4941 if (NumOps == 1) {
4942 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
4943 DL.getDebugLoc(), VTList, Ops[0]);
4944 } else if (NumOps == 2) {
4945 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
4946 DL.getDebugLoc(), VTList, Ops[0],
4947 Ops[1]);
4948 } else if (NumOps == 3) {
4949 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
4950 DL.getDebugLoc(), VTList, Ops[0],
4951 Ops[1], Ops[2]);
4952 } else {
4953 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4954 VTList, Ops, NumOps);
4955 }
4956 CSEMap.InsertNode(N, IP);
4957 } else {
4958 if (NumOps == 1) {
4959 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
4960 DL.getDebugLoc(), VTList, Ops[0]);
4961 } else if (NumOps == 2) {
4962 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
4963 DL.getDebugLoc(), VTList, Ops[0],
4964 Ops[1]);
4965 } else if (NumOps == 3) {
4966 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
4967 DL.getDebugLoc(), VTList, Ops[0],
4968 Ops[1], Ops[2]);
4969 } else {
4970 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4971 VTList, Ops, NumOps);
4972 }
4973 }
4974 AllNodes.push_back(N);
4975 #ifndef NDEBUG
4976 VerifySDNode(N);
4977 #endif
4978 return SDValue(N, 0);
4979 }
4981 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
4982 return getNode(Opcode, DL, VTList, nullptr, 0);
4983 }
4985 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
4986 SDValue N1) {
4987 SDValue Ops[] = { N1 };
4988 return getNode(Opcode, DL, VTList, Ops, 1);
4989 }
4991 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
4992 SDValue N1, SDValue N2) {
4993 SDValue Ops[] = { N1, N2 };
4994 return getNode(Opcode, DL, VTList, Ops, 2);
4995 }
4997 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
4998 SDValue N1, SDValue N2, SDValue N3) {
4999 SDValue Ops[] = { N1, N2, N3 };
5000 return getNode(Opcode, DL, VTList, Ops, 3);
5001 }
5003 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5004 SDValue N1, SDValue N2, SDValue N3,
5005 SDValue N4) {
5006 SDValue Ops[] = { N1, N2, N3, N4 };
5007 return getNode(Opcode, DL, VTList, Ops, 4);
5008 }
5010 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5011 SDValue N1, SDValue N2, SDValue N3,
5012 SDValue N4, SDValue N5) {
5013 SDValue Ops[] = { N1, N2, N3, N4, N5 };
5014 return getNode(Opcode, DL, VTList, Ops, 5);
5015 }
5017 SDVTList SelectionDAG::getVTList(EVT VT) {
5018 return makeVTList(SDNode::getValueTypeList(VT), 1);
5019 }
5021 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
5022 FoldingSetNodeID ID;
5023 ID.AddInteger(2U);
5024 ID.AddInteger(VT1.getRawBits());
5025 ID.AddInteger(VT2.getRawBits());
5027 void *IP = nullptr;
5028 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5029 if (!Result) {
5030 EVT *Array = Allocator.Allocate<EVT>(2);
5031 Array[0] = VT1;
5032 Array[1] = VT2;
5033 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
5034 VTListMap.InsertNode(Result, IP);
5035 }
5036 return Result->getSDVTList();
5037 }
5039 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
5040 FoldingSetNodeID ID;
5041 ID.AddInteger(3U);
5042 ID.AddInteger(VT1.getRawBits());
5043 ID.AddInteger(VT2.getRawBits());
5044 ID.AddInteger(VT3.getRawBits());
5046 void *IP = nullptr;
5047 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5048 if (!Result) {
5049 EVT *Array = Allocator.Allocate<EVT>(3);
5050 Array[0] = VT1;
5051 Array[1] = VT2;
5052 Array[2] = VT3;
5053 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
5054 VTListMap.InsertNode(Result, IP);
5055 }
5056 return Result->getSDVTList();
5057 }
5059 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
5060 FoldingSetNodeID ID;
5061 ID.AddInteger(4U);
5062 ID.AddInteger(VT1.getRawBits());
5063 ID.AddInteger(VT2.getRawBits());
5064 ID.AddInteger(VT3.getRawBits());
5065 ID.AddInteger(VT4.getRawBits());
5067 void *IP = nullptr;
5068 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5069 if (!Result) {
5070 EVT *Array = Allocator.Allocate<EVT>(4);
5071 Array[0] = VT1;
5072 Array[1] = VT2;
5073 Array[2] = VT3;
5074 Array[3] = VT4;
5075 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
5076 VTListMap.InsertNode(Result, IP);
5077 }
5078 return Result->getSDVTList();
5079 }
5081 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
5082 unsigned NumVTs = VTs.size();
5083 FoldingSetNodeID ID;
5084 ID.AddInteger(NumVTs);
5085 for (unsigned index = 0; index < NumVTs; index++) {
5086 ID.AddInteger(VTs[index].getRawBits());
5087 }
5089 void *IP = nullptr;
5090 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5091 if (!Result) {
5092 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
5093 std::copy(VTs.begin(), VTs.end(), Array);
5094 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
5095 VTListMap.InsertNode(Result, IP);
5096 }
5097 return Result->getSDVTList();
5098 }
5101 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
5102 /// specified operands. If the resultant node already exists in the DAG,
5103 /// this does not modify the specified node, instead it returns the node that
5104 /// already exists. If the resultant node does not exist in the DAG, the
5105 /// input node is returned. As a degenerate case, if you specify the same
5106 /// input operands as the node already has, the input node is returned.
5107 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
5108 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
5110 // Check to see if there is no change.
5111 if (Op == N->getOperand(0)) return N;
5113 // See if the modified node already exists.
5114 void *InsertPos = nullptr;
5115 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
5116 return Existing;
5118 // Nope it doesn't. Remove the node from its current place in the maps.
5119 if (InsertPos)
5120 if (!RemoveNodeFromCSEMaps(N))
5121 InsertPos = nullptr;
5123 // Now we update the operands.
5124 N->OperandList[0].set(Op);
5126 // If this gets put into a CSE map, add it.
5127 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5128 return N;
5129 }
5131 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
5132 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
5134 // Check to see if there is no change.
5135 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
5136 return N; // No operands changed, just return the input node.
5138 // See if the modified node already exists.
5139 void *InsertPos = nullptr;
5140 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
5141 return Existing;
5143 // Nope it doesn't. Remove the node from its current place in the maps.
5144 if (InsertPos)
5145 if (!RemoveNodeFromCSEMaps(N))
5146 InsertPos = nullptr;
5148 // Now we update the operands.
5149 if (N->OperandList[0] != Op1)
5150 N->OperandList[0].set(Op1);
5151 if (N->OperandList[1] != Op2)
5152 N->OperandList[1].set(Op2);
5154 // If this gets put into a CSE map, add it.
5155 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5156 return N;
5157 }
5159 SDNode *SelectionDAG::
5160 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
5161 SDValue Ops[] = { Op1, Op2, Op3 };
5162 return UpdateNodeOperands(N, Ops, 3);
5163 }
5165 SDNode *SelectionDAG::
5166 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
5167 SDValue Op3, SDValue Op4) {
5168 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
5169 return UpdateNodeOperands(N, Ops, 4);
5170 }
5172 SDNode *SelectionDAG::
5173 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
5174 SDValue Op3, SDValue Op4, SDValue Op5) {
5175 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
5176 return UpdateNodeOperands(N, Ops, 5);
5177 }
5179 SDNode *SelectionDAG::
5180 UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) {
5181 assert(N->getNumOperands() == NumOps &&
5182 "Update with wrong number of operands");
5184 // Check to see if there is no change.
5185 bool AnyChange = false;
5186 for (unsigned i = 0; i != NumOps; ++i) {
5187 if (Ops[i] != N->getOperand(i)) {
5188 AnyChange = true;
5189 break;
5190 }
5191 }
5193 // No operands changed, just return the input node.
5194 if (!AnyChange) return N;
5196 // See if the modified node already exists.
5197 void *InsertPos = nullptr;
5198 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos))
5199 return Existing;
5201 // Nope it doesn't. Remove the node from its current place in the maps.
5202 if (InsertPos)
5203 if (!RemoveNodeFromCSEMaps(N))
5204 InsertPos = nullptr;
5206 // Now we update the operands.
5207 for (unsigned i = 0; i != NumOps; ++i)
5208 if (N->OperandList[i] != Ops[i])
5209 N->OperandList[i].set(Ops[i]);
5211 // If this gets put into a CSE map, add it.
5212 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5213 return N;
5214 }
5216 /// DropOperands - Release the operands and set this node to have
5217 /// zero operands.
5218 void SDNode::DropOperands() {
5219 // Unlike the code in MorphNodeTo that does this, we don't need to
5220 // watch for dead nodes here.
5221 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
5222 SDUse &Use = *I++;
5223 Use.set(SDValue());
5224 }
5225 }
5227 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
5228 /// machine opcode.
5229 ///
5230 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5231 EVT VT) {
5232 SDVTList VTs = getVTList(VT);
5233 return SelectNodeTo(N, MachineOpc, VTs, nullptr, 0);
5234 }
5236 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5237 EVT VT, SDValue Op1) {
5238 SDVTList VTs = getVTList(VT);
5239 SDValue Ops[] = { Op1 };
5240 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1);
5241 }
5243 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5244 EVT VT, SDValue Op1,
5245 SDValue Op2) {
5246 SDVTList VTs = getVTList(VT);
5247 SDValue Ops[] = { Op1, Op2 };
5248 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2);
5249 }
5251 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5252 EVT VT, SDValue Op1,
5253 SDValue Op2, SDValue Op3) {
5254 SDVTList VTs = getVTList(VT);
5255 SDValue Ops[] = { Op1, Op2, Op3 };
5256 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
5257 }
5259 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5260 EVT VT, const SDValue *Ops,
5261 unsigned NumOps) {
5262 SDVTList VTs = getVTList(VT);
5263 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
5264 }
5266 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5267 EVT VT1, EVT VT2, const SDValue *Ops,
5268 unsigned NumOps) {
5269 SDVTList VTs = getVTList(VT1, VT2);
5270 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
5271 }
5273 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5274 EVT VT1, EVT VT2) {
5275 SDVTList VTs = getVTList(VT1, VT2);
5276 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)nullptr, 0);
5277 }
5279 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5280 EVT VT1, EVT VT2, EVT VT3,
5281 const SDValue *Ops, unsigned NumOps) {
5282 SDVTList VTs = getVTList(VT1, VT2, VT3);
5283 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
5284 }
5286 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5287 EVT VT1, EVT VT2, EVT VT3, EVT VT4,
5288 const SDValue *Ops, unsigned NumOps) {
5289 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5290 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps);
5291 }
5293 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5294 EVT VT1, EVT VT2,
5295 SDValue Op1) {
5296 SDVTList VTs = getVTList(VT1, VT2);
5297 SDValue Ops[] = { Op1 };
5298 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1);
5299 }
5301 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5302 EVT VT1, EVT VT2,
5303 SDValue Op1, SDValue Op2) {
5304 SDVTList VTs = getVTList(VT1, VT2);
5305 SDValue Ops[] = { Op1, Op2 };
5306 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2);
5307 }
5309 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5310 EVT VT1, EVT VT2,
5311 SDValue Op1, SDValue Op2,
5312 SDValue Op3) {
5313 SDVTList VTs = getVTList(VT1, VT2);
5314 SDValue Ops[] = { Op1, Op2, Op3 };
5315 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
5316 }
5318 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5319 EVT VT1, EVT VT2, EVT VT3,
5320 SDValue Op1, SDValue Op2,
5321 SDValue Op3) {
5322 SDVTList VTs = getVTList(VT1, VT2, VT3);
5323 SDValue Ops[] = { Op1, Op2, Op3 };
5324 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3);
5325 }
5327 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5328 SDVTList VTs, const SDValue *Ops,
5329 unsigned NumOps) {
5330 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps);
5331 // Reset the NodeID to -1.
5332 N->setNodeId(-1);
5333 return N;
5334 }
5336 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
5337 /// the line number information on the merged node since it is not possible to
5338 /// preserve the information that operation is associated with multiple lines.
5339 /// This will make the debugger working better at -O0, were there is a higher
5340 /// probability having other instructions associated with that line.
5341 ///
5342 /// For IROrder, we keep the smaller of the two
5343 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
5344 DebugLoc NLoc = N->getDebugLoc();
5345 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
5346 (OLoc.getDebugLoc() != NLoc)) {
5347 N->setDebugLoc(DebugLoc());
5348 }
5349 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
5350 N->setIROrder(Order);
5351 return N;
5352 }
5354 /// MorphNodeTo - This *mutates* the specified node to have the specified
5355 /// return type, opcode, and operands.
5356 ///
5357 /// Note that MorphNodeTo returns the resultant node. If there is already a
5358 /// node of the specified opcode and operands, it returns that node instead of
5359 /// the current one. Note that the SDLoc need not be the same.
5360 ///
5361 /// Using MorphNodeTo is faster than creating a new node and swapping it in
5362 /// with ReplaceAllUsesWith both because it often avoids allocating a new
5363 /// node, and because it doesn't require CSE recalculation for any of
5364 /// the node's users.
5365 ///
5366 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
5367 SDVTList VTs, const SDValue *Ops,
5368 unsigned NumOps) {
5369 // If an identical node already exists, use it.
5370 void *IP = nullptr;
5371 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
5372 FoldingSetNodeID ID;
5373 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps);
5374 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
5375 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
5376 }
5378 if (!RemoveNodeFromCSEMaps(N))
5379 IP = nullptr;
5381 // Start the morphing.
5382 N->NodeType = Opc;
5383 N->ValueList = VTs.VTs;
5384 N->NumValues = VTs.NumVTs;
5386 // Clear the operands list, updating used nodes to remove this from their
5387 // use list. Keep track of any operands that become dead as a result.
5388 SmallPtrSet<SDNode*, 16> DeadNodeSet;
5389 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
5390 SDUse &Use = *I++;
5391 SDNode *Used = Use.getNode();
5392 Use.set(SDValue());
5393 if (Used->use_empty())
5394 DeadNodeSet.insert(Used);
5395 }
5397 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
5398 // Initialize the memory references information.
5399 MN->setMemRefs(nullptr, nullptr);
5400 // If NumOps is larger than the # of operands we can have in a
5401 // MachineSDNode, reallocate the operand list.
5402 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
5403 if (MN->OperandsNeedDelete)
5404 delete[] MN->OperandList;
5405 if (NumOps > array_lengthof(MN->LocalOperands))
5406 // We're creating a final node that will live unmorphed for the
5407 // remainder of the current SelectionDAG iteration, so we can allocate
5408 // the operands directly out of a pool with no recycling metadata.
5409 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
5410 Ops, NumOps);
5411 else
5412 MN->InitOperands(MN->LocalOperands, Ops, NumOps);
5413 MN->OperandsNeedDelete = false;
5414 } else
5415 MN->InitOperands(MN->OperandList, Ops, NumOps);
5416 } else {
5417 // If NumOps is larger than the # of operands we currently have, reallocate
5418 // the operand list.
5419 if (NumOps > N->NumOperands) {
5420 if (N->OperandsNeedDelete)
5421 delete[] N->OperandList;
5422 N->InitOperands(new SDUse[NumOps], Ops, NumOps);
5423 N->OperandsNeedDelete = true;
5424 } else
5425 N->InitOperands(N->OperandList, Ops, NumOps);
5426 }
5428 // Delete any nodes that are still dead after adding the uses for the
5429 // new operands.
5430 if (!DeadNodeSet.empty()) {
5431 SmallVector<SDNode *, 16> DeadNodes;
5432 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(),
5433 E = DeadNodeSet.end(); I != E; ++I)
5434 if ((*I)->use_empty())
5435 DeadNodes.push_back(*I);
5436 RemoveDeadNodes(DeadNodes);
5437 }
5439 if (IP)
5440 CSEMap.InsertNode(N, IP); // Memoize the new node.
5441 return N;
5442 }
5445 /// getMachineNode - These are used for target selectors to create a new node
5446 /// with specified return type(s), MachineInstr opcode, and operands.
5447 ///
5448 /// Note that getMachineNode returns the resultant node. If there is already a
5449 /// node of the specified opcode and operands, it returns that node instead of
5450 /// the current one.
5451 MachineSDNode *
5452 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
5453 SDVTList VTs = getVTList(VT);
5454 return getMachineNode(Opcode, dl, VTs, None);
5455 }
5457 MachineSDNode *
5458 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
5459 SDVTList VTs = getVTList(VT);
5460 SDValue Ops[] = { Op1 };
5461 return getMachineNode(Opcode, dl, VTs, Ops);
5462 }
5464 MachineSDNode *
5465 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5466 SDValue Op1, SDValue Op2) {
5467 SDVTList VTs = getVTList(VT);
5468 SDValue Ops[] = { Op1, Op2 };
5469 return getMachineNode(Opcode, dl, VTs, Ops);
5470 }
5472 MachineSDNode *
5473 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5474 SDValue Op1, SDValue Op2, SDValue Op3) {
5475 SDVTList VTs = getVTList(VT);
5476 SDValue Ops[] = { Op1, Op2, Op3 };
5477 return getMachineNode(Opcode, dl, VTs, Ops);
5478 }
5480 MachineSDNode *
5481 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5482 ArrayRef<SDValue> Ops) {
5483 SDVTList VTs = getVTList(VT);
5484 return getMachineNode(Opcode, dl, VTs, Ops);
5485 }
5487 MachineSDNode *
5488 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
5489 SDVTList VTs = getVTList(VT1, VT2);
5490 return getMachineNode(Opcode, dl, VTs, None);
5491 }
5493 MachineSDNode *
5494 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5495 EVT VT1, EVT VT2, SDValue Op1) {
5496 SDVTList VTs = getVTList(VT1, VT2);
5497 SDValue Ops[] = { Op1 };
5498 return getMachineNode(Opcode, dl, VTs, Ops);
5499 }
5501 MachineSDNode *
5502 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5503 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
5504 SDVTList VTs = getVTList(VT1, VT2);
5505 SDValue Ops[] = { Op1, Op2 };
5506 return getMachineNode(Opcode, dl, VTs, Ops);
5507 }
5509 MachineSDNode *
5510 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5511 EVT VT1, EVT VT2, SDValue Op1,
5512 SDValue Op2, SDValue Op3) {
5513 SDVTList VTs = getVTList(VT1, VT2);
5514 SDValue Ops[] = { Op1, Op2, Op3 };
5515 return getMachineNode(Opcode, dl, VTs, Ops);
5516 }
5518 MachineSDNode *
5519 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5520 EVT VT1, EVT VT2,
5521 ArrayRef<SDValue> Ops) {
5522 SDVTList VTs = getVTList(VT1, VT2);
5523 return getMachineNode(Opcode, dl, VTs, Ops);
5524 }
5526 MachineSDNode *
5527 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5528 EVT VT1, EVT VT2, EVT VT3,
5529 SDValue Op1, SDValue Op2) {
5530 SDVTList VTs = getVTList(VT1, VT2, VT3);
5531 SDValue Ops[] = { Op1, Op2 };
5532 return getMachineNode(Opcode, dl, VTs, Ops);
5533 }
5535 MachineSDNode *
5536 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5537 EVT VT1, EVT VT2, EVT VT3,
5538 SDValue Op1, SDValue Op2, SDValue Op3) {
5539 SDVTList VTs = getVTList(VT1, VT2, VT3);
5540 SDValue Ops[] = { Op1, Op2, Op3 };
5541 return getMachineNode(Opcode, dl, VTs, Ops);
5542 }
5544 MachineSDNode *
5545 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5546 EVT VT1, EVT VT2, EVT VT3,
5547 ArrayRef<SDValue> Ops) {
5548 SDVTList VTs = getVTList(VT1, VT2, VT3);
5549 return getMachineNode(Opcode, dl, VTs, Ops);
5550 }
5552 MachineSDNode *
5553 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
5554 EVT VT2, EVT VT3, EVT VT4,
5555 ArrayRef<SDValue> Ops) {
5556 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5557 return getMachineNode(Opcode, dl, VTs, Ops);
5558 }
5560 MachineSDNode *
5561 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
5562 ArrayRef<EVT> ResultTys,
5563 ArrayRef<SDValue> Ops) {
5564 SDVTList VTs = getVTList(ResultTys);
5565 return getMachineNode(Opcode, dl, VTs, Ops);
5566 }
5568 MachineSDNode *
5569 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
5570 ArrayRef<SDValue> OpsArray) {
5571 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
5572 MachineSDNode *N;
5573 void *IP = nullptr;
5574 const SDValue *Ops = OpsArray.data();
5575 unsigned NumOps = OpsArray.size();
5577 if (DoCSE) {
5578 FoldingSetNodeID ID;
5579 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps);
5580 IP = nullptr;
5581 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
5582 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
5583 }
5584 }
5586 // Allocate a new MachineSDNode.
5587 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
5588 DL.getDebugLoc(), VTs);
5590 // Initialize the operands list.
5591 if (NumOps > array_lengthof(N->LocalOperands))
5592 // We're creating a final node that will live unmorphed for the
5593 // remainder of the current SelectionDAG iteration, so we can allocate
5594 // the operands directly out of a pool with no recycling metadata.
5595 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
5596 Ops, NumOps);
5597 else
5598 N->InitOperands(N->LocalOperands, Ops, NumOps);
5599 N->OperandsNeedDelete = false;
5601 if (DoCSE)
5602 CSEMap.InsertNode(N, IP);
5604 AllNodes.push_back(N);
5605 #ifndef NDEBUG
5606 VerifyMachineNode(N);
5607 #endif
5608 return N;
5609 }
5611 /// getTargetExtractSubreg - A convenience function for creating
5612 /// TargetOpcode::EXTRACT_SUBREG nodes.
5613 SDValue
5614 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
5615 SDValue Operand) {
5616 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
5617 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
5618 VT, Operand, SRIdxVal);
5619 return SDValue(Subreg, 0);
5620 }
5622 /// getTargetInsertSubreg - A convenience function for creating
5623 /// TargetOpcode::INSERT_SUBREG nodes.
5624 SDValue
5625 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
5626 SDValue Operand, SDValue Subreg) {
5627 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
5628 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
5629 VT, Operand, Subreg, SRIdxVal);
5630 return SDValue(Result, 0);
5631 }
5633 /// getNodeIfExists - Get the specified node if it's already available, or
5634 /// else return NULL.
5635 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
5636 const SDValue *Ops, unsigned NumOps) {
5637 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
5638 FoldingSetNodeID ID;
5639 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
5640 void *IP = nullptr;
5641 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
5642 return E;
5643 }
5644 return nullptr;
5645 }
5647 /// getDbgValue - Creates a SDDbgValue node.
5648 ///
5649 /// SDNode
5650 SDDbgValue *
5651 SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R,
5652 bool IsIndirect, uint64_t Off,
5653 DebugLoc DL, unsigned O) {
5654 return new (Allocator) SDDbgValue(MDPtr, N, R, IsIndirect, Off, DL, O);
5655 }
5657 /// Constant
5658 SDDbgValue *
5659 SelectionDAG::getConstantDbgValue(MDNode *MDPtr, const Value *C,
5660 uint64_t Off,
5661 DebugLoc DL, unsigned O) {
5662 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O);
5663 }
5665 /// FrameIndex
5666 SDDbgValue *
5667 SelectionDAG::getFrameIndexDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
5668 DebugLoc DL, unsigned O) {
5669 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O);
5670 }
5672 namespace {
5674 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
5675 /// pointed to by a use iterator is deleted, increment the use iterator
5676 /// so that it doesn't dangle.
5677 ///
5678 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
5679 SDNode::use_iterator &UI;
5680 SDNode::use_iterator &UE;
5682 void NodeDeleted(SDNode *N, SDNode *E) override {
5683 // Increment the iterator as needed.
5684 while (UI != UE && N == *UI)
5685 ++UI;
5686 }
5688 public:
5689 RAUWUpdateListener(SelectionDAG &d,
5690 SDNode::use_iterator &ui,
5691 SDNode::use_iterator &ue)
5692 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
5693 };
5695 }
5697 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5698 /// This can cause recursive merging of nodes in the DAG.
5699 ///
5700 /// This version assumes From has a single result value.
5701 ///
5702 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
5703 SDNode *From = FromN.getNode();
5704 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
5705 "Cannot replace with this method!");
5706 assert(From != To.getNode() && "Cannot replace uses of with self");
5708 // Iterate over all the existing uses of From. New uses will be added
5709 // to the beginning of the use list, which we avoid visiting.
5710 // This specifically avoids visiting uses of From that arise while the
5711 // replacement is happening, because any such uses would be the result
5712 // of CSE: If an existing node looks like From after one of its operands
5713 // is replaced by To, we don't want to replace of all its users with To
5714 // too. See PR3018 for more info.
5715 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
5716 RAUWUpdateListener Listener(*this, UI, UE);
5717 while (UI != UE) {
5718 SDNode *User = *UI;
5720 // This node is about to morph, remove its old self from the CSE maps.
5721 RemoveNodeFromCSEMaps(User);
5723 // A user can appear in a use list multiple times, and when this
5724 // happens the uses are usually next to each other in the list.
5725 // To help reduce the number of CSE recomputations, process all
5726 // the uses of this user that we can find this way.
5727 do {
5728 SDUse &Use = UI.getUse();
5729 ++UI;
5730 Use.set(To);
5731 } while (UI != UE && *UI == User);
5733 // Now that we have modified User, add it back to the CSE maps. If it
5734 // already exists there, recursively merge the results together.
5735 AddModifiedNodeToCSEMaps(User);
5736 }
5738 // If we just RAUW'd the root, take note.
5739 if (FromN == getRoot())
5740 setRoot(To);
5741 }
5743 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5744 /// This can cause recursive merging of nodes in the DAG.
5745 ///
5746 /// This version assumes that for each value of From, there is a
5747 /// corresponding value in To in the same position with the same type.
5748 ///
5749 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
5750 #ifndef NDEBUG
5751 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
5752 assert((!From->hasAnyUseOfValue(i) ||
5753 From->getValueType(i) == To->getValueType(i)) &&
5754 "Cannot use this version of ReplaceAllUsesWith!");
5755 #endif
5757 // Handle the trivial case.
5758 if (From == To)
5759 return;
5761 // Iterate over just the existing users of From. See the comments in
5762 // the ReplaceAllUsesWith above.
5763 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
5764 RAUWUpdateListener Listener(*this, UI, UE);
5765 while (UI != UE) {
5766 SDNode *User = *UI;
5768 // This node is about to morph, remove its old self from the CSE maps.
5769 RemoveNodeFromCSEMaps(User);
5771 // A user can appear in a use list multiple times, and when this
5772 // happens the uses are usually next to each other in the list.
5773 // To help reduce the number of CSE recomputations, process all
5774 // the uses of this user that we can find this way.
5775 do {
5776 SDUse &Use = UI.getUse();
5777 ++UI;
5778 Use.setNode(To);
5779 } while (UI != UE && *UI == User);
5781 // Now that we have modified User, add it back to the CSE maps. If it
5782 // already exists there, recursively merge the results together.
5783 AddModifiedNodeToCSEMaps(User);
5784 }
5786 // If we just RAUW'd the root, take note.
5787 if (From == getRoot().getNode())
5788 setRoot(SDValue(To, getRoot().getResNo()));
5789 }
5791 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
5792 /// This can cause recursive merging of nodes in the DAG.
5793 ///
5794 /// This version can replace From with any result values. To must match the
5795 /// number and types of values returned by From.
5796 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
5797 if (From->getNumValues() == 1) // Handle the simple case efficiently.
5798 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
5800 // Iterate over just the existing users of From. See the comments in
5801 // the ReplaceAllUsesWith above.
5802 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
5803 RAUWUpdateListener Listener(*this, UI, UE);
5804 while (UI != UE) {
5805 SDNode *User = *UI;
5807 // This node is about to morph, remove its old self from the CSE maps.
5808 RemoveNodeFromCSEMaps(User);
5810 // A user can appear in a use list multiple times, and when this
5811 // happens the uses are usually next to each other in the list.
5812 // To help reduce the number of CSE recomputations, process all
5813 // the uses of this user that we can find this way.
5814 do {
5815 SDUse &Use = UI.getUse();
5816 const SDValue &ToOp = To[Use.getResNo()];
5817 ++UI;
5818 Use.set(ToOp);
5819 } while (UI != UE && *UI == User);
5821 // Now that we have modified User, add it back to the CSE maps. If it
5822 // already exists there, recursively merge the results together.
5823 AddModifiedNodeToCSEMaps(User);
5824 }
5826 // If we just RAUW'd the root, take note.
5827 if (From == getRoot().getNode())
5828 setRoot(SDValue(To[getRoot().getResNo()]));
5829 }
5831 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
5832 /// uses of other values produced by From.getNode() alone. The Deleted
5833 /// vector is handled the same way as for ReplaceAllUsesWith.
5834 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
5835 // Handle the really simple, really trivial case efficiently.
5836 if (From == To) return;
5838 // Handle the simple, trivial, case efficiently.
5839 if (From.getNode()->getNumValues() == 1) {
5840 ReplaceAllUsesWith(From, To);
5841 return;
5842 }
5844 // Iterate over just the existing users of From. See the comments in
5845 // the ReplaceAllUsesWith above.
5846 SDNode::use_iterator UI = From.getNode()->use_begin(),
5847 UE = From.getNode()->use_end();
5848 RAUWUpdateListener Listener(*this, UI, UE);
5849 while (UI != UE) {
5850 SDNode *User = *UI;
5851 bool UserRemovedFromCSEMaps = false;
5853 // A user can appear in a use list multiple times, and when this
5854 // happens the uses are usually next to each other in the list.
5855 // To help reduce the number of CSE recomputations, process all
5856 // the uses of this user that we can find this way.
5857 do {
5858 SDUse &Use = UI.getUse();
5860 // Skip uses of different values from the same node.
5861 if (Use.getResNo() != From.getResNo()) {
5862 ++UI;
5863 continue;
5864 }
5866 // If this node hasn't been modified yet, it's still in the CSE maps,
5867 // so remove its old self from the CSE maps.
5868 if (!UserRemovedFromCSEMaps) {
5869 RemoveNodeFromCSEMaps(User);
5870 UserRemovedFromCSEMaps = true;
5871 }
5873 ++UI;
5874 Use.set(To);
5875 } while (UI != UE && *UI == User);
5877 // We are iterating over all uses of the From node, so if a use
5878 // doesn't use the specific value, no changes are made.
5879 if (!UserRemovedFromCSEMaps)
5880 continue;
5882 // Now that we have modified User, add it back to the CSE maps. If it
5883 // already exists there, recursively merge the results together.
5884 AddModifiedNodeToCSEMaps(User);
5885 }
5887 // If we just RAUW'd the root, take note.
5888 if (From == getRoot())
5889 setRoot(To);
5890 }
5892 namespace {
5893 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
5894 /// to record information about a use.
5895 struct UseMemo {
5896 SDNode *User;
5897 unsigned Index;
5898 SDUse *Use;
5899 };
5901 /// operator< - Sort Memos by User.
5902 bool operator<(const UseMemo &L, const UseMemo &R) {
5903 return (intptr_t)L.User < (intptr_t)R.User;
5904 }
5905 }
5907 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
5908 /// uses of other values produced by From.getNode() alone. The same value
5909 /// may appear in both the From and To list. The Deleted vector is
5910 /// handled the same way as for ReplaceAllUsesWith.
5911 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
5912 const SDValue *To,
5913 unsigned Num){
5914 // Handle the simple, trivial case efficiently.
5915 if (Num == 1)
5916 return ReplaceAllUsesOfValueWith(*From, *To);
5918 // Read up all the uses and make records of them. This helps
5919 // processing new uses that are introduced during the
5920 // replacement process.
5921 SmallVector<UseMemo, 4> Uses;
5922 for (unsigned i = 0; i != Num; ++i) {
5923 unsigned FromResNo = From[i].getResNo();
5924 SDNode *FromNode = From[i].getNode();
5925 for (SDNode::use_iterator UI = FromNode->use_begin(),
5926 E = FromNode->use_end(); UI != E; ++UI) {
5927 SDUse &Use = UI.getUse();
5928 if (Use.getResNo() == FromResNo) {
5929 UseMemo Memo = { *UI, i, &Use };
5930 Uses.push_back(Memo);
5931 }
5932 }
5933 }
5935 // Sort the uses, so that all the uses from a given User are together.
5936 std::sort(Uses.begin(), Uses.end());
5938 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
5939 UseIndex != UseIndexEnd; ) {
5940 // We know that this user uses some value of From. If it is the right
5941 // value, update it.
5942 SDNode *User = Uses[UseIndex].User;
5944 // This node is about to morph, remove its old self from the CSE maps.
5945 RemoveNodeFromCSEMaps(User);
5947 // The Uses array is sorted, so all the uses for a given User
5948 // are next to each other in the list.
5949 // To help reduce the number of CSE recomputations, process all
5950 // the uses of this user that we can find this way.
5951 do {
5952 unsigned i = Uses[UseIndex].Index;
5953 SDUse &Use = *Uses[UseIndex].Use;
5954 ++UseIndex;
5956 Use.set(To[i]);
5957 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
5959 // Now that we have modified User, add it back to the CSE maps. If it
5960 // already exists there, recursively merge the results together.
5961 AddModifiedNodeToCSEMaps(User);
5962 }
5963 }
5965 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
5966 /// based on their topological order. It returns the maximum id and a vector
5967 /// of the SDNodes* in assigned order by reference.
5968 unsigned SelectionDAG::AssignTopologicalOrder() {
5970 unsigned DAGSize = 0;
5972 // SortedPos tracks the progress of the algorithm. Nodes before it are
5973 // sorted, nodes after it are unsorted. When the algorithm completes
5974 // it is at the end of the list.
5975 allnodes_iterator SortedPos = allnodes_begin();
5977 // Visit all the nodes. Move nodes with no operands to the front of
5978 // the list immediately. Annotate nodes that do have operands with their
5979 // operand count. Before we do this, the Node Id fields of the nodes
5980 // may contain arbitrary values. After, the Node Id fields for nodes
5981 // before SortedPos will contain the topological sort index, and the
5982 // Node Id fields for nodes At SortedPos and after will contain the
5983 // count of outstanding operands.
5984 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
5985 SDNode *N = I++;
5986 checkForCycles(N);
5987 unsigned Degree = N->getNumOperands();
5988 if (Degree == 0) {
5989 // A node with no uses, add it to the result array immediately.
5990 N->setNodeId(DAGSize++);
5991 allnodes_iterator Q = N;
5992 if (Q != SortedPos)
5993 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
5994 assert(SortedPos != AllNodes.end() && "Overran node list");
5995 ++SortedPos;
5996 } else {
5997 // Temporarily use the Node Id as scratch space for the degree count.
5998 N->setNodeId(Degree);
5999 }
6000 }
6002 // Visit all the nodes. As we iterate, move nodes into sorted order,
6003 // such that by the time the end is reached all nodes will be sorted.
6004 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
6005 SDNode *N = I;
6006 checkForCycles(N);
6007 // N is in sorted position, so all its uses have one less operand
6008 // that needs to be sorted.
6009 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
6010 UI != UE; ++UI) {
6011 SDNode *P = *UI;
6012 unsigned Degree = P->getNodeId();
6013 assert(Degree != 0 && "Invalid node degree");
6014 --Degree;
6015 if (Degree == 0) {
6016 // All of P's operands are sorted, so P may sorted now.
6017 P->setNodeId(DAGSize++);
6018 if (P != SortedPos)
6019 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
6020 assert(SortedPos != AllNodes.end() && "Overran node list");
6021 ++SortedPos;
6022 } else {
6023 // Update P's outstanding operand count.
6024 P->setNodeId(Degree);
6025 }
6026 }
6027 if (I == SortedPos) {
6028 #ifndef NDEBUG
6029 SDNode *S = ++I;
6030 dbgs() << "Overran sorted position:\n";
6031 S->dumprFull();
6032 #endif
6033 llvm_unreachable(nullptr);
6034 }
6035 }
6037 assert(SortedPos == AllNodes.end() &&
6038 "Topological sort incomplete!");
6039 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
6040 "First node in topological sort is not the entry token!");
6041 assert(AllNodes.front().getNodeId() == 0 &&
6042 "First node in topological sort has non-zero id!");
6043 assert(AllNodes.front().getNumOperands() == 0 &&
6044 "First node in topological sort has operands!");
6045 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
6046 "Last node in topologic sort has unexpected id!");
6047 assert(AllNodes.back().use_empty() &&
6048 "Last node in topologic sort has users!");
6049 assert(DAGSize == allnodes_size() && "Node count mismatch!");
6050 return DAGSize;
6051 }
6053 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
6054 /// value is produced by SD.
6055 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
6056 DbgInfo->add(DB, SD, isParameter);
6057 if (SD)
6058 SD->setHasDebugValue(true);
6059 }
6061 /// TransferDbgValues - Transfer SDDbgValues.
6062 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
6063 if (From == To || !From.getNode()->getHasDebugValue())
6064 return;
6065 SDNode *FromNode = From.getNode();
6066 SDNode *ToNode = To.getNode();
6067 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
6068 SmallVector<SDDbgValue *, 2> ClonedDVs;
6069 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
6070 I != E; ++I) {
6071 SDDbgValue *Dbg = *I;
6072 if (Dbg->getKind() == SDDbgValue::SDNODE) {
6073 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(),
6074 Dbg->isIndirect(),
6075 Dbg->getOffset(), Dbg->getDebugLoc(),
6076 Dbg->getOrder());
6077 ClonedDVs.push_back(Clone);
6078 }
6079 }
6080 for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
6081 E = ClonedDVs.end(); I != E; ++I)
6082 AddDbgValue(*I, ToNode, false);
6083 }
6085 //===----------------------------------------------------------------------===//
6086 // SDNode Class
6087 //===----------------------------------------------------------------------===//
6089 HandleSDNode::~HandleSDNode() {
6090 DropOperands();
6091 }
6093 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
6094 DebugLoc DL, const GlobalValue *GA,
6095 EVT VT, int64_t o, unsigned char TF)
6096 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
6097 TheGlobal = GA;
6098 }
6100 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
6101 SDValue X, unsigned SrcAS,
6102 unsigned DestAS)
6103 : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
6104 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
6106 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6107 EVT memvt, MachineMemOperand *mmo)
6108 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
6109 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
6110 MMO->isNonTemporal(), MMO->isInvariant());
6111 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6112 assert(isNonTemporal() == MMO->isNonTemporal() &&
6113 "Non-temporal encoding error!");
6114 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
6115 }
6117 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6118 const SDValue *Ops, unsigned NumOps, EVT memvt,
6119 MachineMemOperand *mmo)
6120 : SDNode(Opc, Order, dl, VTs, Ops, NumOps),
6121 MemoryVT(memvt), MMO(mmo) {
6122 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
6123 MMO->isNonTemporal(), MMO->isInvariant());
6124 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6125 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
6126 }
6128 /// Profile - Gather unique data for the node.
6129 ///
6130 void SDNode::Profile(FoldingSetNodeID &ID) const {
6131 AddNodeIDNode(ID, this);
6132 }
6134 namespace {
6135 struct EVTArray {
6136 std::vector<EVT> VTs;
6138 EVTArray() {
6139 VTs.reserve(MVT::LAST_VALUETYPE);
6140 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
6141 VTs.push_back(MVT((MVT::SimpleValueType)i));
6142 }
6143 };
6144 }
6146 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
6147 static ManagedStatic<EVTArray> SimpleVTArray;
6148 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
6150 /// getValueTypeList - Return a pointer to the specified value type.
6151 ///
6152 const EVT *SDNode::getValueTypeList(EVT VT) {
6153 if (VT.isExtended()) {
6154 sys::SmartScopedLock<true> Lock(*VTMutex);
6155 return &(*EVTs->insert(VT).first);
6156 } else {
6157 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
6158 "Value type out of range!");
6159 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
6160 }
6161 }
6163 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
6164 /// indicated value. This method ignores uses of other values defined by this
6165 /// operation.
6166 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
6167 assert(Value < getNumValues() && "Bad value!");
6169 // TODO: Only iterate over uses of a given value of the node
6170 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
6171 if (UI.getUse().getResNo() == Value) {
6172 if (NUses == 0)
6173 return false;
6174 --NUses;
6175 }
6176 }
6178 // Found exactly the right number of uses?
6179 return NUses == 0;
6180 }
6183 /// hasAnyUseOfValue - Return true if there are any use of the indicated
6184 /// value. This method ignores uses of other values defined by this operation.
6185 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
6186 assert(Value < getNumValues() && "Bad value!");
6188 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
6189 if (UI.getUse().getResNo() == Value)
6190 return true;
6192 return false;
6193 }
6196 /// isOnlyUserOf - Return true if this node is the only use of N.
6197 ///
6198 bool SDNode::isOnlyUserOf(SDNode *N) const {
6199 bool Seen = false;
6200 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
6201 SDNode *User = *I;
6202 if (User == this)
6203 Seen = true;
6204 else
6205 return false;
6206 }
6208 return Seen;
6209 }
6211 /// isOperand - Return true if this node is an operand of N.
6212 ///
6213 bool SDValue::isOperandOf(SDNode *N) const {
6214 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
6215 if (*this == N->getOperand(i))
6216 return true;
6217 return false;
6218 }
6220 bool SDNode::isOperandOf(SDNode *N) const {
6221 for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
6222 if (this == N->OperandList[i].getNode())
6223 return true;
6224 return false;
6225 }
6227 /// reachesChainWithoutSideEffects - Return true if this operand (which must
6228 /// be a chain) reaches the specified operand without crossing any
6229 /// side-effecting instructions on any chain path. In practice, this looks
6230 /// through token factors and non-volatile loads. In order to remain efficient,
6231 /// this only looks a couple of nodes in, it does not do an exhaustive search.
6232 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
6233 unsigned Depth) const {
6234 if (*this == Dest) return true;
6236 // Don't search too deeply, we just want to be able to see through
6237 // TokenFactor's etc.
6238 if (Depth == 0) return false;
6240 // If this is a token factor, all inputs to the TF happen in parallel. If any
6241 // of the operands of the TF does not reach dest, then we cannot do the xform.
6242 if (getOpcode() == ISD::TokenFactor) {
6243 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
6244 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1))
6245 return false;
6246 return true;
6247 }
6249 // Loads don't have side effects, look through them.
6250 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
6251 if (!Ld->isVolatile())
6252 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
6253 }
6254 return false;
6255 }
6257 /// hasPredecessor - Return true if N is a predecessor of this node.
6258 /// N is either an operand of this node, or can be reached by recursively
6259 /// traversing up the operands.
6260 /// NOTE: This is an expensive method. Use it carefully.
6261 bool SDNode::hasPredecessor(const SDNode *N) const {
6262 SmallPtrSet<const SDNode *, 32> Visited;
6263 SmallVector<const SDNode *, 16> Worklist;
6264 return hasPredecessorHelper(N, Visited, Worklist);
6265 }
6267 bool
6268 SDNode::hasPredecessorHelper(const SDNode *N,
6269 SmallPtrSet<const SDNode *, 32> &Visited,
6270 SmallVectorImpl<const SDNode *> &Worklist) const {
6271 if (Visited.empty()) {
6272 Worklist.push_back(this);
6273 } else {
6274 // Take a look in the visited set. If we've already encountered this node
6275 // we needn't search further.
6276 if (Visited.count(N))
6277 return true;
6278 }
6280 // Haven't visited N yet. Continue the search.
6281 while (!Worklist.empty()) {
6282 const SDNode *M = Worklist.pop_back_val();
6283 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
6284 SDNode *Op = M->getOperand(i).getNode();
6285 if (Visited.insert(Op))
6286 Worklist.push_back(Op);
6287 if (Op == N)
6288 return true;
6289 }
6290 }
6292 return false;
6293 }
6295 uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
6296 assert(Num < NumOperands && "Invalid child # of SDNode!");
6297 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
6298 }
6300 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
6301 assert(N->getNumValues() == 1 &&
6302 "Can't unroll a vector with multiple results!");
6304 EVT VT = N->getValueType(0);
6305 unsigned NE = VT.getVectorNumElements();
6306 EVT EltVT = VT.getVectorElementType();
6307 SDLoc dl(N);
6309 SmallVector<SDValue, 8> Scalars;
6310 SmallVector<SDValue, 4> Operands(N->getNumOperands());
6312 // If ResNE is 0, fully unroll the vector op.
6313 if (ResNE == 0)
6314 ResNE = NE;
6315 else if (NE > ResNE)
6316 NE = ResNE;
6318 unsigned i;
6319 for (i= 0; i != NE; ++i) {
6320 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
6321 SDValue Operand = N->getOperand(j);
6322 EVT OperandVT = Operand.getValueType();
6323 if (OperandVT.isVector()) {
6324 // A vector operand; extract a single element.
6325 const TargetLowering *TLI = TM.getTargetLowering();
6326 EVT OperandEltVT = OperandVT.getVectorElementType();
6327 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl,
6328 OperandEltVT,
6329 Operand,
6330 getConstant(i, TLI->getVectorIdxTy()));
6331 } else {
6332 // A scalar operand; just use it as is.
6333 Operands[j] = Operand;
6334 }
6335 }
6337 switch (N->getOpcode()) {
6338 default:
6339 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
6340 &Operands[0], Operands.size()));
6341 break;
6342 case ISD::VSELECT:
6343 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT,
6344 &Operands[0], Operands.size()));
6345 break;
6346 case ISD::SHL:
6347 case ISD::SRA:
6348 case ISD::SRL:
6349 case ISD::ROTL:
6350 case ISD::ROTR:
6351 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
6352 getShiftAmountOperand(Operands[0].getValueType(),
6353 Operands[1])));
6354 break;
6355 case ISD::SIGN_EXTEND_INREG:
6356 case ISD::FP_ROUND_INREG: {
6357 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
6358 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
6359 Operands[0],
6360 getValueType(ExtVT)));
6361 }
6362 }
6363 }
6365 for (; i < ResNE; ++i)
6366 Scalars.push_back(getUNDEF(EltVT));
6368 return getNode(ISD::BUILD_VECTOR, dl,
6369 EVT::getVectorVT(*getContext(), EltVT, ResNE),
6370 &Scalars[0], Scalars.size());
6371 }
6374 /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a
6375 /// location that is 'Dist' units away from the location that the 'Base' load
6376 /// is loading from.
6377 bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base,
6378 unsigned Bytes, int Dist) const {
6379 if (LD->getChain() != Base->getChain())
6380 return false;
6381 EVT VT = LD->getValueType(0);
6382 if (VT.getSizeInBits() / 8 != Bytes)
6383 return false;
6385 SDValue Loc = LD->getOperand(1);
6386 SDValue BaseLoc = Base->getOperand(1);
6387 if (Loc.getOpcode() == ISD::FrameIndex) {
6388 if (BaseLoc.getOpcode() != ISD::FrameIndex)
6389 return false;
6390 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo();
6391 int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
6392 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
6393 int FS = MFI->getObjectSize(FI);
6394 int BFS = MFI->getObjectSize(BFI);
6395 if (FS != BFS || FS != (int)Bytes) return false;
6396 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
6397 }
6399 // Handle X+C
6400 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc &&
6401 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes)
6402 return true;
6404 const GlobalValue *GV1 = nullptr;
6405 const GlobalValue *GV2 = nullptr;
6406 int64_t Offset1 = 0;
6407 int64_t Offset2 = 0;
6408 const TargetLowering *TLI = TM.getTargetLowering();
6409 bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1);
6410 bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
6411 if (isGA1 && isGA2 && GV1 == GV2)
6412 return Offset1 == (Offset2 + Dist*Bytes);
6413 return false;
6414 }
6417 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
6418 /// it cannot be inferred.
6419 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
6420 // If this is a GlobalAddress + cst, return the alignment.
6421 const GlobalValue *GV;
6422 int64_t GVOffset = 0;
6423 const TargetLowering *TLI = TM.getTargetLowering();
6424 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
6425 unsigned PtrWidth = TLI->getPointerTypeSizeInBits(GV->getType());
6426 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
6427 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne,
6428 TLI->getDataLayout());
6429 unsigned AlignBits = KnownZero.countTrailingOnes();
6430 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
6431 if (Align)
6432 return MinAlign(Align, GVOffset);
6433 }
6435 // If this is a direct reference to a stack slot, use information about the
6436 // stack slot's alignment.
6437 int FrameIdx = 1 << 31;
6438 int64_t FrameOffset = 0;
6439 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
6440 FrameIdx = FI->getIndex();
6441 } else if (isBaseWithConstantOffset(Ptr) &&
6442 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
6443 // Handle FI+Cst
6444 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6445 FrameOffset = Ptr.getConstantOperandVal(1);
6446 }
6448 if (FrameIdx != (1 << 31)) {
6449 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo();
6450 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
6451 FrameOffset);
6452 return FIInfoAlign;
6453 }
6455 return 0;
6456 }
6458 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
6459 /// which is split (or expanded) into two not necessarily identical pieces.
6460 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
6461 // Currently all types are split in half.
6462 EVT LoVT, HiVT;
6463 if (!VT.isVector()) {
6464 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
6465 } else {
6466 unsigned NumElements = VT.getVectorNumElements();
6467 assert(!(NumElements & 1) && "Splitting vector, but not in half!");
6468 LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
6469 NumElements/2);
6470 }
6471 return std::make_pair(LoVT, HiVT);
6472 }
6474 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
6475 /// low/high part.
6476 std::pair<SDValue, SDValue>
6477 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
6478 const EVT &HiVT) {
6479 assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
6480 N.getValueType().getVectorNumElements() &&
6481 "More vector elements requested than available!");
6482 SDValue Lo, Hi;
6483 Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
6484 getConstant(0, TLI->getVectorIdxTy()));
6485 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
6486 getConstant(LoVT.getVectorNumElements(), TLI->getVectorIdxTy()));
6487 return std::make_pair(Lo, Hi);
6488 }
6490 void SelectionDAG::ExtractVectorElements(SDValue Op,
6491 SmallVectorImpl<SDValue> &Args,
6492 unsigned Start, unsigned Count) {
6493 EVT VT = Op.getValueType();
6494 if (Count == 0)
6495 Count = VT.getVectorNumElements();
6497 EVT EltVT = VT.getVectorElementType();
6498 EVT IdxTy = TLI->getVectorIdxTy();
6499 SDLoc SL(Op);
6500 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
6501 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
6502 Op, getConstant(i, IdxTy)));
6503 }
6504 }
6506 // getAddressSpace - Return the address space this GlobalAddress belongs to.
6507 unsigned GlobalAddressSDNode::getAddressSpace() const {
6508 return getGlobal()->getType()->getAddressSpace();
6509 }
6512 Type *ConstantPoolSDNode::getType() const {
6513 if (isMachineConstantPoolEntry())
6514 return Val.MachineCPVal->getType();
6515 return Val.ConstVal->getType();
6516 }
6518 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue,
6519 APInt &SplatUndef,
6520 unsigned &SplatBitSize,
6521 bool &HasAnyUndefs,
6522 unsigned MinSplatBits,
6523 bool isBigEndian) const {
6524 EVT VT = getValueType(0);
6525 assert(VT.isVector() && "Expected a vector type");
6526 unsigned sz = VT.getSizeInBits();
6527 if (MinSplatBits > sz)
6528 return false;
6530 SplatValue = APInt(sz, 0);
6531 SplatUndef = APInt(sz, 0);
6533 // Get the bits. Bits with undefined values (when the corresponding element
6534 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
6535 // in SplatValue. If any of the values are not constant, give up and return
6536 // false.
6537 unsigned int nOps = getNumOperands();
6538 assert(nOps > 0 && "isConstantSplat has 0-size build vector");
6539 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits();
6541 for (unsigned j = 0; j < nOps; ++j) {
6542 unsigned i = isBigEndian ? nOps-1-j : j;
6543 SDValue OpVal = getOperand(i);
6544 unsigned BitPos = j * EltBitSize;
6546 if (OpVal.getOpcode() == ISD::UNDEF)
6547 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize);
6548 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal))
6549 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize).
6550 zextOrTrunc(sz) << BitPos;
6551 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal))
6552 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos;
6553 else
6554 return false;
6555 }
6557 // The build_vector is all constants or undefs. Find the smallest element
6558 // size that splats the vector.
6560 HasAnyUndefs = (SplatUndef != 0);
6561 while (sz > 8) {
6563 unsigned HalfSize = sz / 2;
6564 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
6565 APInt LowValue = SplatValue.trunc(HalfSize);
6566 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
6567 APInt LowUndef = SplatUndef.trunc(HalfSize);
6569 // If the two halves do not match (ignoring undef bits), stop here.
6570 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
6571 MinSplatBits > HalfSize)
6572 break;
6574 SplatValue = HighValue | LowValue;
6575 SplatUndef = HighUndef & LowUndef;
6577 sz = HalfSize;
6578 }
6580 SplatBitSize = sz;
6581 return true;
6582 }
6584 ConstantSDNode *BuildVectorSDNode::getConstantSplatValue() const {
6585 SDValue Op0 = getOperand(0);
6586 if (Op0.getOpcode() != ISD::Constant)
6587 return nullptr;
6589 for (unsigned i = 1, e = getNumOperands(); i != e; ++i)
6590 if (getOperand(i) != Op0)
6591 return nullptr;
6593 return cast<ConstantSDNode>(Op0);
6594 }
6596 bool BuildVectorSDNode::isConstant() const {
6597 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
6598 unsigned Opc = getOperand(i).getOpcode();
6599 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
6600 return false;
6601 }
6602 return true;
6603 }
6605 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
6606 // Find the first non-undef value in the shuffle mask.
6607 unsigned i, e;
6608 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
6609 /* search */;
6611 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
6613 // Make sure all remaining elements are either undef or the same as the first
6614 // non-undef value.
6615 for (int Idx = Mask[i]; i != e; ++i)
6616 if (Mask[i] >= 0 && Mask[i] != Idx)
6617 return false;
6618 return true;
6619 }
6621 #ifdef XDEBUG
6622 static void checkForCyclesHelper(const SDNode *N,
6623 SmallPtrSet<const SDNode*, 32> &Visited,
6624 SmallPtrSet<const SDNode*, 32> &Checked) {
6625 // If this node has already been checked, don't check it again.
6626 if (Checked.count(N))
6627 return;
6629 // If a node has already been visited on this depth-first walk, reject it as
6630 // a cycle.
6631 if (!Visited.insert(N)) {
6632 dbgs() << "Offending node:\n";
6633 N->dumprFull();
6634 errs() << "Detected cycle in SelectionDAG\n";
6635 abort();
6636 }
6638 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
6639 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked);
6641 Checked.insert(N);
6642 Visited.erase(N);
6643 }
6644 #endif
6646 void llvm::checkForCycles(const llvm::SDNode *N) {
6647 #ifdef XDEBUG
6648 assert(N && "Checking nonexistent SDNode");
6649 SmallPtrSet<const SDNode*, 32> visited;
6650 SmallPtrSet<const SDNode*, 32> checked;
6651 checkForCyclesHelper(N, visited, checked);
6652 #endif
6653 }
6655 void llvm::checkForCycles(const llvm::SelectionDAG *DAG) {
6656 checkForCycles(DAG->getRoot().getNode());
6657 }