Lines Matching refs:VT
206 EVT VT = getOptimalMemOpType(Op, FuncAttributes); in findOptimalMemOpLowering() local
208 if (VT == MVT::Other) { in findOptimalMemOpLowering()
212 VT = MVT::LAST_INTEGER_VALUETYPE; in findOptimalMemOpLowering()
214 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) && in findOptimalMemOpLowering()
215 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign())) in findOptimalMemOpLowering()
216 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); in findOptimalMemOpLowering()
217 assert(VT.isInteger()); in findOptimalMemOpLowering()
227 if (VT.bitsGT(LVT)) in findOptimalMemOpLowering()
228 VT = LVT; in findOptimalMemOpLowering()
234 unsigned VTSize = VT.getSizeInBits() / 8; in findOptimalMemOpLowering()
237 EVT NewVT = VT; in findOptimalMemOpLowering()
241 if (VT.isVector() || VT.isFloatingPoint()) { in findOptimalMemOpLowering()
242 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; in findOptimalMemOpLowering()
269 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), in findOptimalMemOpLowering()
274 VT = NewVT; in findOptimalMemOpLowering()
282 MemOps.push_back(VT); in findOptimalMemOpLowering()
291 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, in softenSetCCOperands() argument
297 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS, in softenSetCCOperands()
301 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, in softenSetCCOperands() argument
312 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) in softenSetCCOperands()
321 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : in softenSetCCOperands()
322 (VT == MVT::f64) ? RTLIB::OEQ_F64 : in softenSetCCOperands()
323 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128; in softenSetCCOperands()
327 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : in softenSetCCOperands()
328 (VT == MVT::f64) ? RTLIB::UNE_F64 : in softenSetCCOperands()
329 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128; in softenSetCCOperands()
333 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : in softenSetCCOperands()
334 (VT == MVT::f64) ? RTLIB::OGE_F64 : in softenSetCCOperands()
335 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128; in softenSetCCOperands()
339 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : in softenSetCCOperands()
340 (VT == MVT::f64) ? RTLIB::OLT_F64 : in softenSetCCOperands()
341 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128; in softenSetCCOperands()
345 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : in softenSetCCOperands()
346 (VT == MVT::f64) ? RTLIB::OLE_F64 : in softenSetCCOperands()
347 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128; in softenSetCCOperands()
351 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : in softenSetCCOperands()
352 (VT == MVT::f64) ? RTLIB::OGT_F64 : in softenSetCCOperands()
353 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128; in softenSetCCOperands()
359 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : in softenSetCCOperands()
360 (VT == MVT::f64) ? RTLIB::UO_F64 : in softenSetCCOperands()
361 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128; in softenSetCCOperands()
368 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : in softenSetCCOperands()
369 (VT == MVT::f64) ? RTLIB::UO_F64 : in softenSetCCOperands()
370 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128; in softenSetCCOperands()
371 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : in softenSetCCOperands()
372 (VT == MVT::f64) ? RTLIB::OEQ_F64 : in softenSetCCOperands()
373 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128; in softenSetCCOperands()
380 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : in softenSetCCOperands()
381 (VT == MVT::f64) ? RTLIB::OGE_F64 : in softenSetCCOperands()
382 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128; in softenSetCCOperands()
385 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : in softenSetCCOperands()
386 (VT == MVT::f64) ? RTLIB::OGT_F64 : in softenSetCCOperands()
387 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128; in softenSetCCOperands()
390 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : in softenSetCCOperands()
391 (VT == MVT::f64) ? RTLIB::OLE_F64 : in softenSetCCOperands()
392 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128; in softenSetCCOperands()
395 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : in softenSetCCOperands()
396 (VT == MVT::f64) ? RTLIB::OLT_F64 : in softenSetCCOperands()
397 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128; in softenSetCCOperands()
546 EVT VT = Op.getValueType(); in ShrinkDemandedConstant() local
547 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT); in ShrinkDemandedConstant()
548 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC, in ShrinkDemandedConstant()
563 EVT VT = Op.getValueType(); in ShrinkDemandedConstant() local
564 APInt DemandedElts = VT.isVector() in ShrinkDemandedConstant()
565 ? APInt::getAllOnes(VT.getVectorNumElements()) in ShrinkDemandedConstant()
582 EVT VT = Op.getValueType(); in ShrinkDemandedOp() local
587 if (VT.isVector()) in ShrinkDemandedOp()
606 if (TLI.isTruncateFree(VT, SmallVT) && TLI.isZExtFree(SmallVT, VT)) { in ShrinkDemandedOp()
613 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X); in ShrinkDemandedOp()
657 EVT VT = Op.getValueType(); in SimplifyDemandedBits() local
662 APInt DemandedElts = VT.isFixedLengthVector() in SimplifyDemandedBits()
663 ? APInt::getAllOnes(VT.getVectorNumElements()) in SimplifyDemandedBits()
673 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBits() local
685 return DAG.getUNDEF(VT); in SimplifyMultipleUseDemandedBits()
693 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
849 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
865 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
878 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
892 assert(!VT.isScalableVector()); in SimplifyMultipleUseDemandedBits()
918 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
933 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBits() local
937 APInt DemandedElts = VT.isFixedLengthVector() in SimplifyMultipleUseDemandedBits()
938 ? APInt::getAllOnes(VT.getVectorNumElements()) in SimplifyMultipleUseDemandedBits()
1059 EVT VT = Op.getValueType(); in combineShiftToAVG() local
1061 std::max<unsigned>(VT.getScalarSizeInBits() - KnownBits, 8); in combineShiftToAVG()
1063 if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits()) in combineShiftToAVG()
1065 if (VT.isVector()) in combineShiftToAVG()
1066 NVT = EVT::getVectorVT(*DAG.getContext(), NVT, VT.getVectorElementCount()); in combineShiftToAVG()
1070 if (TLO.LegalOperations() && !TLI.isOperationLegal(AVGOpc, VT)) in combineShiftToAVG()
1076 NVT = VT; in combineShiftToAVG()
1091 return DAG.getExtOrTrunc(IsSigned, ResultAVG, DL, VT); in combineShiftToAVG()
1112 EVT VT = Op.getValueType(); in SimplifyDemandedBits() local
1115 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) && in SimplifyDemandedBits()
1156 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedBits()
1165 if (VT.isScalableVector()) in SimplifyDemandedBits()
1168 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedBits()
1216 if (VT.isScalableVector()) in SimplifyDemandedBits()
1254 if (VT.isScalableVector()) in SimplifyDemandedBits()
1291 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub, in SimplifyDemandedBits()
1299 if (VT.isScalableVector()) in SimplifyDemandedBits()
1318 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, in SimplifyDemandedBits()
1326 if (VT.isScalableVector()) in SimplifyDemandedBits()
1346 assert(!VT.isScalableVector()); in SimplifyDemandedBits()
1382 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask); in SimplifyDemandedBits()
1417 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1424 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() && in SimplifyDemandedBits()
1437 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1439 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd, in SimplifyDemandedBits()
1460 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT)); in SimplifyDemandedBits()
1478 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1530 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1549 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, in SimplifyDemandedBits()
1551 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12); in SimplifyDemandedBits()
1552 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2); in SimplifyDemandedBits()
1554 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY)); in SimplifyDemandedBits()
1590 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1601 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT); in SimplifyDemandedBits()
1602 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1610 SDValue New = TLO.DAG.getNOT(dl, Op0, VT); in SimplifyDemandedBits()
1632 SDValue Not = TLO.DAG.getNOT(dl, Op0.getOperand(0), VT); in SimplifyDemandedBits()
1633 return TLO.CombineTo(Op, TLO.DAG.getNode(Op0Opcode, dl, VT, Not, in SimplifyDemandedBits()
1655 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1760 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1778 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1798 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1801 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1830 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1838 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() && in SimplifyDemandedBits()
1846 if (isNarrowingProfitable(VT, SmallVT) && in SimplifyDemandedBits()
1848 isTruncateFree(VT, SmallVT) && isZExtFree(SmallVT, VT) && in SimplifyDemandedBits()
1858 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1867 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth && in SimplifyDemandedBits()
1870 if (isNarrowingProfitable(VT, HalfVT) && in SimplifyDemandedBits()
1872 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) && in SimplifyDemandedBits()
1888 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift); in SimplifyDemandedBits()
1956 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1970 if ((BitWidth % 2) == 0 && !VT.isVector()) { in SimplifyDemandedBits()
1973 if (isNarrowingProfitable(VT, HalfVT) && in SimplifyDemandedBits()
1975 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) && in SimplifyDemandedBits()
1985 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift)); in SimplifyDemandedBits()
2003 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2037 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2052 if (VT.isVector()) in SimplifyDemandedBits()
2054 VT.getVectorElementCount()); in SimplifyDemandedBits()
2060 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, in SimplifyDemandedBits()
2099 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
2106 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
2118 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2177 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedOp0, in SimplifyDemandedBits()
2219 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && in SimplifyDemandedBits()
2222 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2224 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && in SimplifyDemandedBits()
2227 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2256 return TLO.CombineTo(Op, TLO.DAG.getNode(BitOp, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBits()
2321 if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) { in SimplifyDemandedBits()
2323 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(ShiftAmount, VT, dl); in SimplifyDemandedBits()
2324 SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt); in SimplifyDemandedBits()
2341 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector()) in SimplifyDemandedBits()
2342 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
2364 TLO.DAG.getShiftAmountConstant(BitWidth - ExVTBits, VT, dl); in SimplifyDemandedBits()
2366 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
2420 if (VT.isScalableVector()) in SimplifyDemandedBits()
2435 VT.getSizeInBits() == SrcVT.getSizeInBits()) in SimplifyDemandedBits()
2436 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
2440 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
2441 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
2461 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2465 if (VT.isScalableVector()) in SimplifyDemandedBits()
2487 VT.getSizeInBits() == SrcVT.getSizeInBits()) in SimplifyDemandedBits()
2488 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
2491 if (getBooleanContents(VT) != ZeroOrNegativeOneBooleanContent || in SimplifyDemandedBits()
2496 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
2497 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
2513 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) { in SimplifyDemandedBits()
2517 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags)); in SimplifyDemandedBits()
2524 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2528 if (VT.isScalableVector()) in SimplifyDemandedBits()
2541 VT.getSizeInBits() == SrcVT.getSizeInBits()) in SimplifyDemandedBits()
2542 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
2555 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2573 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2583 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT)) in SimplifyDemandedBits()
2589 if (isTruncateFree(Src, VT) && in SimplifyDemandedBits()
2590 !isTruncateFree(Src.getValueType(), VT)) { in SimplifyDemandedBits()
2613 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(ShVal, VT, dl); in SimplifyDemandedBits()
2615 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2617 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2669 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx); in SimplifyDemandedBits()
2680 if (VT.isScalableVector()) in SimplifyDemandedBits()
2688 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() && in SimplifyDemandedBits()
2691 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT); in SimplifyDemandedBits()
2693 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 && in SimplifyDemandedBits()
2696 EVT Ty = OpVTLegal ? VT : MVT::i32; in SimplifyDemandedBits()
2702 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2704 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT); in SimplifyDemandedBits()
2706 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2767 SDValue NewOp = TLO.DAG.getBitcast(VT, DemandedSrc); in SimplifyDemandedBits()
2789 SDValue AmtC = TLO.DAG.getShiftAmountConstant(CTZ, VT, dl); in SimplifyDemandedBits()
2790 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC); in SimplifyDemandedBits()
2798 SDValue One = TLO.DAG.getConstant(1, dl, VT); in SimplifyDemandedBits()
2799 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One); in SimplifyDemandedBits()
2851 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags); in SimplifyDemandedBits()
2865 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT); in SimplifyDemandedBits()
2870 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags); in SimplifyDemandedBits()
2895 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(ShlAmt, VT, dl); in SimplifyDemandedBits()
2896 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC); in SimplifyDemandedBits()
2897 SDValue Res = TLO.DAG.getNode(NT, dl, VT, Y, Shl); in SimplifyDemandedBits()
2901 if (isOperationLegalOrCustom(ISD::SHL, VT)) { in SimplifyDemandedBits()
2957 if (VT.isInteger()) in SimplifyDemandedBits()
2958 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT)); in SimplifyDemandedBits()
2959 if (VT.isFloatingPoint()) in SimplifyDemandedBits()
2963 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT)); in SimplifyDemandedBits()
2998 EVT VT = BO.getValueType(); in getKnownUndefForVectorBinop() local
2999 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && in getKnownUndefForVectorBinop()
3002 EVT EltVT = VT.getVectorElementType(); in getKnownUndefForVectorBinop()
3003 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1; in getKnownUndefForVectorBinop()
3045 EVT VT = Op.getValueType(); in SimplifyDemandedVectorElts() local
3049 assert(VT.isVector() && "Expected vector op"); in SimplifyDemandedVectorElts()
3058 if (VT.isScalableVector()) in SimplifyDemandedVectorElts()
3061 assert(VT.getVectorNumElements() == NumElts && in SimplifyDemandedVectorElts()
3077 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3085 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in SimplifyDemandedVectorElts()
3097 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, in SimplifyDemandedVectorElts()
3108 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3238 Op, TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, in SimplifyDemandedVectorElts()
3258 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops)); in SimplifyDemandedVectorElts()
3301 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedSubOps); in SimplifyDemandedVectorElts()
3325 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in SimplifyDemandedVectorElts()
3326 TLO.DAG.getUNDEF(VT), Sub, in SimplifyDemandedVectorElts()
3344 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, in SimplifyDemandedVectorElts()
3372 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, in SimplifyDemandedVectorElts()
3497 buildLegalVectorShuffle(VT, DL, LHS, RHS, NewMask, TLO.DAG); in SimplifyDemandedVectorElts()
3538 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedVectorElts()
3544 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
3562 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold)); in SimplifyDemandedVectorElts()
3668 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
3694 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
3718 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3818 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, in buildLegalVectorShuffle() argument
3821 bool LegalMask = isShuffleMaskLegal(Mask, VT); in buildLegalVectorShuffle()
3825 LegalMask = isShuffleMaskLegal(Mask, VT); in buildLegalVectorShuffle()
3831 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask); in buildLegalVectorShuffle()
3956 bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT, in isExtendedTrueVal() argument
3958 if (VT == MVT::i1) in isExtendedTrueVal()
3961 TargetLowering::BooleanContent Cnt = getBooleanContents(VT); in isExtendedTrueVal()
3976 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithAnd() argument
3996 return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT); in foldSetCCWithAnd()
4014 return DAG.getSetCC(DL, VT, Trunc, Zero, in foldSetCCWithAnd()
4047 return DAG.getSetCC(DL, VT, N0, Zero, Cond); in foldSetCCWithAnd()
4063 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond); in foldSetCCWithAnd()
4222 EVT VT = X.getValueType(); in optimizeSetCCByHoistingAndByConstFromLogicalShift() local
4226 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4227 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4235 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithBinOp() argument
4251 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond); in foldSetCCWithBinOp()
4259 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond); in foldSetCCWithBinOp()
4270 return DAG.getSetCC(DL, VT, X, YShl1, Cond); in foldSetCCWithBinOp()
4273 static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, in simplifySetCCWithCTPOP() argument
4280 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && in simplifySetCCWithCTPOP()
4313 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC); in simplifySetCCWithCTPOP()
4333 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond); in simplifySetCCWithCTPOP()
4341 return DAG.getSetCC(dl, VT, Xor, Add, CmpCond); in simplifySetCCWithCTPOP()
4347 static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithRotate() argument
4367 return DAG.getSetCC(dl, VT, R, N1, Cond); in foldSetCCWithRotate()
4379 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithRotate()
4383 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithRotate()
4390 static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithFunnelShift() argument
4443 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithFunnelShift()
4450 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithFunnelShift()
4458 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
4468 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl)) in SimplifySetCC()
4484 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
4495 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
4497 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG)) in SimplifySetCC()
4500 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG)) in SimplifySetCC()
4507 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG)) in SimplifySetCC()
4521 SDValue IsXZero = DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
4522 SDValue IsYZero = DAG.getSetCC(dl, VT, N0.getOperand(1), N1, Cond); in SimplifySetCC()
4524 return DAG.getNode(LogicOp, dl, VT, IsXZero, IsYZero); in SimplifySetCC()
4546 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero, in SimplifySetCC()
4605 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1), in SimplifySetCC()
4608 return DAG.getSetCC(dl, VT, Trunc, C, Cond); in SimplifySetCC()
4622 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 && in SimplifySetCC()
4637 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), in SimplifySetCC()
4712 return DAG.getSetCC(dl, VT, And, DAG.getConstant(0LL, dl, newVT), Cond); in SimplifySetCC()
4728 return DAG.getConstant(0, dl, VT); in SimplifySetCC()
4732 return DAG.getConstant(1, dl, VT); in SimplifySetCC()
4736 return DAG.getConstant(C1.isNegative(), dl, VT); in SimplifySetCC()
4740 return DAG.getConstant(C1.isNonNegative(), dl, VT); in SimplifySetCC()
4763 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType()); in SimplifySetCC()
4782 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
4792 return DAG.getSetCC(dl, VT, ZextOp, in SimplifySetCC()
4798 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) && in SimplifySetCC()
4799 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) && in SimplifySetCC()
4806 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
4813 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in SimplifySetCC()
4841 return DAG.getSetCC(dl, VT, Val, N1, in SimplifySetCC()
4862 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond); in SimplifySetCC()
4867 if (Op0.getValueType().bitsGT(VT)) in SimplifySetCC()
4868 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4869 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4870 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
4871 else if (Op0.getValueType().bitsLT(VT)) in SimplifySetCC()
4872 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4873 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4874 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
4876 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
4882 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
4897 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
4906 return DAG.getSetCC(dl, VT, N0.getOperand(0), in SimplifySetCC()
4912 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4935 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
4937 if (!VT.isVector()) { // TODO: Support this for vectors. in SimplifySetCC()
4942 isCondCodeLegal(NewCC, VT.getSimpleVT())) && in SimplifySetCC()
4945 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4955 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
4958 if (!VT.isVector()) { // TODO: Support this for vectors. in SimplifySetCC()
4962 isCondCodeLegal(NewCC, VT.getSimpleVT())) && in SimplifySetCC()
4965 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4974 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false in SimplifySetCC()
4977 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
4980 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4984 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4992 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false in SimplifySetCC()
4995 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
4998 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
5002 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5012 VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5059 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond); in SimplifySetCC()
5080 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
5085 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5093 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5110 if ((VT.getSizeInBits() == 1 || in SimplifySetCC()
5113 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) && in SimplifySetCC()
5122 ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5133 ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5155 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond); in SimplifySetCC()
5184 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond); in SimplifySetCC()
5199 return DAG.getSetCC(dl, VT, N0, N0, Cond); in SimplifySetCC()
5207 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond); in SimplifySetCC()
5221 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op, in SimplifySetCC()
5244 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
5258 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
5260 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
5267 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
5278 return DAG.getSetCC(dl, VT, N1.getOperand(0), N0.getOperand(0), Cond); in SimplifySetCC()
5283 return DAG.getSetCC(dl, VT, Not, N0.getOperand(0), Cond); in SimplifySetCC()
5295 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond); in SimplifySetCC()
5297 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond); in SimplifySetCC()
5301 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0), in SimplifySetCC()
5304 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1), in SimplifySetCC()
5318 dl, VT, N0.getOperand(0), in SimplifySetCC()
5326 dl, VT, N0.getOperand(0), in SimplifySetCC()
5336 dl, VT, N0.getOperand(1), in SimplifySetCC()
5351 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
5357 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI)) in SimplifySetCC()
5360 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
5369 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) { in SimplifySetCC()
5371 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5374 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5421 if (VT.getScalarType() != MVT::i1) { in SimplifySetCC()
5426 N0 = DAG.getNode(ExtendCode, dl, VT, N0); in SimplifySetCC()
5624 MVT VT) const { in getRegForInlineAsmConstraint()
5650 if (RI->isTypeLegalForClass(*RC, VT)) in getRegForInlineAsmConstraint()
5771 EVT VT = getAsmOperandValueType(DL, OpTy, true); in ParseConstraints() local
5772 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other; in ParseConstraints()
6101 EVT VT = N->getValueType(0); in BuildExactSDIV() local
6102 EVT SVT = VT.getScalarType(); in BuildExactSDIV()
6103 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in BuildExactSDIV()
6131 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildExactSDIV()
6137 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildExactSDIV()
6148 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
6152 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
6161 EVT VT = N->getValueType(0); in BuildExactUDIV() local
6162 EVT SVT = VT.getScalarType(); in BuildExactUDIV()
6163 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in BuildExactUDIV()
6194 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildExactUDIV()
6200 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildExactUDIV()
6211 Res = DAG.getNode(ISD::SRL, dl, VT, Res, Shift, Flags); in BuildExactUDIV()
6215 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactUDIV()
6251 EVT VT = N->getValueType(0); in buildSDIVPow2WithCMov() local
6255 SDValue Zero = DAG.getConstant(0, DL, VT); in buildSDIVPow2WithCMov()
6256 APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2); in buildSDIVPow2WithCMov()
6257 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT); in buildSDIVPow2WithCMov()
6260 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in buildSDIVPow2WithCMov()
6262 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in buildSDIVPow2WithCMov()
6263 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in buildSDIVPow2WithCMov()
6271 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, VT)); in buildSDIVPow2WithCMov()
6279 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA); in buildSDIVPow2WithCMov()
6290 EVT VT = N->getValueType(0); in BuildSDIV() local
6291 EVT SVT = VT.getScalarType(); in BuildSDIV()
6292 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in BuildSDIV()
6294 unsigned EltBits = VT.getScalarSizeInBits(); in BuildSDIV()
6299 if (!isTypeLegal(VT)) { in BuildSDIV()
6301 if (VT.isVector() || !VT.isSimple()) in BuildSDIV()
6306 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) in BuildSDIV()
6309 MulVT = getTypeToTransformTo(*DAG.getContext(), VT); in BuildSDIV()
6360 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildSDIV()
6361 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildSDIV()
6363 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks); in BuildSDIV()
6369 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildSDIV()
6370 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildSDIV()
6372 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]); in BuildSDIV()
6386 if (!isTypeLegal(VT)) { in BuildSDIV()
6392 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6395 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) in BuildSDIV()
6396 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
6397 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { in BuildSDIV()
6399 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
6403 unsigned Size = VT.getScalarSizeInBits(); in BuildSDIV()
6405 if (VT.isVector()) in BuildSDIV()
6407 VT.getVectorElementCount()); in BuildSDIV()
6414 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6426 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
6428 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
6432 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
6437 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
6439 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
6441 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
6452 EVT VT = N->getValueType(0); in BuildUDIV() local
6453 EVT SVT = VT.getScalarType(); in BuildUDIV()
6454 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in BuildUDIV()
6456 unsigned EltBits = VT.getScalarSizeInBits(); in BuildUDIV()
6461 if (!isTypeLegal(VT)) { in BuildUDIV()
6463 if (VT.isVector() || !VT.isSimple()) in BuildUDIV()
6468 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) in BuildUDIV()
6471 MulVT = getTypeToTransformTo(*DAG.getContext(), VT); in BuildUDIV()
6541 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildUDIV()
6542 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors); in BuildUDIV()
6549 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildUDIV()
6550 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]); in BuildUDIV()
6561 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
6569 if (!isTypeLegal(VT)) { in BuildUDIV()
6575 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6578 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) in BuildUDIV()
6579 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
6580 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { in BuildUDIV()
6582 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
6586 unsigned Size = VT.getScalarSizeInBits(); in BuildUDIV()
6588 if (VT.isVector()) in BuildUDIV()
6590 VT.getVectorElementCount()); in BuildUDIV()
6597 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6610 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
6615 if (VT.isVector()) in BuildUDIV()
6618 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
6622 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
6627 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
6631 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in BuildUDIV()
6633 SDValue One = DAG.getConstant(1, dl, VT); in BuildUDIV()
6635 return DAG.getSelect(dl, VT, IsOne, N0, Q); in BuildUDIV()
6702 EVT VT = REMNode.getValueType(); in prepareUREMEqFold() local
6703 EVT SVT = VT.getScalarType(); in prepareUREMEqFold()
6704 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in prepareUREMEqFold()
6708 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareUREMEqFold()
6825 PVal = DAG.getBuildVector(VT, DL, PAmts); in prepareUREMEqFold()
6827 QVal = DAG.getBuildVector(VT, DL, QAmts); in prepareUREMEqFold()
6832 PVal = DAG.getSplatVector(VT, DL, PAmts[0]); in prepareUREMEqFold()
6834 QVal = DAG.getSplatVector(VT, DL, QAmts[0]); in prepareUREMEqFold()
6842 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) in prepareUREMEqFold()
6846 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode); in prepareUREMEqFold()
6850 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareUREMEqFold()
6857 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareUREMEqFold()
6860 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareUREMEqFold()
6874 assert(VT.isVector() && "Can/should only get here for vectors."); in prepareUREMEqFold()
6960 EVT VT = REMNode.getValueType(); in prepareSREMEqFold() local
6961 EVT SVT = VT.getScalarType(); in prepareSREMEqFold()
6962 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in prepareSREMEqFold()
6967 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareSREMEqFold()
7100 PVal = DAG.getBuildVector(VT, DL, PAmts); in prepareSREMEqFold()
7101 AVal = DAG.getBuildVector(VT, DL, AAmts); in prepareSREMEqFold()
7103 QVal = DAG.getBuildVector(VT, DL, QAmts); in prepareSREMEqFold()
7109 PVal = DAG.getSplatVector(VT, DL, PAmts[0]); in prepareSREMEqFold()
7110 AVal = DAG.getSplatVector(VT, DL, AAmts[0]); in prepareSREMEqFold()
7112 QVal = DAG.getSplatVector(VT, DL, QAmts[0]); in prepareSREMEqFold()
7122 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareSREMEqFold()
7127 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) in prepareSREMEqFold()
7131 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal); in prepareSREMEqFold()
7139 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareSREMEqFold()
7142 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareSREMEqFold()
7158 assert(VT.isVector() && "Can/should only get here for vectors."); in prepareSREMEqFold()
7164 !isOperationLegalOrCustom(ISD::AND, VT) || in prepareSREMEqFold()
7165 !isCondCodeLegalOrCustom(Cond, VT.getSimpleVT()) || in prepareSREMEqFold()
7172 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
7174 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
7176 DAG.getConstant(APInt::getZero(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
7183 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax); in prepareSREMEqFold()
7212 EVT VT = Op.getValueType(); in getSqrtInputTest() local
7213 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSqrtInputTest()
7214 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in getSqrtInputTest()
7227 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT); in getSqrtInputTest()
7229 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT); in getSqrtInputTest()
7230 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in getSqrtInputTest()
7252 EVT VT = Op.getValueType(); in getNegatedExpression() local
7258 isFPExtFree(VT, Op.getOperand(0).getValueType()); in getNegatedExpression()
7280 isOperationLegal(ISD::ConstantFP, VT) || in getNegatedExpression()
7281 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT, in getNegatedExpression()
7289 SDValue CFP = DAG.getConstantFP(V, DL, VT); in getNegatedExpression()
7306 (isOperationLegal(ISD::ConstantFP, VT) && in getNegatedExpression()
7307 isOperationLegal(ISD::BUILD_VECTOR, VT)) || in getNegatedExpression()
7310 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT, in getNegatedExpression()
7328 return DAG.getBuildVector(VT, DL, Ops); in getNegatedExpression()
7335 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT)) in getNegatedExpression()
7358 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags); in getNegatedExpression()
7367 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags); in getNegatedExpression()
7389 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags); in getNegatedExpression()
7414 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags); in getNegatedExpression()
7428 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags); in getNegatedExpression()
7470 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags); in getNegatedExpression()
7479 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags); in getNegatedExpression()
7491 return DAG.getNode(Opcode, DL, VT, NegV); in getNegatedExpression()
7496 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1)); in getNegatedExpression()
7531 return DAG.getSelect(DL, VT, Op.getOperand(0), NegLHS, NegRHS); in getNegatedExpression()
7542 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, in expandMUL_LOHI() argument
7563 unsigned OuterBitSize = VT.getScalarSizeInBits(); in expandMUL_LOHI()
7613 if (!VT.isVector() && Opcode == ISD::MUL && in expandMUL_LOHI()
7626 SDValue Shift = DAG.getShiftAmountConstant(ShiftAmount, VT, dl); in expandMUL_LOHI()
7629 isOperationLegalOrCustom(ISD::SRL, VT) && in expandMUL_LOHI()
7631 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
7633 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
7656 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
7657 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7658 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
7659 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
7662 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7668 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7674 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandMUL_LOHI()
7676 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) && in expandMUL_LOHI()
7677 isOperationLegalOrCustom(ISD::ADDE, VT)); in expandMUL_LOHI()
7679 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
7682 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
7687 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7699 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7702 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7703 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
7706 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7707 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
7712 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7759 EVT VT = N->getValueType(0); in expandDIVREMByConstant() local
7775 assert(VT.getScalarSizeInBits() == BitWidth && in expandDIVREMByConstant()
7873 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH); in expandDIVREMByConstant()
7874 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH); in expandDIVREMByConstant()
7876 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem); in expandDIVREMByConstant()
7882 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend, in expandDIVREMByConstant()
7883 DAG.getConstant(MulFactor, dl, VT)); in expandDIVREMByConstant()
7917 EVT VT = Node->getValueType(0); in expandVPFunnelShift() local
7926 unsigned BW = VT.getScalarSizeInBits(); in expandVPFunnelShift()
7938 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask, in expandVPFunnelShift()
7940 ShY = DAG.getNode(ISD::VP_SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask, in expandVPFunnelShift()
7961 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL); in expandVPFunnelShift()
7962 SDValue ShY1 = DAG.getNode(ISD::VP_SRL, DL, VT, Y, One, Mask, VL); in expandVPFunnelShift()
7963 ShY = DAG.getNode(ISD::VP_SRL, DL, VT, ShY1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7965 SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL); in expandVPFunnelShift()
7966 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7967 ShY = DAG.getNode(ISD::VP_SRL, DL, VT, Y, ShAmt, Mask, VL); in expandVPFunnelShift()
7970 return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL); in expandVPFunnelShift()
7978 EVT VT = Node->getValueType(0); in expandFunnelShift() local
7980 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandFunnelShift()
7981 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandFunnelShift()
7982 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandFunnelShift()
7983 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandFunnelShift()
7990 unsigned BW = VT.getScalarSizeInBits(); in expandFunnelShift()
7998 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) && in expandFunnelShift()
7999 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) { in expandFunnelShift()
8004 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z); in expandFunnelShift()
8010 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One); in expandFunnelShift()
8011 X = DAG.getNode(ISD::SRL, DL, VT, X, One); in expandFunnelShift()
8013 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One); in expandFunnelShift()
8014 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One); in expandFunnelShift()
8018 return DAG.getNode(RevOpcode, DL, VT, X, Y, Z); in expandFunnelShift()
8030 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt); in expandFunnelShift()
8031 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt); in expandFunnelShift()
8049 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt); in expandFunnelShift()
8050 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One); in expandFunnelShift()
8051 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt); in expandFunnelShift()
8053 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One); in expandFunnelShift()
8054 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt); in expandFunnelShift()
8055 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt); in expandFunnelShift()
8058 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in expandFunnelShift()
8064 EVT VT = Node->getValueType(0); in expandROT() local
8065 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in expandROT()
8076 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) && in expandROT()
8077 isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) { in expandROT()
8079 return DAG.getNode(RevRot, DL, VT, Op0, Sub); in expandROT()
8082 if (!AllowVectorOps && VT.isVector() && in expandROT()
8083 (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandROT()
8084 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandROT()
8085 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandROT()
8086 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) || in expandROT()
8087 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandROT()
8100 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt); in expandROT()
8102 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt); in expandROT()
8108 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt); in expandROT()
8112 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt); in expandROT()
8114 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal); in expandROT()
8120 EVT VT = Node->getValueType(0); in expandShiftParts() local
8121 unsigned VTBits = VT.getScalarSizeInBits(); in expandShiftParts()
8139 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
8141 : DAG.getConstant(0, dl, VT); in expandShiftParts()
8145 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8146 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
8148 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8149 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
8161 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8162 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8164 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8165 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8422 EVT VT = Node->getValueType(0); in expandFMINNUM_FMAXNUM() local
8424 if (VT.isScalableVector()) in expandFMINNUM_FMAXNUM()
8428 if (isOperationLegalOrCustom(NewOp, VT)) { in expandFMINNUM_FMAXNUM()
8436 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
8440 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
8445 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags()); in expandFMINNUM_FMAXNUM()
8459 if (isOperationLegalOrCustom(IEEE2018Op, VT)) in expandFMINNUM_FMAXNUM()
8460 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0), in expandFMINNUM_FMAXNUM()
8476 EVT VT = N->getValueType(0); in expandFMINIMUM_FMAXIMUM() local
8477 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFMINIMUM_FMAXIMUM()
8491 if (isOperationLegalOrCustom(CompOpcIeee, VT)) { in expandFMINIMUM_FMAXIMUM()
8492 MinMax = DAG.getNode(CompOpcIeee, DL, VT, LHS, RHS, Flags); in expandFMINIMUM_FMAXIMUM()
8494 } else if (isOperationLegalOrCustom(CompOpc, VT)) { in expandFMINIMUM_FMAXIMUM()
8495 MinMax = DAG.getNode(CompOpc, DL, VT, LHS, RHS, Flags); in expandFMINIMUM_FMAXIMUM()
8497 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandFMINIMUM_FMAXIMUM()
8503 MinMax = DAG.getSelect(DL, VT, Compare, LHS, RHS, Flags); in expandFMINIMUM_FMAXIMUM()
8510 *DAG.getContext(), APFloat::getNaN(DAG.EVTToAPFloatSemantics(VT))); in expandFMINIMUM_FMAXIMUM()
8511 MinMax = DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, LHS, RHS, ISD::SETUO), in expandFMINIMUM_FMAXIMUM()
8512 DAG.getConstantFP(*FPNaN, DL, VT), MinMax, Flags); in expandFMINIMUM_FMAXIMUM()
8519 DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ); in expandFMINIMUM_FMAXIMUM()
8523 DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, LHS, TestZero), LHS, in expandFMINIMUM_FMAXIMUM()
8526 DL, VT, DAG.getNode(ISD::IS_FPCLASS, DL, CCVT, RHS, TestZero), RHS, in expandFMINIMUM_FMAXIMUM()
8528 MinMax = DAG.getSelect(DL, VT, IsZero, RCmp, MinMax, Flags); in expandFMINIMUM_FMAXIMUM()
8820 static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) { in canExpandVectorCTPOP() argument
8821 assert(VT.isVector() && "Expected vector type"); in canExpandVectorCTPOP()
8822 unsigned Len = VT.getScalarSizeInBits(); in canExpandVectorCTPOP()
8823 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in canExpandVectorCTPOP()
8824 TLI.isOperationLegalOrCustom(ISD::SUB, VT) && in canExpandVectorCTPOP()
8825 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in canExpandVectorCTPOP()
8826 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) && in canExpandVectorCTPOP()
8827 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT); in canExpandVectorCTPOP()
8832 EVT VT = Node->getValueType(0); in expandCTPOP() local
8833 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandCTPOP()
8835 unsigned Len = VT.getScalarSizeInBits(); in expandCTPOP()
8836 assert(VT.isInteger() && "CTPOP not implemented for this type."); in expandCTPOP()
8843 if (VT.isVector() && !canExpandVectorCTPOP(*this, VT)) in expandCTPOP()
8849 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandCTPOP()
8851 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandCTPOP()
8853 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandCTPOP()
8856 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
8857 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8858 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8862 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
8863 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8864 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8868 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8869 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8870 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8880 if (Len == 16 && !VT.isVector()) { in expandCTPOP()
8882 return DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8883 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8884 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8886 DAG.getConstant(0xFF, dl, VT)); in expandCTPOP()
8892 ISD::MUL, getTypeToTransformTo(*DAG.getContext(), VT))) { in expandCTPOP()
8894 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandCTPOP()
8895 V = DAG.getNode(ISD::MUL, dl, VT, Op, Mask01); in expandCTPOP()
8899 SDValue ShiftC = DAG.getShiftAmountConstant(Shift, VT, dl); in expandCTPOP()
8900 V = DAG.getNode(ISD::ADD, dl, VT, V, in expandCTPOP()
8901 DAG.getNode(ISD::SHL, dl, VT, V, ShiftC)); in expandCTPOP()
8904 return DAG.getNode(ISD::SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT)); in expandCTPOP()
8909 EVT VT = Node->getValueType(0); in expandVPCTPOP() local
8910 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPCTPOP()
8914 unsigned Len = VT.getScalarSizeInBits(); in expandVPCTPOP()
8915 assert(VT.isInteger() && "VP_CTPOP not implemented for this type."); in expandVPCTPOP()
8924 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandVPCTPOP()
8926 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandVPCTPOP()
8928 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandVPCTPOP()
8933 Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8934 DAG.getNode(ISD::VP_SRL, dl, VT, Op, in expandVPCTPOP()
8937 Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL); in expandVPCTPOP()
8940 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL); in expandVPCTPOP()
8941 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8942 DAG.getNode(ISD::VP_SRL, dl, VT, Op, in expandVPCTPOP()
8945 Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL); in expandVPCTPOP()
8948 Tmp4 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(4, dl, ShVT), in expandVPCTPOP()
8950 Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL); in expandVPCTPOP()
8951 Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL); in expandVPCTPOP()
8959 ISD::VP_MUL, getTypeToTransformTo(*DAG.getContext(), VT))) { in expandVPCTPOP()
8961 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandVPCTPOP()
8962 V = DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL); in expandVPCTPOP()
8966 SDValue ShiftC = DAG.getShiftAmountConstant(Shift, VT, dl); in expandVPCTPOP()
8967 V = DAG.getNode(ISD::VP_ADD, dl, VT, V, in expandVPCTPOP()
8968 DAG.getNode(ISD::VP_SHL, dl, VT, V, ShiftC, Mask, VL), in expandVPCTPOP()
8972 return DAG.getNode(ISD::VP_SRL, dl, VT, V, DAG.getConstant(Len - 8, dl, ShVT), in expandVPCTPOP()
8978 EVT VT = Node->getValueType(0); in expandCTLZ() local
8979 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandCTLZ()
8981 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandCTLZ()
8985 isOperationLegalOrCustom(ISD::CTLZ, VT)) in expandCTLZ()
8986 return DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
8989 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) { in expandCTLZ()
8991 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCTLZ()
8992 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
8993 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTLZ()
8995 return DAG.getSelect(dl, VT, SrcIsZero, in expandCTLZ()
8996 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ); in expandCTLZ()
9001 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) || in expandCTLZ()
9002 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTLZ()
9003 !canExpandVectorCTPOP(*this, VT)) || in expandCTLZ()
9004 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTLZ()
9005 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandCTLZ()
9019 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
9020 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
9022 Op = DAG.getNOT(dl, Op, VT); in expandCTLZ()
9023 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
9028 EVT VT = Node->getValueType(0); in expandVPCTLZ() local
9029 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPCTLZ()
9033 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandVPCTLZ()
9044 Op = DAG.getNode(ISD::VP_OR, dl, VT, Op, in expandVPCTLZ()
9045 DAG.getNode(ISD::VP_SRL, dl, VT, Op, Tmp, Mask, VL), Mask, in expandVPCTLZ()
9048 Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask, in expandVPCTLZ()
9050 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL); in expandVPCTLZ()
9054 const SDLoc &DL, EVT VT, SDValue Op, in CTTZTableLookup() argument
9064 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); in CTTZTableLookup()
9066 ISD::SRL, DL, VT, in CTTZTableLookup()
9067 DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg), in CTTZTableLookup()
9068 DAG.getConstant(DeBruijn, DL, VT)), in CTTZTableLookup()
9069 DAG.getConstant(ShiftAmt, DL, VT)); in CTTZTableLookup()
9083 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(), in CTTZTableLookup()
9090 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in CTTZTableLookup()
9091 SDValue Zero = DAG.getConstant(0, DL, VT); in CTTZTableLookup()
9093 return DAG.getSelect(DL, VT, SrcIsZero, in CTTZTableLookup()
9094 DAG.getConstant(BitWidth, DL, VT), ExtLoad); in CTTZTableLookup()
9099 EVT VT = Node->getValueType(0); in expandCTTZ() local
9101 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandCTTZ()
9105 isOperationLegalOrCustom(ISD::CTTZ, VT)) in expandCTTZ()
9106 return DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
9109 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) { in expandCTTZ()
9111 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCTTZ()
9112 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
9113 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTTZ()
9115 return DAG.getSelect(dl, VT, SrcIsZero, in expandCTTZ()
9116 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ); in expandCTTZ()
9121 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) || in expandCTTZ()
9122 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTTZ()
9123 !isOperationLegalOrCustom(ISD::CTLZ, VT) && in expandCTTZ()
9124 !canExpandVectorCTPOP(*this, VT)) || in expandCTTZ()
9125 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTTZ()
9126 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) || in expandCTTZ()
9127 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandCTTZ()
9131 if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) && in expandCTTZ()
9132 !isOperationLegal(ISD::CTLZ, VT)) in expandCTTZ()
9133 if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt)) in expandCTTZ()
9141 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
9142 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
9145 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) { in expandCTTZ()
9146 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
9147 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
9150 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
9158 EVT VT = Node->getValueType(0); in expandVPCTTZ() local
9161 SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op, in expandVPCTTZ()
9162 DAG.getConstant(-1, dl, VT), Mask, VL); in expandVPCTTZ()
9163 SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op, in expandVPCTTZ()
9164 DAG.getConstant(1, dl, VT), Mask, VL); in expandVPCTTZ()
9165 SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL); in expandVPCTTZ()
9166 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL); in expandVPCTTZ()
9205 EVT VT = N->getValueType(0); in expandABS() local
9209 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
9210 isOperationLegal(ISD::SMAX, VT)) { in expandABS()
9211 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
9213 return DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
9214 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
9218 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
9219 isOperationLegal(ISD::UMIN, VT)) { in expandABS()
9220 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
9222 return DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
9223 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
9227 if (IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
9228 isOperationLegal(ISD::SMIN, VT)) { in expandABS()
9229 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
9231 return DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
9232 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
9236 if (VT.isVector() && in expandABS()
9237 (!isOperationLegalOrCustom(ISD::SRA, VT) || in expandABS()
9238 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) || in expandABS()
9239 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) || in expandABS()
9240 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandABS()
9245 ISD::SRA, dl, VT, Op, in expandABS()
9246 DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, dl)); in expandABS()
9247 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
9251 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift); in expandABS()
9254 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
9259 EVT VT = N->getValueType(0); in expandABD() local
9268 if (isOperationLegal(MaxOpc, VT) && isOperationLegal(MinOpc, VT)) { in expandABD()
9269 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS); in expandABD()
9270 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS); in expandABD()
9271 return DAG.getNode(ISD::SUB, dl, VT, Max, Min); in expandABD()
9275 if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT)) in expandABD()
9276 return DAG.getNode(ISD::OR, dl, VT, in expandABD()
9277 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS), in expandABD()
9278 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS)); in expandABD()
9280 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandABD()
9287 if (CCVT == VT && getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandABD()
9288 SDValue Diff = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in expandABD()
9289 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Diff, Cmp); in expandABD()
9290 return DAG.getNode(ISD::SUB, dl, VT, Cmp, Xor); in expandABD()
9295 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS), in expandABD()
9296 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS)); in expandABD()
9301 EVT VT = N->getValueType(0); in expandAVG() local
9323 SDValue Sum = DAG.getNode(ISD::ADD, dl, VT, LHS, RHS); in expandAVG()
9325 Sum = DAG.getNode(ISD::ADD, dl, VT, Sum, DAG.getConstant(1, dl, VT)); in expandAVG()
9326 return DAG.getNode(ShiftOpc, dl, VT, Sum, in expandAVG()
9327 DAG.getShiftAmountConstant(1, VT, dl)); in expandAVG()
9331 if (VT.isScalarInteger()) { in expandAVG()
9332 unsigned BW = VT.getScalarSizeInBits(); in expandAVG()
9333 EVT ExtVT = VT.getIntegerVT(*DAG.getContext(), 2 * BW); in expandAVG()
9334 if (isTypeLegal(ExtVT) && isTruncateFree(ExtVT, VT)) { in expandAVG()
9344 return DAG.getNode(ISD::TRUNCATE, dl, VT, Avg); in expandAVG()
9354 SDValue Sign = DAG.getNode(SignOpc, dl, VT, LHS, RHS); in expandAVG()
9355 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandAVG()
9357 DAG.getNode(ShiftOpc, dl, VT, Xor, DAG.getShiftAmountConstant(1, VT, dl)); in expandAVG()
9358 return DAG.getNode(SumOpc, dl, VT, Sign, Shift); in expandAVG()
9363 EVT VT = N->getValueType(0); in expandBSWAP() local
9366 if (!VT.isSimple()) in expandBSWAP()
9369 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandBSWAP()
9371 switch (VT.getSimpleVT().getScalarType().SimpleTy) { in expandBSWAP()
9376 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9378 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9379 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9380 DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9381 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9382 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9383 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9384 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9385 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9386 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9387 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9389 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9390 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9391 DAG.getConstant(255ULL<<8, dl, VT)); in expandBSWAP()
9392 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9393 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9394 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
9395 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9396 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9397 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
9398 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9399 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9400 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
9401 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
9402 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9403 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
9404 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
9405 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9406 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
9407 DAG.getConstant(255ULL<<8, dl, VT)); in expandBSWAP()
9408 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9409 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
9410 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
9411 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9412 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9413 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
9414 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9415 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
9421 EVT VT = N->getValueType(0); in expandVPBSWAP() local
9426 if (!VT.isSimple()) in expandVPBSWAP()
9429 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPBSWAP()
9431 switch (VT.getSimpleVT().getScalarType().SimpleTy) { in expandVPBSWAP()
9435 Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9437 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9439 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL); in expandVPBSWAP()
9441 Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9443 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT), in expandVPBSWAP()
9445 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9447 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9449 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9450 DAG.getConstant(0xFF00, dl, VT), Mask, EVL); in expandVPBSWAP()
9451 Tmp1 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9453 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9454 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9455 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9457 Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9459 Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9460 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL); in expandVPBSWAP()
9461 Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9463 Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9464 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL); in expandVPBSWAP()
9465 Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9467 Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9468 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL); in expandVPBSWAP()
9469 Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9471 Tmp4 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9473 Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4, in expandVPBSWAP()
9474 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL); in expandVPBSWAP()
9475 Tmp3 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9477 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3, in expandVPBSWAP()
9478 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL); in expandVPBSWAP()
9479 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9481 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9482 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL); in expandVPBSWAP()
9483 Tmp1 = DAG.getNode(ISD::VP_SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9485 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL); in expandVPBSWAP()
9486 Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL); in expandVPBSWAP()
9487 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9488 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9489 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL); in expandVPBSWAP()
9490 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9491 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL); in expandVPBSWAP()
9497 EVT VT = N->getValueType(0); in expandBITREVERSE() local
9499 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandBITREVERSE()
9500 unsigned Sz = VT.getScalarSizeInBits(); in expandBITREVERSE()
9514 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
9517 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9518 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9519 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9520 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9521 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9524 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9525 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9526 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9527 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9528 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9531 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9532 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9533 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9534 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9535 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9539 Tmp = DAG.getConstant(0, dl, VT); in expandBITREVERSE()
9543 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
9546 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
9549 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
9550 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
9560 EVT VT = N->getValueType(0); in expandVPBITREVERSE() local
9564 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPBITREVERSE()
9565 unsigned Sz = VT.getScalarSizeInBits(); in expandVPBITREVERSE()
9579 Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op); in expandVPBITREVERSE()
9582 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9584 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9585 DAG.getConstant(Mask4, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9586 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT), in expandVPBITREVERSE()
9588 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9590 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9593 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9595 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9596 DAG.getConstant(Mask2, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9597 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT), in expandVPBITREVERSE()
9599 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9601 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9604 Tmp2 = DAG.getNode(ISD::VP_SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9606 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9607 DAG.getConstant(Mask1, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9608 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT), in expandVPBITREVERSE()
9610 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9612 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9791 EVT VT = LD->getValueType(0); in expandUnalignedLoad() local
9796 if (VT.isFloatingPoint() || VT.isVector()) { in expandUnalignedLoad()
9810 if (LoadedVT != VT) in expandUnalignedLoad()
9811 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in expandUnalignedLoad()
9812 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
9874 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, in expandUnalignedLoad()
9903 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9908 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9913 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9918 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9925 SDValue ShiftAmount = DAG.getShiftAmountConstant(NumBits, VT, dl); in expandUnalignedLoad()
9926 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
9927 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
9942 EVT VT = Val.getValueType(); in expandUnalignedStore() local
9949 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in expandUnalignedStore()
10045 ISD::AND, dl, VT, Lo, in expandUnalignedStore()
10046 DAG.getConstant(APInt::getLowBitsSet(VT.getSizeInBits(), NumBits), dl, in expandUnalignedStore()
10047 VT)); in expandUnalignedStore()
10048 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
10229 EVT VT = Op.getOperand(0).getValueType(); in lowerCmpEqZeroToCtlzSrl() local
10231 if (VT.bitsLT(MVT::i32)) { in lowerCmpEqZeroToCtlzSrl()
10232 VT = MVT::i32; in lowerCmpEqZeroToCtlzSrl()
10233 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
10235 unsigned Log2b = Log2_32(VT.getSizeInBits()); in lowerCmpEqZeroToCtlzSrl()
10236 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
10237 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
10247 EVT VT = Op0.getValueType(); in expandIntMINMAX() local
10248 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandIntMINMAX()
10253 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT && in expandIntMINMAX()
10254 getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandIntMINMAX()
10256 SDValue Zero = DAG.getConstant(0, DL, VT); in expandIntMINMAX()
10257 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
10258 DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ)); in expandIntMINMAX()
10263 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) && in expandIntMINMAX()
10264 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
10265 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
10266 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1)); in expandIntMINMAX()
10271 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) && in expandIntMINMAX()
10272 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
10273 return DAG.getNode(ISD::ADD, DL, VT, Op0, in expandIntMINMAX()
10274 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0)); in expandIntMINMAX()
10279 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandIntMINMAX()
10293 return DAG.getSelect(DL, VT, Cond, Op0, Op1); in expandIntMINMAX()
10300 return DAG.getSelect(DL, VT, Cond, Op1, Op0); in expandIntMINMAX()
10304 return DAG.getSelect(DL, VT, Cond, Op0, Op1); in expandIntMINMAX()
10329 EVT VT = LHS.getValueType(); in expandAddSubSat() local
10332 assert(VT == RHS.getValueType() && "Expected operands to be the same type"); in expandAddSubSat()
10333 assert(VT.isInteger() && "Expected operands to be integers"); in expandAddSubSat()
10336 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) { in expandAddSubSat()
10337 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
10338 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
10342 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) { in expandAddSubSat()
10343 SDValue InvRHS = DAG.getNOT(dl, RHS, VT); in expandAddSubSat()
10344 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
10345 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
10369 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandAddSubSat()
10373 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandAddSubSat()
10374 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandAddSubSat()
10377 SDValue Zero = DAG.getConstant(0, dl, VT); in expandAddSubSat()
10378 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in expandAddSubSat()
10381 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandAddSubSat()
10383 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
10384 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
10387 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff); in expandAddSubSat()
10391 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandAddSubSat()
10393 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
10394 SDValue Not = DAG.getNOT(dl, OverflowMask, VT); in expandAddSubSat()
10395 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
10398 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff); in expandAddSubSat()
10419 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandAddSubSat()
10420 return DAG.getSelect(dl, VT, Overflow, SatMax, SumDiff); in expandAddSubSat()
10427 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
10428 return DAG.getSelect(dl, VT, Overflow, SatMin, SumDiff); in expandAddSubSat()
10434 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
10435 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff, in expandAddSubSat()
10436 DAG.getConstant(BitWidth - 1, dl, VT)); in expandAddSubSat()
10437 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin); in expandAddSubSat()
10438 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); in expandAddSubSat()
10445 EVT VT = LHS.getValueType(); in expandCMP() local
10447 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCMP()
10481 EVT VT = LHS.getValueType(); in expandShlSat() local
10487 assert(VT == RHS.getValueType() && "Expected operands to be the same type"); in expandShlSat()
10488 assert(VT.isInteger() && "Expected operands to be integers"); in expandShlSat()
10490 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandShlSat()
10495 unsigned BW = VT.getScalarSizeInBits(); in expandShlSat()
10496 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandShlSat()
10497 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
10499 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
10503 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT); in expandShlSat()
10504 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT); in expandShlSat()
10506 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT); in expandShlSat()
10507 SatVal = DAG.getSelect(dl, VT, Cond, SatMin, SatMax); in expandShlSat()
10509 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT); in expandShlSat()
10512 return DAG.getSelect(dl, VT, Cond, SatVal, Result); in expandShlSat()
10539 EVT VT = LL.getValueType(); in forceExpandWideMUL() local
10540 unsigned Bits = VT.getSizeInBits(); in forceExpandWideMUL()
10543 DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, VT); in forceExpandWideMUL()
10544 SDValue LLL = DAG.getNode(ISD::AND, dl, VT, LL, Mask); in forceExpandWideMUL()
10545 SDValue RLL = DAG.getNode(ISD::AND, dl, VT, RL, Mask); in forceExpandWideMUL()
10547 SDValue T = DAG.getNode(ISD::MUL, dl, VT, LLL, RLL); in forceExpandWideMUL()
10548 SDValue TL = DAG.getNode(ISD::AND, dl, VT, T, Mask); in forceExpandWideMUL()
10550 SDValue Shift = DAG.getShiftAmountConstant(HalfBits, VT, dl); in forceExpandWideMUL()
10551 SDValue TH = DAG.getNode(ISD::SRL, dl, VT, T, Shift); in forceExpandWideMUL()
10552 SDValue LLH = DAG.getNode(ISD::SRL, dl, VT, LL, Shift); in forceExpandWideMUL()
10553 SDValue RLH = DAG.getNode(ISD::SRL, dl, VT, RL, Shift); in forceExpandWideMUL()
10555 SDValue U = DAG.getNode(ISD::ADD, dl, VT, in forceExpandWideMUL()
10556 DAG.getNode(ISD::MUL, dl, VT, LLH, RLL), TH); in forceExpandWideMUL()
10557 SDValue UL = DAG.getNode(ISD::AND, dl, VT, U, Mask); in forceExpandWideMUL()
10558 SDValue UH = DAG.getNode(ISD::SRL, dl, VT, U, Shift); in forceExpandWideMUL()
10560 SDValue V = DAG.getNode(ISD::ADD, dl, VT, in forceExpandWideMUL()
10561 DAG.getNode(ISD::MUL, dl, VT, LLL, RLH), UL); in forceExpandWideMUL()
10562 SDValue VH = DAG.getNode(ISD::SRL, dl, VT, V, Shift); in forceExpandWideMUL()
10565 DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::MUL, dl, VT, LLH, RLH), in forceExpandWideMUL()
10566 DAG.getNode(ISD::ADD, dl, VT, UH, VH)); in forceExpandWideMUL()
10567 Lo = DAG.getNode(ISD::ADD, dl, VT, TL, in forceExpandWideMUL()
10568 DAG.getNode(ISD::SHL, dl, VT, V, Shift)); in forceExpandWideMUL()
10570 Hi = DAG.getNode(ISD::ADD, dl, VT, W, in forceExpandWideMUL()
10571 DAG.getNode(ISD::ADD, dl, VT, in forceExpandWideMUL()
10572 DAG.getNode(ISD::MUL, dl, VT, RH, LL), in forceExpandWideMUL()
10573 DAG.getNode(ISD::MUL, dl, VT, RL, LH))); in forceExpandWideMUL()
10608 EVT VT = LHS.getValueType(); in forceExpandWideMUL() local
10609 assert(RHS.getValueType() == VT && "Mismatching operand types"); in forceExpandWideMUL()
10616 unsigned LoSize = VT.getFixedSizeInBits(); in forceExpandWideMUL()
10618 ISD::SRA, dl, VT, LHS, in forceExpandWideMUL()
10621 ISD::SRA, dl, VT, RHS, in forceExpandWideMUL()
10624 HiLHS = DAG.getConstant(0, dl, VT); in forceExpandWideMUL()
10625 HiRHS = DAG.getConstant(0, dl, VT); in forceExpandWideMUL()
10627 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); in forceExpandWideMUL()
10642 EVT VT = LHS.getValueType(); in expandFixedPointMul() local
10648 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFixedPointMul()
10649 unsigned VTSize = VT.getScalarSizeInBits(); in expandFixedPointMul()
10654 if (isOperationLegalOrCustom(ISD::MUL, VT)) in expandFixedPointMul()
10655 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10656 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) { in expandFixedPointMul()
10658 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10661 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
10665 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandFixedPointMul()
10666 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
10669 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
10671 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); in expandFixedPointMul()
10672 return DAG.getSelect(dl, VT, Overflow, Result, Product); in expandFixedPointMul()
10673 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) { in expandFixedPointMul()
10675 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10680 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
10681 return DAG.getSelect(dl, VT, Overflow, SatMax, Product); in expandFixedPointMul()
10696 if (isOperationLegalOrCustom(LoHiOp, VT)) { in expandFixedPointMul()
10697 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS); in expandFixedPointMul()
10700 } else if (isOperationLegalOrCustom(HiOp, VT)) { in expandFixedPointMul()
10701 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10702 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS); in expandFixedPointMul()
10709 Lo = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in expandFixedPointMul()
10713 Hi = DAG.getNode(ISD::TRUNCATE, dl, VT, Shifted); in expandFixedPointMul()
10714 } else if (VT.isVector()) { in expandFixedPointMul()
10729 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
10730 DAG.getShiftAmountConstant(Scale, VT, dl)); in expandFixedPointMul()
10742 dl, VT); in expandFixedPointMul()
10744 DAG.getConstant(MaxVal, dl, VT), Result, in expandFixedPointMul()
10753 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT); in expandFixedPointMul()
10754 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT); in expandFixedPointMul()
10757 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
10758 DAG.getShiftAmountConstant(VTSize - 1, VT, dl)); in expandFixedPointMul()
10762 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
10766 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result); in expandFixedPointMul()
10774 dl, VT); in expandFixedPointMul()
10780 dl, VT); in expandFixedPointMul()
10793 EVT VT = LHS.getValueType(); in expandFixedPointDiv() local
10796 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFixedPointDiv()
10826 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
10827 DAG.getShiftAmountConstant(LHSShift, VT, dl)); in expandFixedPointDiv()
10829 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
10830 DAG.getShiftAmountConstant(RHSShift, VT, dl)); in expandFixedPointDiv()
10841 if (isTypeLegal(VT) && in expandFixedPointDiv()
10842 isOperationLegalOrCustom(ISD::SDIVREM, VT)) { in expandFixedPointDiv()
10844 DAG.getVTList(VT, VT), in expandFixedPointDiv()
10849 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
10851 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
10854 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointDiv()
10859 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
10860 DAG.getConstant(1, dl, VT)); in expandFixedPointDiv()
10861 Quot = DAG.getSelect(dl, VT, in expandFixedPointDiv()
10865 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
10960 EVT VT = Node->getValueType(0); in expandMULO() local
10961 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandMULO()
10973 SDValue ShiftAmt = DAG.getShiftAmountConstant(C.logBase2(), VT, dl); in expandMULO()
10974 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
10977 dl, VT, Result, ShiftAmt), in expandMULO()
10983 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); in expandMULO()
10984 if (VT.isVector()) in expandMULO()
10986 EVT::getVectorVT(*DAG.getContext(), WideVT, VT.getVectorElementCount()); in expandMULO()
10993 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) { in expandMULO()
10994 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
10995 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); in expandMULO()
10996 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) { in expandMULO()
10997 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, in expandMULO()
11004 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
11006 DAG.getShiftAmountConstant(VT.getScalarSizeInBits(), WideVT, dl); in expandMULO()
11007 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
11010 if (VT.isVector()) in expandMULO()
11019 VT.getScalarSizeInBits() - 1, BottomHalf.getValueType(), dl); in expandMULO()
11020 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
11024 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
11041 EVT VT = Op.getValueType(); in expandVecReduce() local
11043 if (VT.isScalableVector()) in expandVecReduce()
11048 if (VT.isPow2VectorType()) { in expandVecReduce()
11049 while (VT.getVectorNumElements() > 1) { in expandVecReduce()
11050 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in expandVecReduce()
11057 VT = HalfVT; in expandVecReduce()
11061 EVT EltVT = VT.getVectorElementType(); in expandVecReduce()
11062 unsigned NumElts = VT.getVectorNumElements(); in expandVecReduce()
11083 EVT VT = VecOp.getValueType(); in expandVecReduceSeq() local
11084 EVT EltVT = VT.getVectorElementType(); in expandVecReduceSeq()
11086 if (VT.isScalableVector()) in expandVecReduceSeq()
11090 unsigned NumElts = VT.getVectorNumElements(); in expandVecReduceSeq()
11106 EVT VT = Node->getValueType(0); in expandREM() local
11113 if (isOperationLegalOrCustom(DivRemOpc, VT)) { in expandREM()
11114 SDVTList VTs = DAG.getVTList(VT, VT); in expandREM()
11118 if (isOperationLegalOrCustom(DivOpc, VT)) { in expandREM()
11120 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor); in expandREM()
11121 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
11122 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
11308 EVT VT = Node->getValueType(0); in expandFP_ROUND() local
11310 if (VT.getScalarType() == MVT::bf16) { in expandFP_ROUND()
11312 return DAG.getNode(ISD::FP_TO_BF16, dl, VT, Node->getOperand(0)); in expandFP_ROUND()
11325 EVT F32 = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in expandFP_ROUND()
11354 return DAG.getNode(ISD::BITCAST, dl, VT, Op); in expandFP_ROUND()
11365 EVT VT = Node->getValueType(0); in expandVectorSplice() local
11382 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false); in expandVectorSplice()
11384 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), in expandVectorSplice()
11385 VT.getVectorElementCount() * 2); in expandVectorSplice()
11397 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinValue())); in expandVectorSplice()
11404 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2)); in expandVectorSplice()
11406 return DAG.getLoad(VT, DL, StoreV2, StackPtr, in expandVectorSplice()
11413 TypeSize EltByteSize = VT.getVectorElementType().getStoreSize(); in expandVectorSplice()
11417 if (TrailingElts > VT.getVectorMinNumElements()) { in expandVectorSplice()
11421 VT.getStoreSize().getKnownMinValue())); in expandVectorSplice()
11429 return DAG.getLoad(VT, DL, StoreV2, StackPtr2, in expandVectorSplice()
11535 bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, in LegalizeSetCCCondCode() argument
11654 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
11655 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
11657 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC1, Mask, EVL); in LegalizeSetCCCondCode()
11658 SetCC2 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC2, Mask, EVL); in LegalizeSetCCCondCode()
11663 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
11664 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
11666 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, LHS, CC1, Mask, EVL); in LegalizeSetCCCondCode()
11667 SetCC2 = DAG.getSetCCVP(dl, VT, RHS, RHS, CC2, Mask, EVL); in LegalizeSetCCCondCode()
11674 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); in LegalizeSetCCCondCode()
11679 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2, Mask, EVL); in LegalizeSetCCCondCode()