Lines Matching refs:State
195 void VPLiveOut::fixPhi(VPlan &Plan, VPTransformState &State) { in fixPhi() argument
199 : VPLane::getLastLaneForVF(State.VF); in fixPhi()
209 BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB]; in fixPhi()
212 State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt()); in fixPhi()
213 Value *V = State.get(ExitValue, VPIteration(State.UF - 1, Lane)); in fixPhi()
372 Value *VPInstruction::generatePerLane(VPTransformState &State, in generatePerLane() argument
374 IRBuilderBase &Builder = State.Builder; in generatePerLane()
378 return Builder.CreatePtrAdd(State.get(getOperand(0), Lane), in generatePerLane()
379 State.get(getOperand(1), Lane), Name); in generatePerLane()
382 Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) { in generatePerPart() argument
383 IRBuilderBase &Builder = State.Builder; in generatePerPart()
387 Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed); in generatePerPart()
388 Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed); in generatePerPart()
398 Value *A = State.get(getOperand(0), Part); in generatePerPart()
403 Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed); in generatePerPart()
404 Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed); in generatePerPart()
408 Value *Cond = State.get(getOperand(0), Part); in generatePerPart()
409 Value *Op1 = State.get(getOperand(1), Part); in generatePerPart()
410 Value *Op2 = State.get(getOperand(2), Part); in generatePerPart()
415 Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0)); in generatePerPart()
417 Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0)); in generatePerPart()
421 if (State.VF.isScalar()) in generatePerPart()
426 auto *PredTy = VectorType::get(Int1Ty, State.VF); in generatePerPart()
445 auto *V1 = State.get(getOperand(0), 0); in generatePerPart()
446 Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1); in generatePerPart()
449 Value *V2 = State.get(getOperand(1), Part); in generatePerPart()
454 return State.get(this, 0, /*IsScalar*/ true); in generatePerPart()
456 Value *ScalarTC = State.get(getOperand(0), {0, 0}); in generatePerPart()
458 createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF); in generatePerPart()
466 auto GetEVL = [=](VPTransformState &State, Value *AVL) { in generatePerPart() argument
471 assert(State.VF.isScalable() && "Expected scalable vector factor."); in generatePerPart()
472 Value *VFArg = State.Builder.getInt32(State.VF.getKnownMinValue()); in generatePerPart()
474 Value *EVL = State.Builder.CreateIntrinsic( in generatePerPart()
475 State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length, in generatePerPart()
476 {AVL, VFArg, State.Builder.getTrue()}); in generatePerPart()
483 Value *Index = State.get(getOperand(0), VPIteration(0, 0)); in generatePerPart()
484 Value *TripCount = State.get(getOperand(1), VPIteration(0, 0)); in generatePerPart()
485 Value *AVL = State.Builder.CreateSub(TripCount, Index); in generatePerPart()
486 Value *EVL = GetEVL(State, AVL); in generatePerPart()
490 auto *IV = State.get(getOperand(0), VPIteration(0, 0)); in generatePerPart()
496 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part); in generatePerPart()
504 Value *Cond = State.get(getOperand(0), VPIteration(Part, 0)); in generatePerPart()
518 CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]); in generatePerPart()
525 Value *IV = State.get(getOperand(0), Part, /*IsScalar*/ true); in generatePerPart()
526 Value *TC = State.get(getOperand(1), Part, /*IsScalar*/ true); in generatePerPart()
540 State.CFG.VPBB2IRBB[Header]); in generatePerPart()
547 return State.get(this, 0, /*IsScalar*/ true); in generatePerPart()
560 VectorParts RdxParts(State.UF); in generatePerPart()
561 for (unsigned Part = 0; Part < State.UF; ++Part) in generatePerPart()
562 RdxParts[Part] = State.get(LoopExitingDef, Part, PhiR->isInLoop()); in generatePerPart()
568 if (State.VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) { in generatePerPart()
569 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF); in generatePerPart()
570 for (unsigned Part = 0; Part < State.UF; ++Part) in generatePerPart()
580 ReducedPartRdx = RdxParts[State.UF - 1]; in generatePerPart()
585 for (unsigned Part = 1; Part < State.UF; ++Part) { in generatePerPart()
597 if ((State.VF.isVector() || in generatePerPart()
622 return State.get(this, 0, /*IsScalar*/ true); in generatePerPart()
628 if (State.VF.isVector()) { in generatePerPart()
629 assert(Offset <= State.VF.getKnownMinValue() && in generatePerPart()
632 Res = State.get( in generatePerPart()
634 VPIteration(State.UF - 1, VPLane::getLaneFromEnd(State.VF, Offset))); in generatePerPart()
636 assert(Offset <= State.UF && "invalid offset to extract from"); in generatePerPart()
638 Res = State.get(getOperand(0), State.UF - Offset); in generatePerPart()
645 Value *A = State.get(getOperand(0), Part); in generatePerPart()
646 Value *B = State.get(getOperand(1), Part); in generatePerPart()
652 Value *Ptr = State.get(getOperand(0), Part, /* IsScalar */ true); in generatePerPart()
653 Value *Addend = State.get(getOperand(1), Part, /* IsScalar */ true); in generatePerPart()
658 return State.get(this, 0, /*IsScalar*/ true); in generatePerPart()
660 State.get(getOperand(0), Part, /* IsScalar */ true); in generatePerPart()
662 State.get(getOperand(1), Part, /* IsScalar */ true); in generatePerPart()
666 State.CFG in generatePerPart()
702 void VPInstruction::execute(VPTransformState &State) { in execute() argument
703 assert(!State.Instance && "VPInstruction executing an Instance"); in execute()
704 IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder); in execute()
709 State.Builder.setFastMathFlags(getFastMathFlags()); in execute()
710 State.setDebugLocFrom(getDebugLoc()); in execute()
716 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
718 for (unsigned Lane = 0, NumLanes = State.VF.getKnownMinValue(); in execute()
720 Value *GeneratedValue = generatePerLane(State, VPIteration(Part, Lane)); in execute()
722 State.set(this, GeneratedValue, VPIteration(Part, Lane)); in execute()
728 Value *Part0 = State.get(this, 0, /*IsScalar*/ GeneratesPerFirstLaneOnly); in execute()
729 State.set(this, Part0, Part, in execute()
734 Value *GeneratedValue = generatePerPart(State, Part); in execute()
740 State.VF.isScalar()) && in execute()
742 State.set(this, GeneratedValue, Part, in execute()
865 void VPWidenCallRecipe::execute(VPTransformState &State) { in execute() argument
866 assert(State.VF.isVector() && "not widening"); in execute()
870 State.setDebugLocFrom(getDebugLoc()); in execute()
876 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
882 CalledScalarFn->getReturnType()->getScalarType(), State.VF)); in execute()
890 Arg = State.get(I.value(), VPIteration(0, 0)); in execute()
895 Arg = State.get(I.value(), VPIteration(Part, 0)); in execute()
897 Arg = State.get(I.value(), Part); in execute()
907 Module *M = State.Builder.GetInsertBlock()->getModule(); in execute()
922 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles); in execute()
928 State.set(this, V, Part); in execute()
929 State.addMetadata(V, CI); in execute()
976 void VPWidenSelectRecipe::execute(VPTransformState &State) { in execute() argument
977 State.setDebugLocFrom(getDebugLoc()); in execute()
984 isInvariantCond() ? State.get(getCond(), VPIteration(0, 0)) : nullptr; in execute()
986 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
987 Value *Cond = InvarCond ? InvarCond : State.get(getCond(), Part); in execute()
988 Value *Op0 = State.get(getOperand(1), Part); in execute()
989 Value *Op1 = State.get(getOperand(2), Part); in execute()
990 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1); in execute()
991 State.set(this, Sel, Part); in execute()
992 State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue())); in execute()
1046 void VPWidenRecipe::execute(VPTransformState &State) { in execute() argument
1047 State.setDebugLocFrom(getDebugLoc()); in execute()
1048 auto &Builder = State.Builder; in execute()
1076 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1079 Ops.push_back(State.get(VPOp, Part)); in execute()
1087 State.set(this, V, Part); in execute()
1088 State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue())); in execute()
1094 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1095 Value *Op = State.get(getOperand(0), Part); in execute()
1098 State.set(this, Freeze, Part); in execute()
1106 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1107 Value *A = State.get(getOperand(0), Part); in execute()
1108 Value *B = State.get(getOperand(1), Part); in execute()
1119 State.set(this, C, Part); in execute()
1120 State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue())); in execute()
1135 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1136 assert(VectorType::get(State.TypeAnalysis.inferScalarType(this), in execute()
1137 State.VF) == State.get(this, Part)->getType() && in execute()
1154 void VPWidenCastRecipe::execute(VPTransformState &State) { in execute() argument
1155 State.setDebugLocFrom(getDebugLoc()); in execute()
1156 auto &Builder = State.Builder; in execute()
1158 assert(State.VF.isVector() && "Not vectorizing?"); in execute()
1159 Type *DestTy = VectorType::get(getResultType(), State.VF); in execute()
1161 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1164 State.set(this, State.get(this, 0), Part); in execute()
1167 Value *A = State.get(Op, Part); in execute()
1169 State.set(this, Cast, Part); in execute()
1170 State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue())); in execute()
1254 void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) { in execute() argument
1255 assert(!State.Instance && "Int or FP induction being replicated."); in execute()
1260 IRBuilderBase &Builder = State.Builder; in execute()
1262 assert(State.VF.isVector() && "must have vector VF"); in execute()
1274 Value *Step = State.get(getStepValue(), VPIteration(0, 0)); in execute()
1281 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
1292 Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start); in execute()
1294 SplatStart, Zero, Step, ID.getInductionOpcode(), State.VF, State.Builder); in execute()
1313 RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, State.VF); in execute()
1315 RuntimeVF = getRuntimeVF(Builder, StepType, State.VF); in execute()
1324 ? ConstantVector::getSplat(State.VF, cast<Constant>(Mul)) in execute()
1325 : Builder.CreateVectorSplat(State.VF, Mul); in execute()
1331 VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt()); in execute()
1334 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1335 State.set(this, LastInduction, Part); in execute()
1338 State.addMetadata(LastInduction, EntryVal); in execute()
1399 void VPScalarIVStepsRecipe::execute(VPTransformState &State) { in execute() argument
1401 IRBuilder<>::FastMathFlagGuard FMFG(State.Builder); in execute()
1403 State.Builder.setFastMathFlags(getFastMathFlags()); in execute()
1408 Value *BaseIV = State.get(getOperand(0), VPIteration(0, 0)); in execute()
1409 Value *Step = State.get(getStepValue(), VPIteration(0, 0)); in execute()
1410 IRBuilderBase &Builder = State.Builder; in execute()
1436 if (!FirstLaneOnly && State.VF.isScalable()) { in execute()
1437 VecIVTy = VectorType::get(BaseIVTy, State.VF); in execute()
1439 Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF)); in execute()
1440 SplatStep = Builder.CreateVectorSplat(State.VF, Step); in execute()
1441 SplatIV = Builder.CreateVectorSplat(State.VF, BaseIV); in execute()
1445 unsigned EndPart = State.UF; in execute()
1447 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue(); in execute()
1448 if (State.Instance) { in execute()
1449 StartPart = State.Instance->Part; in execute()
1451 StartLane = State.Instance->Lane.getKnownLane(); in execute()
1455 Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part); in execute()
1457 if (!FirstLaneOnly && State.VF.isScalable()) { in execute()
1458 auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0); in execute()
1464 State.set(this, Add, Part); in execute()
1478 assert((State.VF.isScalable() || isa<Constant>(StartIdx)) && in execute()
1483 State.set(this, Add, VPIteration(Part, Lane)); in execute()
1498 void VPWidenGEPRecipe::execute(VPTransformState &State) { in execute() argument
1499 assert(State.VF.isVector() && "not widening"); in execute()
1522 Ops.push_back(State.get(getOperand(I), VPIteration(0, 0))); in execute()
1525 State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0], in execute()
1527 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1528 Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, NewGEP); in execute()
1529 State.set(this, EntryPart, Part); in execute()
1530 State.addMetadata(EntryPart, GEP); in execute()
1540 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1544 ? State.get(getOperand(0), VPIteration(0, 0)) in execute()
1545 : State.get(getOperand(0), Part); in execute()
1553 Indices.push_back(State.get(Operand, VPIteration(0, 0))); in execute()
1555 Indices.push_back(State.get(Operand, Part)); in execute()
1560 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr, in execute()
1562 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) && in execute()
1564 State.set(this, NewGEP, Part); in execute()
1565 State.addMetadata(NewGEP, GEP); in execute()
1586 void VPVectorPointerRecipe ::execute(VPTransformState &State) { in execute() argument
1587 auto &Builder = State.Builder; in execute()
1588 State.setDebugLocFrom(getDebugLoc()); in execute()
1589 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1596 Type *IndexTy = State.VF.isScalable() && (IsReverse || Part > 0) in execute()
1599 Value *Ptr = State.get(getOperand(0), VPIteration(0, 0)); in execute()
1606 Value *RunTimeVF = getRuntimeVF(Builder, IndexTy, State.VF); in execute()
1616 Value *Increment = createStepForVF(Builder, IndexTy, State.VF, Part); in execute()
1620 State.set(this, PartPtr, Part, /*IsScalar*/ true); in execute()
1637 void VPBlendRecipe::execute(VPTransformState &State) { in execute() argument
1638 State.setDebugLocFrom(getDebugLoc()); in execute()
1655 VectorParts Entry(State.UF); in execute()
1658 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1661 Value *In0 = State.get(getIncomingValue(In), Part, OnlyFirstLaneUsed); in execute()
1667 Value *Cond = State.get(getMask(In), Part, OnlyFirstLaneUsed); in execute()
1669 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); in execute()
1673 for (unsigned Part = 0; Part < State.UF; ++Part) in execute()
1674 State.set(this, Entry[Part], Part, OnlyFirstLaneUsed); in execute()
1701 void VPReductionRecipe::execute(VPTransformState &State) { in execute() argument
1702 assert(!State.Instance && "Reduction being replicated."); in execute()
1703 Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true); in execute()
1706 IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder); in execute()
1707 State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags()); in execute()
1708 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
1709 Value *NewVecOp = State.get(getVecOp(), Part); in execute()
1711 Value *NewCond = State.get(Cond, Part, State.VF.isScalar()); in execute()
1716 if (State.VF.isVector()) { in execute()
1717 Iden = State.Builder.CreateVectorSplat(VecTy->getElementCount(), Iden); in execute()
1720 Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Iden); in execute()
1726 if (State.VF.isVector()) in execute()
1727 NewRed = createOrderedReduction(State.Builder, RdxDesc, NewVecOp, in execute()
1730 NewRed = State.Builder.CreateBinOp( in execute()
1735 PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true); in execute()
1736 NewRed = createTargetReduction(State.Builder, RdxDesc, NewVecOp); in execute()
1739 NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(), in execute()
1744 NextInChain = State.Builder.CreateBinOp( in execute()
1746 State.set(this, NextInChain, Part, /*IsScalar*/ true); in execute()
1750 void VPReductionEVLRecipe::execute(VPTransformState &State) { in execute() argument
1751 assert(!State.Instance && "Reduction being replicated."); in execute()
1752 assert(State.UF == 1 && in execute()
1755 auto &Builder = State.Builder; in execute()
1762 Value *Prev = State.get(getChainOp(), 0, /*IsScalar*/ true); in execute()
1763 Value *VecOp = State.get(getVecOp(), 0); in execute()
1764 Value *EVL = State.get(getEVL(), VPIteration(0, 0)); in execute()
1771 Mask = State.get(CondOp, 0); in execute()
1773 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue()); in execute()
1787 State.set(this, NewRed, 0, /*IsScalar*/ true); in execute()
1889 Value *VPScalarCastRecipe ::generate(VPTransformState &State, unsigned Part) { in generate() argument
1897 Value *Op = State.get(getOperand(0), VPIteration(Part, 0)); in generate()
1898 return State.Builder.CreateCast(Instruction::CastOps(Opcode), Op, ResultTy); in generate()
1905 void VPScalarCastRecipe ::execute(VPTransformState &State) { in execute() argument
1907 for (unsigned Part = 0; Part != State.UF; ++Part) { in execute()
1912 Res = State.get(this, VPIteration(0, 0)); in execute()
1914 Res = generate(State, Part); in execute()
1915 State.set(this, Res, VPIteration(Part, 0)); in execute()
1930 void VPBranchOnMaskRecipe::execute(VPTransformState &State) { in execute() argument
1931 assert(State.Instance && "Branch on Mask works only on single instance."); in execute()
1933 unsigned Part = State.Instance->Part; in execute()
1934 unsigned Lane = State.Instance->Lane.getKnownLane(); in execute()
1939 ConditionBit = State.get(BlockInMask, Part); in execute()
1941 ConditionBit = State.Builder.CreateExtractElement( in execute()
1942 ConditionBit, State.Builder.getInt32(Lane)); in execute()
1944 ConditionBit = State.Builder.getTrue(); in execute()
1948 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator(); in execute()
1951 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit); in execute()
1956 void VPPredInstPHIRecipe::execute(VPTransformState &State) { in execute() argument
1957 assert(State.Instance && "Predicated instruction PHI works per instance."); in execute()
1959 cast<Instruction>(State.get(getOperand(0), *State.Instance)); in execute()
1972 unsigned Part = State.Instance->Part; in execute()
1973 if (State.hasVectorValue(getOperand(0), Part)) { in execute()
1974 Value *VectorValue = State.get(getOperand(0), Part); in execute()
1976 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2); in execute()
1979 if (State.hasVectorValue(this, Part)) in execute()
1980 State.reset(this, VPhi, Part); in execute()
1982 State.set(this, VPhi, Part); in execute()
1985 State.reset(getOperand(0), VPhi, Part); in execute()
1988 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2); in execute()
1992 if (State.hasScalarValue(this, *State.Instance)) in execute()
1993 State.reset(this, Phi, *State.Instance); in execute()
1995 State.set(this, Phi, *State.Instance); in execute()
1998 State.reset(getOperand(0), Phi, *State.Instance); in execute()
2129 void VPInterleaveRecipe::execute(VPTransformState &State) { in execute() argument
2130 assert(!State.Instance && "Interleave group being replicated."); in execute()
2137 auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor); in execute()
2157 getRuntimeVF(State.Builder, State.Builder.getInt32Ty(), State.VF); in execute()
2158 Idx = State.Builder.CreateSub(RuntimeVF, State.Builder.getInt32(1)); in execute()
2159 Idx = State.Builder.CreateMul(Idx, in execute()
2160 State.Builder.getInt32(Group->getFactor())); in execute()
2161 Idx = State.Builder.CreateAdd(Idx, State.Builder.getInt32(Index)); in execute()
2162 Idx = State.Builder.CreateNeg(Idx); in execute()
2164 Idx = State.Builder.getInt32(-Index); in execute()
2167 for (unsigned Part = 0; Part < State.UF; Part++) { in execute()
2168 Value *AddrPart = State.get(Addr, VPIteration(Part, 0)); in execute()
2170 State.setDebugLocFrom(I->getDebugLoc()); in execute()
2187 AddrPart = State.Builder.CreateGEP(ScalarTy, AddrPart, Idx, "", InBounds); in execute()
2191 State.setDebugLocFrom(Instr->getDebugLoc()); in execute()
2194 auto CreateGroupMask = [&BlockInMask, &State, &InterleaveFactor]( in execute()
2196 if (State.VF.isScalable()) { in execute()
2200 auto *BlockInMaskPart = State.get(BlockInMask, Part); in execute()
2202 auto *MaskTy = VectorType::get(State.Builder.getInt1Ty(), in execute()
2203 State.VF.getKnownMinValue() * 2, true); in execute()
2204 return State.Builder.CreateIntrinsic( in execute()
2212 Value *BlockInMaskPart = State.get(BlockInMask, Part); in execute()
2213 Value *ShuffledMask = State.Builder.CreateShuffleVector( in execute()
2215 createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()), in execute()
2217 return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And, in execute()
2227 MaskForGaps = createBitMaskForGaps(State.Builder, in execute()
2228 State.VF.getKnownMinValue(), *Group); in execute()
2234 for (unsigned Part = 0; Part < State.UF; Part++) { in execute()
2238 NewLoad = State.Builder.CreateMaskedLoad(VecTy, AddrParts[Part], in execute()
2242 NewLoad = State.Builder.CreateAlignedLoad( in execute()
2249 const DataLayout &DL = State.CFG.PrevBB->getDataLayout(); in execute()
2254 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute()
2257 Value *DI = State.Builder.CreateIntrinsic( in execute()
2267 Value *StridedVec = State.Builder.CreateExtractValue(DI, I); in execute()
2270 VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF); in execute()
2272 createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL); in execute()
2277 State.Builder.CreateVectorReverse(StridedVec, "reverse"); in execute()
2279 State.set(VPDefs[J], StridedVec, Part); in execute()
2298 createStrideMask(I, InterleaveFactor, State.VF.getKnownMinValue()); in execute()
2299 for (unsigned Part = 0; Part < State.UF; Part++) { in execute()
2300 Value *StridedVec = State.Builder.CreateShuffleVector( in execute()
2305 assert(!State.VF.isScalable() && "VF is assumed to be non scalable."); in execute()
2306 VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF); in execute()
2308 createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL); in execute()
2312 StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse"); in execute()
2314 State.set(VPDefs[J], StridedVec, Part); in execute()
2322 auto *SubVT = VectorType::get(ScalarTy, State.VF); in execute()
2326 createBitMaskForGaps(State.Builder, State.VF.getKnownMinValue(), *Group); in execute()
2327 assert((!MaskForGaps || !State.VF.isScalable()) && in execute()
2330 for (unsigned Part = 0; Part < State.UF; Part++) { in execute()
2346 Value *StoredVec = State.get(StoredValues[StoredIdx], Part); in execute()
2350 StoredVec = State.Builder.CreateVectorReverse(StoredVec, "reverse"); in execute()
2355 StoredVec = createBitOrPointerCast(State.Builder, StoredVec, SubVT, DL); in execute()
2362 interleaveVectors(State.Builder, StoredVecs, "interleaved.vec"); in execute()
2366 NewStoreInstr = State.Builder.CreateMaskedStore( in execute()
2369 NewStoreInstr = State.Builder.CreateAlignedStore(IVec, AddrParts[Part], in execute()
2407 void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) { in execute() argument
2410 EntryPart->insertBefore(State.CFG.PrevBB->getFirstInsertionPt()); in execute()
2412 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
2415 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute()
2416 State.set(this, EntryPart, Part, /*IsScalar*/ true); in execute()
2463 void VPExpandSCEVRecipe::execute(VPTransformState &State) { in execute() argument
2464 assert(!State.Instance && "cannot be used in per-lane"); in execute()
2465 const DataLayout &DL = State.CFG.PrevBB->getDataLayout(); in execute()
2469 &*State.Builder.GetInsertPoint()); in execute()
2470 assert(!State.ExpandedSCEVs.contains(Expr) && in execute()
2472 State.ExpandedSCEVs[Expr] = Res; in execute()
2473 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute()
2474 State.set(this, Res, {Part, 0}); in execute()
2486 void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) { in execute() argument
2487 Value *CanonicalIV = State.get(getOperand(0), 0, /*IsScalar*/ true); in execute()
2489 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); in execute()
2490 ElementCount VF = State.VF; in execute()
2494 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute()
2502 State.set(this, CanonicalVectorIV, Part); in execute()
2516 void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) { in execute() argument
2517 auto &Builder = State.Builder; in execute()
2521 Type *VecTy = State.VF.isScalar() in execute()
2523 : VectorType::get(VectorInit->getType(), State.VF); in execute()
2525 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
2526 if (State.VF.isVector()) { in execute()
2531 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF); in execute()
2539 EntryPart->insertBefore(State.CFG.PrevBB->getFirstInsertionPt()); in execute()
2541 State.set(this, EntryPart, 0); in execute()
2554 void VPReductionPHIRecipe::execute(VPTransformState &State) { in execute() argument
2555 auto &Builder = State.Builder; in execute()
2566 bool ScalarPHI = State.VF.isScalar() || IsInLoop; in execute()
2568 : VectorType::get(StartV->getType(), State.VF); in execute()
2570 BasicBlock *HeaderBB = State.CFG.PrevBB; in execute()
2571 assert(State.CurrentVectorLoop->getHeader() == HeaderBB && in execute()
2573 unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF; in execute()
2577 State.set(this, EntryPart, Part, IsInLoop); in execute()
2580 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
2593 Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident"); in execute()
2600 Iden = Builder.CreateVectorSplat(State.VF, Iden); in execute()
2609 Value *EntryPart = State.get(this, Part, IsInLoop); in execute()
2628 void VPWidenPHIRecipe::execute(VPTransformState &State) { in execute() argument
2632 Value *Op0 = State.get(getOperand(0), 0); in execute()
2634 Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi"); in execute()
2635 State.set(this, VecPhi, 0); in execute()
2661 void VPActiveLaneMaskPHIRecipe::execute(VPTransformState &State) { in execute() argument
2662 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
2663 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute()
2664 Value *StartMask = State.get(getOperand(0), Part); in execute()
2666 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask"); in execute()
2669 State.set(this, EntryPart, Part); in execute()
2684 void VPEVLBasedIVPHIRecipe::execute(VPTransformState &State) { in execute() argument
2685 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this); in execute()
2686 assert(State.UF == 1 && "Expected unroll factor 1 for VP vectorization."); in execute()
2687 Value *Start = State.get(getOperand(0), VPIteration(0, 0)); in execute()
2689 State.Builder.CreatePHI(Start->getType(), 2, "evl.based.iv"); in execute()
2692 State.set(this, EntryPart, 0, /*IsScalar=*/true); in execute()