Lines Matching refs:Part
382 Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) { in generatePerPart() argument
387 Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed); in generatePerPart()
388 Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed); in generatePerPart()
398 Value *A = State.get(getOperand(0), Part); in generatePerPart()
403 Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed); in generatePerPart()
404 Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed); in generatePerPart()
408 Value *Cond = State.get(getOperand(0), Part); in generatePerPart()
409 Value *Op1 = State.get(getOperand(1), Part); in generatePerPart()
410 Value *Op2 = State.get(getOperand(2), Part); in generatePerPart()
415 Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0)); in generatePerPart()
417 Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0)); in generatePerPart()
446 Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1); in generatePerPart()
449 Value *V2 = State.get(getOperand(1), Part); in generatePerPart()
453 if (Part != 0) in generatePerPart()
481 assert(Part == 0 && "No unrolling expected for predicated vectorization."); in generatePerPart()
491 if (Part == 0) in generatePerPart()
496 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part); in generatePerPart()
501 if (Part != 0) in generatePerPart()
504 Value *Cond = State.get(getOperand(0), VPIteration(Part, 0)); in generatePerPart()
522 if (Part != 0) in generatePerPart()
525 Value *IV = State.get(getOperand(0), Part, /*IsScalar*/ true); in generatePerPart()
526 Value *TC = State.get(getOperand(1), Part, /*IsScalar*/ true); in generatePerPart()
546 if (Part != 0) in generatePerPart()
561 for (unsigned Part = 0; Part < State.UF; ++Part) in generatePerPart() local
562 RdxParts[Part] = State.get(LoopExitingDef, Part, PhiR->isInLoop()); in generatePerPart()
570 for (unsigned Part = 0; Part < State.UF; ++Part) in generatePerPart() local
571 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); in generatePerPart()
585 for (unsigned Part = 1; Part < State.UF; ++Part) { in generatePerPart() local
586 Value *RdxPart = RdxParts[Part]; in generatePerPart()
621 if (Part != 0) in generatePerPart()
645 Value *A = State.get(getOperand(0), Part); in generatePerPart()
646 Value *B = State.get(getOperand(1), Part); in generatePerPart()
652 Value *Ptr = State.get(getOperand(0), Part, /* IsScalar */ true); in generatePerPart()
653 Value *Addend = State.get(getOperand(1), Part, /* IsScalar */ true); in generatePerPart()
657 if (Part != 0) in generatePerPart()
660 State.get(getOperand(0), Part, /* IsScalar */ true); in generatePerPart()
662 State.get(getOperand(1), Part, /* IsScalar */ true); in generatePerPart()
716 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
720 Value *GeneratedValue = generatePerLane(State, VPIteration(Part, Lane)); in execute()
722 State.set(this, GeneratedValue, VPIteration(Part, Lane)); in execute()
727 if (Part != 0 && OnlyFirstPartUsed && hasResult()) { in execute()
729 State.set(this, Part0, Part, in execute()
734 Value *GeneratedValue = generatePerPart(State, Part); in execute()
742 State.set(this, GeneratedValue, Part, in execute()
876 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
895 Arg = State.get(I.value(), VPIteration(Part, 0)); in execute()
897 Arg = State.get(I.value(), Part); in execute()
928 State.set(this, V, Part); in execute()
986 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
987 Value *Cond = InvarCond ? InvarCond : State.get(getCond(), Part); in execute()
988 Value *Op0 = State.get(getOperand(1), Part); in execute()
989 Value *Op1 = State.get(getOperand(2), Part); in execute()
991 State.set(this, Sel, Part); in execute()
1076 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1079 Ops.push_back(State.get(VPOp, Part)); in execute()
1087 State.set(this, V, Part); in execute()
1094 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1095 Value *Op = State.get(getOperand(0), Part); in execute()
1098 State.set(this, Freeze, Part); in execute()
1106 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1107 Value *A = State.get(getOperand(0), Part); in execute()
1108 Value *B = State.get(getOperand(1), Part); in execute()
1119 State.set(this, C, Part); in execute()
1135 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1137 State.VF) == State.get(this, Part)->getType() && in execute()
1161 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1162 if (Part > 0 && Op->isLiveIn()) { in execute()
1164 State.set(this, State.get(this, 0), Part); in execute()
1167 Value *A = State.get(Op, Part); in execute()
1169 State.set(this, Cast, Part); in execute()
1334 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1335 State.set(this, LastInduction, Part); in execute()
1449 StartPart = State.Instance->Part; in execute()
1454 for (unsigned Part = StartPart; Part < EndPart; ++Part) { in execute() local
1455 Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part); in execute()
1464 State.set(this, Add, Part); in execute()
1483 State.set(this, Add, VPIteration(Part, Lane)); in execute()
1527 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1529 State.set(this, EntryPart, Part); in execute()
1540 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1545 : State.get(getOperand(0), Part); in execute()
1555 Indices.push_back(State.get(Operand, Part)); in execute()
1564 State.set(this, NewGEP, Part); in execute()
1589 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1596 Type *IndexTy = State.VF.isScalable() && (IsReverse || Part > 0) in execute()
1609 ConstantInt::get(IndexTy, -(int64_t)Part), RunTimeVF); in execute()
1616 Value *Increment = createStepForVF(Builder, IndexTy, State.VF, Part); in execute()
1620 State.set(this, PartPtr, Part, /*IsScalar*/ true); in execute()
1658 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1661 Value *In0 = State.get(getIncomingValue(In), Part, OnlyFirstLaneUsed); in execute()
1663 Entry[Part] = In0; // Initialize with the first incoming value. in execute()
1667 Value *Cond = State.get(getMask(In), Part, OnlyFirstLaneUsed); in execute()
1668 Entry[Part] = in execute()
1669 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); in execute()
1673 for (unsigned Part = 0; Part < State.UF; ++Part) in execute() local
1674 State.set(this, Entry[Part], Part, OnlyFirstLaneUsed); in execute()
1708 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
1709 Value *NewVecOp = State.get(getVecOp(), Part); in execute()
1711 Value *NewCond = State.get(Cond, Part, State.VF.isScalar()); in execute()
1735 PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true); in execute()
1746 State.set(this, NextInChain, Part, /*IsScalar*/ true); in execute()
1889 Value *VPScalarCastRecipe ::generate(VPTransformState &State, unsigned Part) { in generate() argument
1897 Value *Op = State.get(getOperand(0), VPIteration(Part, 0)); in generate()
1907 for (unsigned Part = 0; Part != State.UF; ++Part) { in execute() local
1911 if (Part > 0 && IsUniformAcrossVFsAndUFs) in execute()
1914 Res = generate(State, Part); in execute()
1915 State.set(this, Res, VPIteration(Part, 0)); in execute()
1933 unsigned Part = State.Instance->Part; in execute() local
1939 ConditionBit = State.get(BlockInMask, Part); in execute()
1972 unsigned Part = State.Instance->Part; in execute() local
1973 if (State.hasVectorValue(getOperand(0), Part)) { in execute()
1974 Value *VectorValue = State.get(getOperand(0), Part); in execute()
1979 if (State.hasVectorValue(this, Part)) in execute()
1980 State.reset(this, VPhi, Part); in execute()
1982 State.set(this, VPhi, Part); in execute()
1985 State.reset(getOperand(0), VPhi, Part); in execute()
2167 for (unsigned Part = 0; Part < State.UF; Part++) { in execute() local
2168 Value *AddrPart = State.get(Addr, VPIteration(Part, 0)); in execute()
2195 unsigned Part, Value *MaskForGaps) -> Value * { in execute() argument
2200 auto *BlockInMaskPart = State.get(BlockInMask, Part); in execute()
2212 Value *BlockInMaskPart = State.get(BlockInMask, Part); in execute()
2234 for (unsigned Part = 0; Part < State.UF; Part++) { in execute() local
2237 Value *GroupMask = CreateGroupMask(Part, MaskForGaps); in execute()
2238 NewLoad = State.Builder.CreateMaskedLoad(VecTy, AddrParts[Part], in execute()
2243 VecTy, AddrParts[Part], Group->getAlign(), "wide.vec"); in execute()
2254 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
2258 Intrinsic::vector_deinterleave2, VecTy, NewLoads[Part], in execute()
2279 State.set(VPDefs[J], StridedVec, Part); in execute()
2299 for (unsigned Part = 0; Part < State.UF; Part++) { in execute() local
2301 NewLoads[Part], StrideMask, "strided.vec"); in execute()
2314 State.set(VPDefs[J], StridedVec, Part); in execute()
2330 for (unsigned Part = 0; Part < State.UF; Part++) { in execute() local
2346 Value *StoredVec = State.get(StoredValues[StoredIdx], Part); in execute()
2365 Value *GroupMask = CreateGroupMask(Part, MaskForGaps); in execute()
2367 IVec, AddrParts[Part], Group->getAlign(), GroupMask); in execute()
2369 NewStoreInstr = State.Builder.CreateAlignedStore(IVec, AddrParts[Part], in execute()
2415 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute() local
2416 State.set(this, EntryPart, Part, /*IsScalar*/ true); in execute()
2473 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) in execute() local
2474 State.set(this, Res, {Part, 0}); in execute()
2494 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute() local
2495 Value *VStep = createStepForVF(Builder, STy, VF, Part); in execute()
2502 State.set(this, CanonicalVectorIV, Part); in execute()
2574 for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { in execute() local
2577 State.set(this, EntryPart, Part, IsInLoop); in execute()
2608 for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) { in execute() local
2609 Value *EntryPart = State.get(this, Part, IsInLoop); in execute()
2612 Value *StartVal = (Part == 0) ? StartV : Iden; in execute()
2663 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) { in execute() local
2664 Value *StartMask = State.get(getOperand(0), Part); in execute()
2669 State.set(this, EntryPart, Part); in execute()