diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 3fbeb7e772a2..6eef887a076b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -4100,6 +4100,18 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF, // TODO: Using the original IR may not be accurate. // Currently, ARM will use the underlying IR to calculate gather/scatter // instruction cost. + [[maybe_unused]] auto IsReverseMask = [this, R]() { + VPValue *Mask = getMask(); + if (!Mask) + return false; + + if (isa(R)) + return match(Mask, m_Intrinsic()); + + return match(Mask, m_Reverse(m_VPValue())); + }; + assert(!IsReverseMask() && + "Inconsecutive memory access should not have reverse order"); Type *PtrTy = getAddr()->getScalarType(); const Value *Ptr = getAddr()->getUnderlyingValue(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b0e5a5c4449b..de57fbecae87 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1887,58 +1887,6 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan) { } } -/// Removes the permutation pattern \p Perm from any elementwise operations -/// in the plan, by constructing a new permutation via \p Build. -/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)). -template -static void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) { - for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly( - vp_depth_first_deep(Plan.getEntry()))) { - for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { - auto *Def = dyn_cast(&R); - if (!Def || !vputils::isElementwise(Def)) - continue; - - // At least one of the ops must be a permutation. - if (!any_of(Def->operands(), match_fn(Perm(m_VPValue())))) - continue; - - // All operands must be permuted or a live in (splat). - if (!all_of( - Def->operands(), - match_fn(m_CombineOr(m_OneUse(Perm(m_VPValue())), m_LiveIn())))) - continue; - - VPValue *X; - // Remove the inner permutations. - for (unsigned I = 0; I < Def->getNumOperands(); I++) - if (match(Def->getOperand(I), Perm(m_VPValue(X)))) - Def->setOperand(I, X); - - VPSingleDefRecipe *Res = Build(Def); - Res->insertAfter(Def); - Def->replaceUsesWithIf( - Res, [&Res](VPUser &U, unsigned _) { return &U != Res; }); - } - } -} - -void VPlanTransforms::simplifyReverses(VPlan &Plan) { - // Pull out reverses from any elementwise op. - // binop(reverse(x), reverse(y)) -> reverse(binop(x,y)) - pullOutPermutations( - Plan, [](const auto &X) { return m_Reverse(X); }, - [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); }); - - // reverse(reverse(x)) -> x - VPValue *X; - for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly( - vp_depth_first_deep(Plan.getEntry()))) - for (VPRecipeBase &R : make_early_inc_range(*VPBB)) - if (match(&R, m_Reverse(m_Reverse(m_VPValue(X))))) - R.getVPSingleValue()->replaceAllUsesWith(X); -} - /// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the /// header mask to be simplified further when tail folding, e.g. in /// optimizeEVLMasks. @@ -2935,7 +2883,6 @@ void VPlanTransforms::optimize(VPlan &Plan) { RUN_VPLAN_PASS(reassociateHeaderMask, Plan); RUN_VPLAN_PASS(simplifyRecipes, Plan); RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false); - RUN_VPLAN_PASS(simplifyReverses, Plan); RUN_VPLAN_PASS(removeDeadRecipes, Plan); RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan); @@ -3273,45 +3220,11 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { } } - // Pull out left splices from any elementwise op. - // binop(splice.left(poison, x, evl), live-in) - // -> splice.left(poison, binop(x,live-in), evl) - pullOutPermutations( - Plan, - [&EVL](const auto &X) { - return m_Intrinsic(m_Poison(), X, - m_Specific(EVL)); - }, - [&Plan, &EVL](auto *X) { - return new VPWidenIntrinsicRecipe( - Intrinsic::vector_splice_left, - {Plan.getPoison(X->getScalarType()), X, EVL}, X->getScalarType(), - {}, {}, X->getDebugLoc()); - }); - - // Fold the following splice patterns: - // splice.right(splice.left(poison, x, evl), poison, evl) -> x + // Fold the following splice patterns into vp.reverse for reverse accesses: // vector.reverse(splice.left(poison, x, evl)) -> vp.reverse(x, true, evl) // splice.right(vector.reverse(x), poison, evl) -> vp.reverse(x, true, evl) for (VPUser *U : collectUsersRecursively(EVL)) { - auto *R = cast(U); - // Remove potentially dead left splices from the transform above. - if (match(U, m_Intrinsic()) && - R->getVPSingleValue()->getNumUsers() == 0) { - OldRecipes.push_back(R); - continue; - } - VPValue *X; - if (match(U, m_Intrinsic( - m_Intrinsic( - m_Poison(), m_VPValue(X), m_Specific(EVL)), - m_Poison(), m_Specific(EVL)))) { - R->getVPSingleValue()->replaceAllUsesWith(X); - OldRecipes.push_back(R); - continue; - } - if (!match(U, m_CombineOr( m_Reverse(m_Intrinsic( @@ -3320,12 +3233,13 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { m_Reverse(m_VPValue(X)), m_Poison(), m_Specific(EVL))))) continue; + auto *Def = cast(U); auto *VPReverse = new VPWidenIntrinsicRecipe( Intrinsic::experimental_vp_reverse, {X, Plan.getTrue(), EVL}, - X->getScalarType(), {}, {}, R->getDebugLoc()); - VPReverse->insertBefore(R); - R->getVPSingleValue()->replaceAllUsesWith(VPReverse); - OldRecipes.push_back(R); + X->getScalarType(), {}, {}, Def->getDebugLoc()); + VPReverse->insertBefore(Def); + Def->replaceAllUsesWith(VPReverse); + OldRecipes.push_back(Def); } for (VPRecipeBase *R : reverse(OldRecipes)) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 85375625d34b..37eb9d797fe6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -400,9 +400,6 @@ struct VPlanTransforms { /// Perform instcombine-like simplifications on recipes in \p Plan. static void simplifyRecipes(VPlan &Plan); - /// Cancel out redundant reverses in \p Plan, e.g. reverse(reverse(x)) -> x. - static void simplifyReverses(VPlan &Plan); - /// Remove BranchOnCond recipes with true or false conditions together with /// removing dead edges to their successors. If \p OnlyLatches is true, only /// process loop latches. Returns true if incoming values from any phi-like diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 7b4ffce87434..9f5290433726 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -382,15 +382,6 @@ static bool preservesUniformity(unsigned Opcode) { } } -bool vputils::isElementwise(const VPValue *V) { - unsigned Opcode = TypeSwitch(V) - .Case( - [](auto *R) { return R->getOpcode(); }) - .Default([](auto *) { return 0; }); - // TODO: Handle more opcodes and recipes. - return Instruction::isUnaryOp(Opcode) || Instruction::isBinaryOp(Opcode); -} - bool vputils::isSingleScalar(const VPValue *VPV) { // Live-in, symbolic and canonical-IV region values are single-scalar. if (auto *RV = dyn_cast(VPV)) diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h index 2980b704ec8d..a10406aea498 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h @@ -61,9 +61,6 @@ bool isSingleScalar(const VPValue *VPV); /// VPDerivedIV or the canonical IV). bool isUniformAcrossVFsAndUFs(const VPValue *V); -/// Return true if \p V is elementwise, i.e. none of the lanes are permuted. -bool isElementwise(const VPValue *V); - /// Returns the header block of the first, top-level loop, or null if none /// exist. VPBasicBlock *getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll index 15a9e8d5aa4c..ffaee22096b6 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/alias-mask.ll @@ -286,12 +286,15 @@ define void @alias_mask_reverse_iterate(ptr noalias %ptrA, ptr %ptrB, ptr %ptrC, ; CHECK-TF-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP10]] ; CHECK-TF-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv16i1( [[ACTIVE_LANE_MASK]]) ; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], [[REVERSE]], poison) +; CHECK-TF-NEXT: [[REVERSE3:%.*]] = call @llvm.vector.reverse.nxv16i8( [[WIDE_MASKED_LOAD]]) ; CHECK-TF-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[PTRB]], i64 [[OFFSET_IDX]] ; CHECK-TF-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP10]] ; CHECK-TF-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], [[REVERSE]], poison) -; CHECK-TF-NEXT: [[REVERSE7:%.*]] = add [[WIDE_MASKED_LOAD5]], [[WIDE_MASKED_LOAD]] +; CHECK-TF-NEXT: [[REVERSE6:%.*]] = call @llvm.vector.reverse.nxv16i8( [[WIDE_MASKED_LOAD5]]) +; CHECK-TF-NEXT: [[TMP14:%.*]] = add [[REVERSE6]], [[REVERSE3]] ; CHECK-TF-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[PTRC]], i64 [[OFFSET_IDX]] ; CHECK-TF-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i64 [[TMP10]] +; CHECK-TF-NEXT: [[REVERSE7:%.*]] = call @llvm.vector.reverse.nxv16i8( [[TMP14]]) ; CHECK-TF-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[REVERSE7]], ptr align 1 [[TMP16]], [[REVERSE]]) ; CHECK-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]] ; CHECK-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[IV_START]]) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll index fb17b9a80d09..21d5a2e74028 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll @@ -350,13 +350,13 @@ define void @reverse(ptr noalias %dst, ptr noalias %src) #0 { ; CHECK-NOTF: vector.body: ; CHECK-NOTF-NOT: %{{.*}} = phi ; CHECK-NOTF: %[[LOAD:.*]] = load , ptr -; CHECK-NOTF: %{{.*}} = call @llvm.vector.reverse.nxv2f64( %{{.*}}) +; CHECK-NOTF: %{{.*}} = call @llvm.vector.reverse.nxv2f64( %[[LOAD]]) ; CHECK-TF-NOREV-LABEL: @reverse( ; CHECK-TF-NOREV: vector.body: ; CHECK-TF-NOREV-NOT: %{{.*}} = phi ; CHECK-TF-NOREV: %[[LOAD:.*]] = load , ptr -; CHECK-TF-NOREV: %{{.*}} = call @llvm.vector.reverse.nxv2f64( %{{.*}}) +; CHECK-TF-NOREV: %{{.*}} = call @llvm.vector.reverse.nxv2f64( %[[LOAD]]) ; CHECK-TF-LABEL: @reverse( ; CHECK-TF: vector.body: @@ -381,13 +381,12 @@ entry: for.body: %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %for.body ] - %indvars.ptr = phi ptr [ %dst, %entry ], [ %indvars.ptr.next, %for.body ] %arrayidx = getelementptr inbounds double, ptr %src, i64 %indvars.iv %0 = load double, ptr %arrayidx, align 8 %add = fadd double %0, 1.000000e+00 - store double %add, ptr %indvars.ptr, align 8 + %arrayidx2 = getelementptr inbounds double, ptr %dst, i64 %indvars.iv + store double %add, ptr %arrayidx2, align 8 %indvars.iv.next = add nsw i64 %indvars.iv, -1 - %indvars.ptr.next = getelementptr inbounds double, ptr %indvars.ptr, i64 1 %cmp.not = icmp eq i64 %indvars.iv, 0 br i1 %cmp.not, label %for.end, label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll index 6d16222f25f9..b1cf3270c3a7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse-mask4.ll @@ -56,9 +56,11 @@ define void @vector_reverse_mask_nxv4i1(ptr %a, ptr %cond, i64 %N) #0 { ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[A]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[TMP12]], i64 [[TMP9]] ; CHECK-NEXT: [[REVERSE2:%.*]] = call @llvm.vector.reverse.nxv4i1( [[TMP11]]) -; CHECK-NEXT: [[REVERSE3:%.*]] = call @llvm.masked.load.nxv4f64.p0(ptr align 8 [[TMP13]], [[REVERSE2]], poison), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f64.p0(ptr align 8 [[TMP13]], [[REVERSE2]], poison), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] +; CHECK-NEXT: [[REVERSE3:%.*]] = call @llvm.vector.reverse.nxv4f64( [[WIDE_MASKED_LOAD]]) ; CHECK-NEXT: [[TMP14:%.*]] = fadd [[REVERSE3]], splat (double 1.000000e+00) -; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0( [[TMP14]], ptr align 8 [[TMP13]], [[REVERSE2]]), !alias.scope [[META3]], !noalias [[META0]] +; CHECK-NEXT: [[REVERSE4:%.*]] = call @llvm.vector.reverse.nxv4f64( [[TMP14]]) +; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0( [[REVERSE4]], ptr align 8 [[TMP13]], [[REVERSE2]]), !alias.scope [[META3]], !noalias [[META0]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll index 4f53d31fd98f..a20a3d77464a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll @@ -37,11 +37,15 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{ ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i64 [[TMP22]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP14]], align 8 ; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP15]], align 8 -; CHECK-NEXT: [[TMP16:%.*]] = fadd [[WIDE_LOAD]], splat (double 1.000000e+00) -; CHECK-NEXT: [[TMP17:%.*]] = fadd [[WIDE_LOAD1]], splat (double 1.000000e+00) +; CHECK-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv8f64( [[WIDE_LOAD]]) +; CHECK-NEXT: [[REVERSE2:%.*]] = call @llvm.vector.reverse.nxv8f64( [[WIDE_LOAD1]]) +; CHECK-NEXT: [[TMP18:%.*]] = fadd [[REVERSE]], splat (double 1.000000e+00) +; CHECK-NEXT: [[TMP19:%.*]] = fadd [[REVERSE2]], splat (double 1.000000e+00) ; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds double, ptr [[TMP21]], i64 [[TMP12]] ; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds double, ptr [[TMP21]], i64 [[TMP22]] +; CHECK-NEXT: [[TMP16:%.*]] = call @llvm.vector.reverse.nxv8f64( [[TMP18]]) +; CHECK-NEXT: [[TMP17:%.*]] = call @llvm.vector.reverse.nxv8f64( [[TMP19]]) ; CHECK-NEXT: store [[TMP16]], ptr [[TMP20]], align 8 ; CHECK-NEXT: store [[TMP17]], ptr [[TMP24]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] @@ -112,11 +116,15 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP25]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP17]], align 8 ; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP18]], align 8 -; CHECK-NEXT: [[TMP19:%.*]] = add [[WIDE_LOAD]], splat (i64 1) -; CHECK-NEXT: [[TMP20:%.*]] = add [[WIDE_LOAD3]], splat (i64 1) +; CHECK-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv8i64( [[WIDE_LOAD]]) +; CHECK-NEXT: [[REVERSE4:%.*]] = call @llvm.vector.reverse.nxv8i64( [[WIDE_LOAD3]]) +; CHECK-NEXT: [[TMP21:%.*]] = add [[REVERSE]], splat (i64 1) +; CHECK-NEXT: [[TMP22:%.*]] = add [[REVERSE4]], splat (i64 1) ; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]] ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 [[TMP15]] ; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i64, ptr [[TMP24]], i64 [[TMP25]] +; CHECK-NEXT: [[TMP19:%.*]] = call @llvm.vector.reverse.nxv8i64( [[TMP21]]) +; CHECK-NEXT: [[TMP20:%.*]] = call @llvm.vector.reverse.nxv8i64( [[TMP22]]) ; CHECK-NEXT: store [[TMP19]], ptr [[TMP23]], align 8 ; CHECK-NEXT: store [[TMP20]], ptr [[TMP27]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll index 90c4ed0fb8de..efedb2b7ca5d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse-mask4.ll @@ -50,8 +50,12 @@ define void @vector_reverse_mask_v4i1(ptr noalias %a, ptr noalias %cond, i64 %N) ; CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <4 x i32> ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP8]], <4 x i1> [[REVERSE3]], <4 x double> poison) ; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP9]], <4 x i1> [[REVERSE5]], <4 x double> poison) -; CHECK-NEXT: [[TMP10:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], splat (double 1.000000e+00) -; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD6]], splat (double 1.000000e+00) +; CHECK-NEXT: [[REVERSE6:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[REVERSE7:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD6]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP16:%.*]] = fadd <4 x double> [[REVERSE6]], splat (double 1.000000e+00) +; CHECK-NEXT: [[TMP17:%.*]] = fadd <4 x double> [[REVERSE7]], splat (double 1.000000e+00) +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> poison, <4 x i32> ; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP10]], ptr align 8 [[TMP8]], <4 x i1> [[REVERSE3]]) ; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP11]], ptr align 8 [[TMP9]], <4 x i1> [[REVERSE5]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll index 01494fa91bc6..8c53e7cf924c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-reverse.ll @@ -18,15 +18,15 @@ define void @vector_reverse_f64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[CMP7:%.*]] = icmp sgt i64 [[N]], 0 ; CHECK-NEXT: br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_COND_CLEANUP:label %.*]] ; CHECK: [[FOR_BODY_PREHEADER]]: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[A2]] ; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP0]], 1 -; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 127 +; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 63 ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] @@ -36,17 +36,14 @@ define void @vector_reverse_f64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[OFFSET_IDX]], -1 ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[TMP2]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 -7 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i64 -15 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x double>, ptr [[TMP5]], align 8 -; CHECK-NEXT: [[REVERSE:%.*]] = load <8 x double>, ptr [[TMP8]], align 8 -; CHECK-NEXT: [[REVERSE3:%.*]] = fadd <8 x double> [[WIDE_LOAD]], splat (double 1.000000e+00) +; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <8 x double> [[WIDE_LOAD]], <8 x double> poison, <8 x i32> ; CHECK-NEXT: [[TMP6:%.*]] = fadd <8 x double> [[REVERSE]], splat (double 1.000000e+00) ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP2]] ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 -7 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 -15 +; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <8 x i32> ; CHECK-NEXT: store <8 x double> [[REVERSE3]], ptr [[TMP9]], align 8 -; CHECK-NEXT: store <8 x double> [[TMP6]], ptr [[TMP12]], align 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: @@ -83,15 +80,15 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i64 [[N]], 0 ; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_BODY_PREHEADER:.*]], [[FOR_COND_CLEANUP:label %.*]] ; CHECK: [[FOR_BODY_PREHEADER]]: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[B1]], [[A2]] ; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP0]], 1 -; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 127 +; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], 63 ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[N_VEC]] ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] @@ -101,17 +98,14 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[OFFSET_IDX]], -1 ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 -7 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i64 -15 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP5]], align 8 -; CHECK-NEXT: [[REVERSE:%.*]] = load <8 x i64>, ptr [[TMP8]], align 8 -; CHECK-NEXT: [[REVERSE3:%.*]] = add <8 x i64> [[WIDE_LOAD]], splat (i64 1) +; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <8 x i64> [[WIDE_LOAD]], <8 x i64> poison, <8 x i32> ; CHECK-NEXT: [[TMP6:%.*]] = add <8 x i64> [[REVERSE]], splat (i64 1) ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]] ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 -7 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 -15 +; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <8 x i64> [[TMP6]], <8 x i64> poison, <8 x i32> ; CHECK-NEXT: store <8 x i64> [[REVERSE3]], ptr [[TMP9]], align 8 -; CHECK-NEXT: store <8 x i64> [[TMP6]], ptr [[TMP12]], align 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll index 561599f5be26..5524d74c03f7 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll @@ -62,8 +62,8 @@ define void @test_stride-1_4i32(ptr readonly %data, ptr noalias nocapture %dst, ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i32 [[TMP2]] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 -3 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> splat (i32 5), [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> splat (i32 5), [[REVERSE]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]] ; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[TMP7]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll index 02a695ff30c0..14d2332b6250 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll @@ -535,12 +535,15 @@ define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]] ; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15 ; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1 +; DEFAULT-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> ; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]] ; DEFAULT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15 ; DEFAULT-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1 -; DEFAULT-NEXT: [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]] +; DEFAULT-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> +; DEFAULT-NEXT: [[TMP6:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]] ; DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]] ; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15 +; DEFAULT-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> poison, <16 x i32> ; DEFAULT-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP8]], align 1 ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16 ; DEFAULT-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -570,12 +573,15 @@ define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-PREFER-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -15 ; CHECK-PREFER-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> ; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[REVERSE]], <16 x i8> poison) +; CHECK-PREFER-NEXT: [[REVERSE1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> poison, <16 x i32> ; CHECK-PREFER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP0]] ; CHECK-PREFER-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 -15 ; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[REVERSE]], <16 x i8> poison) -; CHECK-PREFER-NEXT: [[REVERSE4:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]] +; CHECK-PREFER-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD2]], <16 x i8> poison, <16 x i32> +; CHECK-PREFER-NEXT: [[TMP5:%.*]] = add <16 x i8> [[REVERSE3]], [[REVERSE1]] ; CHECK-PREFER-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP0]] ; CHECK-PREFER-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 -15 +; CHECK-PREFER-NEXT: [[REVERSE4:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> poison, <16 x i32> ; CHECK-PREFER-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[REVERSE4]], ptr align 1 [[TMP7]], <16 x i1> [[REVERSE]]) ; CHECK-PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16 ; CHECK-PREFER-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -606,12 +612,15 @@ define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ENABLE-TP-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]] ; CHECK-ENABLE-TP-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15 ; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1 +; CHECK-ENABLE-TP-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> ; CHECK-ENABLE-TP-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]] ; CHECK-ENABLE-TP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15 ; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1 -; CHECK-ENABLE-TP-NEXT: [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]] +; CHECK-ENABLE-TP-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> +; CHECK-ENABLE-TP-NEXT: [[TMP6:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]] ; CHECK-ENABLE-TP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]] ; CHECK-ENABLE-TP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15 +; CHECK-ENABLE-TP-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> poison, <16 x i32> ; CHECK-ENABLE-TP-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP8]], align 1 ; CHECK-ENABLE-TP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16 ; CHECK-ENABLE-TP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -667,12 +676,15 @@ define void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias noc ; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP2]] ; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 -15 ; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1 +; DEFAULT-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> ; DEFAULT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP2]] ; DEFAULT-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -15 ; DEFAULT-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1 -; DEFAULT-NEXT: [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]] +; DEFAULT-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> +; DEFAULT-NEXT: [[TMP7:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]] ; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP2]] ; DEFAULT-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 -15 +; DEFAULT-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> ; DEFAULT-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP9]], align 1 ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; DEFAULT-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -704,12 +716,15 @@ define void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias noc ; CHECK-PREFER-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -15 ; CHECK-PREFER-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> ; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP3]], <16 x i1> [[REVERSE]], <16 x i8> poison) +; CHECK-PREFER-NEXT: [[REVERSE1:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> poison, <16 x i32> ; CHECK-PREFER-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]] ; CHECK-PREFER-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 -15 ; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP5]], <16 x i1> [[REVERSE]], <16 x i8> poison) -; CHECK-PREFER-NEXT: [[REVERSE4:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]] +; CHECK-PREFER-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[WIDE_MASKED_LOAD2]], <16 x i8> poison, <16 x i32> +; CHECK-PREFER-NEXT: [[TMP6:%.*]] = add <16 x i8> [[REVERSE3]], [[REVERSE1]] ; CHECK-PREFER-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]] ; CHECK-PREFER-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 -15 +; CHECK-PREFER-NEXT: [[REVERSE4:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> poison, <16 x i32> ; CHECK-PREFER-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[REVERSE4]], ptr align 1 [[TMP8]], <16 x i1> [[REVERSE]]) ; CHECK-PREFER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-PREFER-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -742,12 +757,15 @@ define void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias noc ; CHECK-ENABLE-TP-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP2]] ; CHECK-ENABLE-TP-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 -15 ; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1 +; CHECK-ENABLE-TP-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> poison, <16 x i32> ; CHECK-ENABLE-TP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP2]] ; CHECK-ENABLE-TP-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -15 ; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1 -; CHECK-ENABLE-TP-NEXT: [[REVERSE3:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]] +; CHECK-ENABLE-TP-NEXT: [[REVERSE2:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD1]], <16 x i8> poison, <16 x i32> +; CHECK-ENABLE-TP-NEXT: [[TMP7:%.*]] = add <16 x i8> [[REVERSE2]], [[REVERSE]] ; CHECK-ENABLE-TP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP2]] ; CHECK-ENABLE-TP-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 -15 +; CHECK-ENABLE-TP-NEXT: [[REVERSE3:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> poison, <16 x i32> ; CHECK-ENABLE-TP-NEXT: store <16 x i8> [[REVERSE3]], ptr [[TMP9]], align 1 ; CHECK-ENABLE-TP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-ENABLE-TP-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -800,139 +818,36 @@ for.body: ; false for the inner-loop. ; define void @sgt_nested_loop(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) #0 { -; DEFAULT-LABEL: define void @sgt_nested_loop( -; DEFAULT-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { -; DEFAULT-NEXT: [[ENTRY:.*:]] -; DEFAULT-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[N]], 0 -; DEFAULT-NEXT: br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]] -; DEFAULT: [[FOR_BODY_PREHEADER]]: -; DEFAULT-NEXT: br label %[[FOR_BODY:.*]] -; DEFAULT: [[FOR_COND_LOOPEXIT:.*]]: -; DEFAULT-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]] -; DEFAULT-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]] -; DEFAULT: [[FOR_COND_CLEANUP_LOOPEXIT]]: -; DEFAULT-NEXT: br label %[[FOR_COND_CLEANUP]] -; DEFAULT: [[FOR_COND_CLEANUP]]: -; DEFAULT-NEXT: ret void -; DEFAULT: [[FOR_BODY]]: -; DEFAULT-NEXT: [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ] -; DEFAULT-NEXT: [[ADD]] = add nuw nsw i32 [[I_022]], 1 -; DEFAULT-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[ADD]], 16 -; DEFAULT-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; DEFAULT: [[VECTOR_PH]]: -; DEFAULT-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[ADD]], 16 -; DEFAULT-NEXT: [[N_VEC:%.*]] = sub i32 [[ADD]], [[N_MOD_VF]] -; DEFAULT-NEXT: [[TMP0:%.*]] = sub i32 [[ADD]], [[N_VEC]] -; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]] -; DEFAULT: [[VECTOR_BODY]]: -; DEFAULT-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; DEFAULT-NEXT: [[TMP1:%.*]] = sub i32 [[ADD]], [[INDEX]] -; DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]] -; DEFAULT-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15 -; DEFAULT-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1 -; DEFAULT-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]] -; DEFAULT-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15 -; DEFAULT-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1 -; DEFAULT-NEXT: [[TMP6:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]] -; DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]] -; DEFAULT-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15 -; DEFAULT-NEXT: store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1 -; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16 -; DEFAULT-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; DEFAULT-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] -; DEFAULT: [[MIDDLE_BLOCK]]: -; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[ADD]], [[N_VEC]] -; DEFAULT-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_LOOPEXIT]], label %[[SCALAR_PH]] -; DEFAULT: [[SCALAR_PH]]: -; -; CHECK-PREFER-LABEL: define void @sgt_nested_loop( -; CHECK-PREFER-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { -; CHECK-PREFER-NEXT: [[ENTRY:.*:]] -; CHECK-PREFER-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[N]], 0 -; CHECK-PREFER-NEXT: br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]] -; CHECK-PREFER: [[FOR_BODY_PREHEADER]]: -; CHECK-PREFER-NEXT: br label %[[FOR_BODY:.*]] -; CHECK-PREFER: [[FOR_COND_LOOPEXIT:.*]]: -; CHECK-PREFER-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]] -; CHECK-PREFER-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]] -; CHECK-PREFER: [[FOR_COND_CLEANUP_LOOPEXIT]]: -; CHECK-PREFER-NEXT: br label %[[FOR_COND_CLEANUP]] -; CHECK-PREFER: [[FOR_COND_CLEANUP]]: -; CHECK-PREFER-NEXT: ret void -; CHECK-PREFER: [[FOR_BODY]]: -; CHECK-PREFER-NEXT: [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ] -; CHECK-PREFER-NEXT: [[ADD]] = add nuw nsw i32 [[I_022]], 1 -; CHECK-PREFER-NEXT: br label %[[VECTOR_PH:.*]] -; CHECK-PREFER: [[VECTOR_PH]]: -; CHECK-PREFER-NEXT: [[N_RND_UP:%.*]] = add i32 [[ADD]], 15 -; CHECK-PREFER-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 16 -; CHECK-PREFER-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] -; CHECK-PREFER-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK-PREFER: [[VECTOR_BODY]]: -; CHECK-PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-PREFER-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[ADD]]) -; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i32 [[ADD]], [[INDEX]] -; CHECK-PREFER-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP0]] -; CHECK-PREFER-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -15 -; CHECK-PREFER-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> -; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP2]], <16 x i1> [[REVERSE]], <16 x i8> poison) -; CHECK-PREFER-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP0]] -; CHECK-PREFER-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 -15 -; CHECK-PREFER-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[TMP4]], <16 x i1> [[REVERSE]], <16 x i8> poison) -; CHECK-PREFER-NEXT: [[TMP5:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD1]], [[WIDE_MASKED_LOAD]] -; CHECK-PREFER-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP0]] -; CHECK-PREFER-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 -15 -; CHECK-PREFER-NEXT: call void @llvm.masked.store.v16i8.p0(<16 x i8> [[TMP5]], ptr align 1 [[TMP7]], <16 x i1> [[REVERSE]]) -; CHECK-PREFER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16 -; CHECK-PREFER-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-PREFER-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] -; CHECK-PREFER: [[MIDDLE_BLOCK]]: -; CHECK-PREFER-NEXT: br label %[[FOR_COND_LOOPEXIT]] -; -; CHECK-ENABLE-TP-LABEL: define void @sgt_nested_loop( -; CHECK-ENABLE-TP-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { -; CHECK-ENABLE-TP-NEXT: [[ENTRY:.*:]] -; CHECK-ENABLE-TP-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[N]], 0 -; CHECK-ENABLE-TP-NEXT: br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]] -; CHECK-ENABLE-TP: [[FOR_BODY_PREHEADER]]: -; CHECK-ENABLE-TP-NEXT: br label %[[FOR_BODY:.*]] -; CHECK-ENABLE-TP: [[FOR_COND_LOOPEXIT:.*]]: -; CHECK-ENABLE-TP-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]] -; CHECK-ENABLE-TP-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]] -; CHECK-ENABLE-TP: [[FOR_COND_CLEANUP_LOOPEXIT]]: -; CHECK-ENABLE-TP-NEXT: br label %[[FOR_COND_CLEANUP]] -; CHECK-ENABLE-TP: [[FOR_COND_CLEANUP]]: -; CHECK-ENABLE-TP-NEXT: ret void -; CHECK-ENABLE-TP: [[FOR_BODY]]: -; CHECK-ENABLE-TP-NEXT: [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ] -; CHECK-ENABLE-TP-NEXT: [[ADD]] = add nuw nsw i32 [[I_022]], 1 -; CHECK-ENABLE-TP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[ADD]], 16 -; CHECK-ENABLE-TP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK-ENABLE-TP: [[VECTOR_PH]]: -; CHECK-ENABLE-TP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[ADD]], 16 -; CHECK-ENABLE-TP-NEXT: [[N_VEC:%.*]] = sub i32 [[ADD]], [[N_MOD_VF]] -; CHECK-ENABLE-TP-NEXT: [[TMP0:%.*]] = sub i32 [[ADD]], [[N_VEC]] -; CHECK-ENABLE-TP-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK-ENABLE-TP: [[VECTOR_BODY]]: -; CHECK-ENABLE-TP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-ENABLE-TP-NEXT: [[TMP1:%.*]] = sub i32 [[ADD]], [[INDEX]] -; CHECK-ENABLE-TP-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]] -; CHECK-ENABLE-TP-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -15 -; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1 -; CHECK-ENABLE-TP-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[TMP1]] -; CHECK-ENABLE-TP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -15 -; CHECK-ENABLE-TP-NEXT: [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1 -; CHECK-ENABLE-TP-NEXT: [[TMP6:%.*]] = add <16 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD]] -; CHECK-ENABLE-TP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[TMP1]] -; CHECK-ENABLE-TP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -15 -; CHECK-ENABLE-TP-NEXT: store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1 -; CHECK-ENABLE-TP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16 -; CHECK-ENABLE-TP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-ENABLE-TP-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] -; CHECK-ENABLE-TP: [[MIDDLE_BLOCK]]: -; CHECK-ENABLE-TP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[ADD]], [[N_VEC]] -; CHECK-ENABLE-TP-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_LOOPEXIT]], label %[[SCALAR_PH]] -; CHECK-ENABLE-TP: [[SCALAR_PH]]: +; COMMON-LABEL: define void @sgt_nested_loop( +; COMMON-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { +; COMMON-NEXT: [[ENTRY:.*:]] +; COMMON-NEXT: [[CMP21:%.*]] = icmp sgt i32 [[N]], 0 +; COMMON-NEXT: br i1 [[CMP21]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]] +; COMMON: [[FOR_BODY_PREHEADER]]: +; COMMON-NEXT: br label %[[FOR_BODY:.*]] +; COMMON: [[FOR_COND_LOOPEXIT:.*]]: +; COMMON-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[ADD:%.*]], [[N]] +; COMMON-NEXT: br i1 [[EXITCOND]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[FOR_BODY]] +; COMMON: [[FOR_COND_CLEANUP_LOOPEXIT]]: +; COMMON-NEXT: br label %[[FOR_COND_CLEANUP]] +; COMMON: [[FOR_COND_CLEANUP]]: +; COMMON-NEXT: ret void +; COMMON: [[FOR_BODY]]: +; COMMON-NEXT: [[I_022:%.*]] = phi i32 [ [[ADD]], %[[FOR_COND_LOOPEXIT]] ], [ 0, %[[FOR_BODY_PREHEADER]] ] +; COMMON-NEXT: [[ADD]] = add nuw nsw i32 [[I_022]], 1 +; COMMON-NEXT: br label %[[FOR_BODY4:.*]] +; COMMON: [[FOR_BODY4]]: +; COMMON-NEXT: [[J_020:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[DEC:%.*]], %[[FOR_BODY4]] ] +; COMMON-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[J_020]] +; COMMON-NEXT: [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; COMMON-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[J_020]] +; COMMON-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX5]], align 1 +; COMMON-NEXT: [[ADD7:%.*]] = add i8 [[TMP1]], [[TMP0]] +; COMMON-NEXT: [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[J_020]] +; COMMON-NEXT: store i8 [[ADD7]], ptr [[ARRAYIDX9]], align 1 +; COMMON-NEXT: [[DEC]] = add nsw i32 [[J_020]], -1 +; COMMON-NEXT: [[CMP2:%.*]] = icmp sgt i32 [[J_020]], 1 +; COMMON-NEXT: br i1 [[CMP2]], label %[[FOR_BODY4]], label %[[FOR_COND_LOOPEXIT]] ; entry: %cmp21 = icmp sgt i32 %N, 0 diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll index f62c18d24286..cb384eaeb64f 100644 --- a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll @@ -278,22 +278,22 @@ define void @f2(ptr noalias %A, ptr noalias %B, i32 %n) { ; VF-TWO-CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x float>, ptr [[TMP67]], align 4 ; VF-TWO-CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP69]], align 4 ; VF-TWO-CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP71]], align 4 -; VF-TWO-CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[WIDE_LOAD2]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP23:%.*]] = fadd fast <4 x float> [[WIDE_LOAD4]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP29:%.*]] = fadd fast <4 x float> [[WIDE_LOAD6]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP25:%.*]] = fadd fast <4 x float> [[WIDE_LOAD8]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP26:%.*]] = fadd fast <4 x float> [[WIDE_LOAD10]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP27:%.*]] = fadd fast <4 x float> [[WIDE_LOAD12]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP28:%.*]] = fadd fast <4 x float> [[WIDE_LOAD14]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP72:%.*]] = shufflevector <4 x float> [[TMP21]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP73:%.*]] = shufflevector <4 x float> [[TMP22]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP74:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP75:%.*]] = shufflevector <4 x float> [[TMP29]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP76:%.*]] = shufflevector <4 x float> [[TMP25]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP77:%.*]] = shufflevector <4 x float> [[TMP26]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP78:%.*]] = shufflevector <4 x float> [[TMP27]], <4 x float> poison, <4 x i32> -; VF-TWO-CHECK-NEXT: [[TMP79:%.*]] = shufflevector <4 x float> [[TMP28]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x float> [[WIDE_LOAD]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x float> [[WIDE_LOAD2]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x float> [[WIDE_LOAD4]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE7:%.*]] = shufflevector <4 x float> [[WIDE_LOAD6]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x float> [[WIDE_LOAD8]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE11:%.*]] = shufflevector <4 x float> [[WIDE_LOAD10]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE13:%.*]] = shufflevector <4 x float> [[WIDE_LOAD12]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE15:%.*]] = shufflevector <4 x float> [[WIDE_LOAD14]], <4 x float> poison, <4 x i32> +; VF-TWO-CHECK-NEXT: [[TMP72:%.*]] = fadd fast <4 x float> [[REVERSE]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP73:%.*]] = fadd fast <4 x float> [[REVERSE3]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP74:%.*]] = fadd fast <4 x float> [[REVERSE5]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP75:%.*]] = fadd fast <4 x float> [[REVERSE7]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP76:%.*]] = fadd fast <4 x float> [[REVERSE9]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP77:%.*]] = fadd fast <4 x float> [[REVERSE11]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP78:%.*]] = fadd fast <4 x float> [[REVERSE13]], splat (float 1.000000e+00) +; VF-TWO-CHECK-NEXT: [[TMP79:%.*]] = fadd fast <4 x float> [[REVERSE15]], splat (float 1.000000e+00) ; VF-TWO-CHECK-NEXT: [[TMP80:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] ; VF-TWO-CHECK-NEXT: [[TMP89:%.*]] = getelementptr inbounds float, ptr [[TMP80]], i64 4 ; VF-TWO-CHECK-NEXT: [[TMP90:%.*]] = getelementptr inbounds float, ptr [[TMP80]], i64 8 @@ -334,8 +334,8 @@ define void @f2(ptr noalias %A, ptr noalias %B, i32 %n) { ; VF-TWO-CHECK-NEXT: [[TMP102:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP101]] ; VF-TWO-CHECK-NEXT: [[TMP104:%.*]] = getelementptr inbounds float, ptr [[TMP102]], i64 -1 ; VF-TWO-CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <2 x float>, ptr [[TMP104]], align 4 -; VF-TWO-CHECK-NEXT: [[TMP45:%.*]] = fadd fast <2 x float> [[WIDE_LOAD23]], splat (float 1.000000e+00) -; VF-TWO-CHECK-NEXT: [[TMP105:%.*]] = shufflevector <2 x float> [[TMP45]], <2 x float> poison, <2 x i32> +; VF-TWO-CHECK-NEXT: [[REVERSE24:%.*]] = shufflevector <2 x float> [[WIDE_LOAD23]], <2 x float> poison, <2 x i32> +; VF-TWO-CHECK-NEXT: [[TMP105:%.*]] = fadd fast <2 x float> [[REVERSE24]], splat (float 1.000000e+00) ; VF-TWO-CHECK-NEXT: [[TMP106:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX21]] ; VF-TWO-CHECK-NEXT: store <2 x float> [[TMP105]], ptr [[TMP106]], align 4 ; VF-TWO-CHECK-NEXT: [[INDEX_NEXT25]] = add nuw i64 [[INDEX21]], 2 @@ -392,22 +392,22 @@ define void @f2(ptr noalias %A, ptr noalias %B, i32 %n) { ; VF-FOUR-CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <4 x float>, ptr [[TMP67]], align 4 ; VF-FOUR-CHECK-NEXT: [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP69]], align 4 ; VF-FOUR-CHECK-NEXT: [[WIDE_LOAD14:%.*]] = load <4 x float>, ptr [[TMP71]], align 4 -; VF-FOUR-CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[WIDE_LOAD2]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP23:%.*]] = fadd fast <4 x float> [[WIDE_LOAD4]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP29:%.*]] = fadd fast <4 x float> [[WIDE_LOAD6]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP25:%.*]] = fadd fast <4 x float> [[WIDE_LOAD8]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP26:%.*]] = fadd fast <4 x float> [[WIDE_LOAD10]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP27:%.*]] = fadd fast <4 x float> [[WIDE_LOAD12]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP28:%.*]] = fadd fast <4 x float> [[WIDE_LOAD14]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP72:%.*]] = shufflevector <4 x float> [[TMP21]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP73:%.*]] = shufflevector <4 x float> [[TMP22]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP74:%.*]] = shufflevector <4 x float> [[TMP23]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP75:%.*]] = shufflevector <4 x float> [[TMP29]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP76:%.*]] = shufflevector <4 x float> [[TMP25]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP77:%.*]] = shufflevector <4 x float> [[TMP26]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP78:%.*]] = shufflevector <4 x float> [[TMP27]], <4 x float> poison, <4 x i32> -; VF-FOUR-CHECK-NEXT: [[TMP79:%.*]] = shufflevector <4 x float> [[TMP28]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x float> [[WIDE_LOAD]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x float> [[WIDE_LOAD2]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x float> [[WIDE_LOAD4]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE7:%.*]] = shufflevector <4 x float> [[WIDE_LOAD6]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x float> [[WIDE_LOAD8]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE11:%.*]] = shufflevector <4 x float> [[WIDE_LOAD10]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE13:%.*]] = shufflevector <4 x float> [[WIDE_LOAD12]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE15:%.*]] = shufflevector <4 x float> [[WIDE_LOAD14]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[TMP72:%.*]] = fadd fast <4 x float> [[REVERSE]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP73:%.*]] = fadd fast <4 x float> [[REVERSE3]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP74:%.*]] = fadd fast <4 x float> [[REVERSE5]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP75:%.*]] = fadd fast <4 x float> [[REVERSE7]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP76:%.*]] = fadd fast <4 x float> [[REVERSE9]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP77:%.*]] = fadd fast <4 x float> [[REVERSE11]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP78:%.*]] = fadd fast <4 x float> [[REVERSE13]], splat (float 1.000000e+00) +; VF-FOUR-CHECK-NEXT: [[TMP79:%.*]] = fadd fast <4 x float> [[REVERSE15]], splat (float 1.000000e+00) ; VF-FOUR-CHECK-NEXT: [[TMP80:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] ; VF-FOUR-CHECK-NEXT: [[TMP89:%.*]] = getelementptr inbounds float, ptr [[TMP80]], i64 4 ; VF-FOUR-CHECK-NEXT: [[TMP90:%.*]] = getelementptr inbounds float, ptr [[TMP80]], i64 8 @@ -448,8 +448,8 @@ define void @f2(ptr noalias %A, ptr noalias %B, i32 %n) { ; VF-FOUR-CHECK-NEXT: [[TMP102:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP101]] ; VF-FOUR-CHECK-NEXT: [[TMP104:%.*]] = getelementptr inbounds float, ptr [[TMP102]], i64 -3 ; VF-FOUR-CHECK-NEXT: [[WIDE_LOAD23:%.*]] = load <4 x float>, ptr [[TMP104]], align 4 -; VF-FOUR-CHECK-NEXT: [[TMP45:%.*]] = fadd fast <4 x float> [[WIDE_LOAD23]], splat (float 1.000000e+00) -; VF-FOUR-CHECK-NEXT: [[TMP105:%.*]] = shufflevector <4 x float> [[TMP45]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[REVERSE24:%.*]] = shufflevector <4 x float> [[WIDE_LOAD23]], <4 x float> poison, <4 x i32> +; VF-FOUR-CHECK-NEXT: [[TMP105:%.*]] = fadd fast <4 x float> [[REVERSE24]], splat (float 1.000000e+00) ; VF-FOUR-CHECK-NEXT: [[TMP106:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX21]] ; VF-FOUR-CHECK-NEXT: store <4 x float> [[TMP105]], ptr [[TMP106]], align 4 ; VF-FOUR-CHECK-NEXT: [[INDEX_NEXT25]] = add nuw i64 [[INDEX21]], 4 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index 1c9e48fdf4aa..d55fc7968f4b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -34,9 +34,11 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) { ; RV64-NEXT: [[TMP5:%.*]] = sub i64 0, [[TMP4]] ; RV64-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP8]], i64 [[TMP5]] ; RV64-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], splat (i1 true), i32 [[TMP19]]) -; RV64-NEXT: [[VP_REVERSE1:%.*]] = add [[VP_OP_LOAD]], splat (i32 1) +; RV64-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP19]]) +; RV64-NEXT: [[TMP14:%.*]] = add [[REVERSE]], splat (i32 1) ; RV64-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]] ; RV64-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[TMP15]], i64 [[TMP5]] +; RV64-NEXT: [[VP_REVERSE1:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[TMP14]], splat (i1 true), i32 [[TMP19]]) ; RV64-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_REVERSE1]], ptr align 4 [[TMP21]], splat (i1 true), i32 [[TMP19]]) ; RV64-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP22]], [[INDEX]] ; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]] @@ -65,9 +67,11 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) { ; RV32-NEXT: [[TMP4:%.*]] = sub i32 0, [[TMP3]] ; RV32-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP8]], i32 [[TMP4]] ; RV32-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], splat (i1 true), i32 [[TMP9]]) -; RV32-NEXT: [[VP_REVERSE1:%.*]] = add [[VP_OP_LOAD]], splat (i32 1) +; RV32-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP9]]) +; RV32-NEXT: [[TMP15:%.*]] = add [[REVERSE]], splat (i32 1) ; RV32-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]] ; RV32-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[TMP16]], i32 [[TMP4]] +; RV32-NEXT: [[VP_REVERSE1:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[TMP15]], splat (i1 true), i32 [[TMP9]]) ; RV32-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_REVERSE1]], ptr align 4 [[TMP22]], splat (i1 true), i32 [[TMP9]]) ; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP23]], [[INDEX]] ; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP23]] @@ -104,11 +108,15 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) { ; RV64-UF2-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 [[TMP11]] ; RV64-UF2-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP18]], align 4 ; RV64-UF2-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP19]], align 4 -; RV64-UF2-NEXT: [[REVERSE3:%.*]] = add [[WIDE_LOAD]], splat (i32 1) -; RV64-UF2-NEXT: [[REVERSE4:%.*]] = add [[WIDE_LOAD1]], splat (i32 1) +; RV64-UF2-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv4i32( [[WIDE_LOAD]]) +; RV64-UF2-NEXT: [[REVERSE2:%.*]] = call @llvm.vector.reverse.nxv4i32( [[WIDE_LOAD1]]) +; RV64-UF2-NEXT: [[TMP20:%.*]] = add [[REVERSE]], splat (i32 1) +; RV64-UF2-NEXT: [[TMP21:%.*]] = add [[REVERSE2]], splat (i32 1) ; RV64-UF2-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]] ; RV64-UF2-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i64 [[TMP15]] ; RV64-UF2-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i64 [[TMP11]] +; RV64-UF2-NEXT: [[REVERSE3:%.*]] = call @llvm.vector.reverse.nxv4i32( [[TMP20]]) +; RV64-UF2-NEXT: [[REVERSE4:%.*]] = call @llvm.vector.reverse.nxv4i32( [[TMP21]]) ; RV64-UF2-NEXT: store [[REVERSE3]], ptr [[TMP27]], align 4 ; RV64-UF2-NEXT: store [[REVERSE4]], ptr [[TMP32]], align 4 ; RV64-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] @@ -185,9 +193,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64-NEXT: [[TMP18:%.*]] = sub i64 0, [[TMP17]] ; RV64-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[TMP23]], i64 [[TMP18]] ; RV64-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP27]], splat (i1 true), i32 [[TMP20]]) -; RV64-NEXT: [[VP_REVERSE3:%.*]] = add [[VP_OP_LOAD]], splat (i32 1) +; RV64-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP20]]) +; RV64-NEXT: [[TMP29:%.*]] = add [[REVERSE]], splat (i32 1) ; RV64-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP22]] ; RV64-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[TMP30]], i64 [[TMP18]] +; RV64-NEXT: [[VP_REVERSE3:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[TMP29]], splat (i1 true), i32 [[TMP20]]) ; RV64-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_REVERSE3]], ptr align 4 [[TMP35]], splat (i1 true), i32 [[TMP20]]) ; RV64-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP36]], [[INDEX]] ; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP36]] @@ -238,9 +248,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV32-NEXT: [[TMP10:%.*]] = sub i32 0, [[TMP9]] ; RV32-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[TMP15]], i32 [[TMP10]] ; RV32-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP28]], splat (i1 true), i32 [[TMP16]]) -; RV32-NEXT: [[VP_REVERSE3:%.*]] = add [[VP_OP_LOAD]], splat (i32 1) +; RV32-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP16]]) +; RV32-NEXT: [[TMP22:%.*]] = add [[REVERSE]], splat (i32 1) ; RV32-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]] ; RV32-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP23]], i32 [[TMP10]] +; RV32-NEXT: [[VP_REVERSE3:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[TMP22]], splat (i1 true), i32 [[TMP16]]) ; RV32-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_REVERSE3]], ptr align 4 [[TMP25]], splat (i1 true), i32 [[TMP16]]) ; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP29]], [[INDEX]] ; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP29]] @@ -310,11 +322,15 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64-UF2-NEXT: [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP24]], i64 [[TMP25]] ; RV64-UF2-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP33]], align 4 ; RV64-UF2-NEXT: [[WIDE_LOAD4:%.*]] = load , ptr [[TMP34]], align 4 -; RV64-UF2-NEXT: [[REVERSE6:%.*]] = add [[WIDE_LOAD]], splat (i32 1) -; RV64-UF2-NEXT: [[REVERSE7:%.*]] = add [[WIDE_LOAD4]], splat (i32 1) +; RV64-UF2-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv4i32( [[WIDE_LOAD]]) +; RV64-UF2-NEXT: [[REVERSE5:%.*]] = call @llvm.vector.reverse.nxv4i32( [[WIDE_LOAD4]]) +; RV64-UF2-NEXT: [[TMP35:%.*]] = add [[REVERSE]], splat (i32 1) +; RV64-UF2-NEXT: [[TMP36:%.*]] = add [[REVERSE5]], splat (i32 1) ; RV64-UF2-NEXT: [[TMP37:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP23]] ; RV64-UF2-NEXT: [[TMP42:%.*]] = getelementptr inbounds i32, ptr [[TMP37]], i64 [[TMP30]] ; RV64-UF2-NEXT: [[TMP47:%.*]] = getelementptr inbounds i32, ptr [[TMP37]], i64 [[TMP25]] +; RV64-UF2-NEXT: [[REVERSE6:%.*]] = call @llvm.vector.reverse.nxv4i32( [[TMP35]]) +; RV64-UF2-NEXT: [[REVERSE7:%.*]] = call @llvm.vector.reverse.nxv4i32( [[TMP36]]) ; RV64-UF2-NEXT: store [[REVERSE6]], ptr [[TMP42]], align 4 ; RV64-UF2-NEXT: store [[REVERSE7]], ptr [[TMP47]], align 4 ; RV64-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] @@ -325,7 +341,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64-UF2-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]] ; RV64-UF2: [[SCALAR_PH]]: ; RV64-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ] -; RV64-UF2-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ] +; RV64-UF2-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ] ; RV64-UF2-NEXT: br label %[[FOR_BODY:.*]] ; RV64-UF2: [[FOR_COND_CLEANUP_LOOPEXIT]]: ; RV64-UF2-NEXT: br label %[[FOR_COND_CLEANUP]] @@ -404,9 +420,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64-NEXT: [[TMP18:%.*]] = sub i64 0, [[TMP17]] ; RV64-NEXT: [[TMP27:%.*]] = getelementptr float, ptr [[TMP23]], i64 [[TMP18]] ; RV64-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP27]], splat (i1 true), i32 [[TMP20]]) -; RV64-NEXT: [[VP_REVERSE3:%.*]] = fadd [[VP_OP_LOAD]], splat (float 1.000000e+00) +; RV64-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP20]]) +; RV64-NEXT: [[TMP29:%.*]] = fadd [[REVERSE]], splat (float 1.000000e+00) ; RV64-NEXT: [[TMP30:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP22]] ; RV64-NEXT: [[TMP35:%.*]] = getelementptr float, ptr [[TMP30]], i64 [[TMP18]] +; RV64-NEXT: [[VP_REVERSE3:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[TMP29]], splat (i1 true), i32 [[TMP20]]) ; RV64-NEXT: call void @llvm.vp.store.nxv4f32.p0( [[VP_REVERSE3]], ptr align 4 [[TMP35]], splat (i1 true), i32 [[TMP20]]) ; RV64-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP36]], [[INDEX]] ; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP36]] @@ -457,9 +475,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV32-NEXT: [[TMP10:%.*]] = sub i32 0, [[TMP9]] ; RV32-NEXT: [[TMP28:%.*]] = getelementptr float, ptr [[TMP15]], i32 [[TMP10]] ; RV32-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP28]], splat (i1 true), i32 [[TMP16]]) -; RV32-NEXT: [[VP_REVERSE3:%.*]] = fadd [[VP_OP_LOAD]], splat (float 1.000000e+00) +; RV32-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP16]]) +; RV32-NEXT: [[TMP22:%.*]] = fadd [[REVERSE]], splat (float 1.000000e+00) ; RV32-NEXT: [[TMP23:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]] ; RV32-NEXT: [[TMP25:%.*]] = getelementptr float, ptr [[TMP23]], i32 [[TMP10]] +; RV32-NEXT: [[VP_REVERSE3:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[TMP22]], splat (i1 true), i32 [[TMP16]]) ; RV32-NEXT: call void @llvm.vp.store.nxv4f32.p0( [[VP_REVERSE3]], ptr align 4 [[TMP25]], splat (i1 true), i32 [[TMP16]]) ; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP29]], [[INDEX]] ; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP29]] @@ -529,11 +549,15 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64-UF2-NEXT: [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[TMP25]] ; RV64-UF2-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP33]], align 4 ; RV64-UF2-NEXT: [[WIDE_LOAD4:%.*]] = load , ptr [[TMP34]], align 4 -; RV64-UF2-NEXT: [[REVERSE6:%.*]] = fadd [[WIDE_LOAD]], splat (float 1.000000e+00) -; RV64-UF2-NEXT: [[REVERSE7:%.*]] = fadd [[WIDE_LOAD4]], splat (float 1.000000e+00) +; RV64-UF2-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv4f32( [[WIDE_LOAD]]) +; RV64-UF2-NEXT: [[REVERSE5:%.*]] = call @llvm.vector.reverse.nxv4f32( [[WIDE_LOAD4]]) +; RV64-UF2-NEXT: [[TMP35:%.*]] = fadd [[REVERSE]], splat (float 1.000000e+00) +; RV64-UF2-NEXT: [[TMP36:%.*]] = fadd [[REVERSE5]], splat (float 1.000000e+00) ; RV64-UF2-NEXT: [[TMP37:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]] ; RV64-UF2-NEXT: [[TMP42:%.*]] = getelementptr inbounds float, ptr [[TMP37]], i64 [[TMP30]] ; RV64-UF2-NEXT: [[TMP47:%.*]] = getelementptr inbounds float, ptr [[TMP37]], i64 [[TMP25]] +; RV64-UF2-NEXT: [[REVERSE6:%.*]] = call @llvm.vector.reverse.nxv4f32( [[TMP35]]) +; RV64-UF2-NEXT: [[REVERSE7:%.*]] = call @llvm.vector.reverse.nxv4f32( [[TMP36]]) ; RV64-UF2-NEXT: store [[REVERSE6]], ptr [[TMP42]], align 4 ; RV64-UF2-NEXT: store [[REVERSE7]], ptr [[TMP47]], align 4 ; RV64-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] @@ -544,7 +568,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; RV64-UF2-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]] ; RV64-UF2: [[SCALAR_PH]]: ; RV64-UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ] -; RV64-UF2-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ] +; RV64-UF2-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ] ; RV64-UF2-NEXT: br label %[[FOR_BODY:.*]] ; RV64-UF2: [[FOR_COND_CLEANUP_LOOPEXIT]]: ; RV64-UF2-NEXT: br label %[[FOR_COND_CLEANUP]] @@ -597,9 +621,11 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) { ; RV64-NEXT: [[TMP5:%.*]] = sub i64 0, [[TMP4]] ; RV64-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[TMP5]] ; RV64-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP13]], splat (i1 true), i32 [[TMP19]]) -; RV64-NEXT: [[VP_REVERSE1:%.*]] = fadd [[VP_OP_LOAD]], splat (float 1.000000e+00) +; RV64-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP19]]) +; RV64-NEXT: [[TMP14:%.*]] = fadd [[REVERSE]], splat (float 1.000000e+00) ; RV64-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]] ; RV64-NEXT: [[TMP21:%.*]] = getelementptr float, ptr [[TMP15]], i64 [[TMP5]] +; RV64-NEXT: [[VP_REVERSE1:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[TMP14]], splat (i1 true), i32 [[TMP19]]) ; RV64-NEXT: call void @llvm.vp.store.nxv4f32.p0( [[VP_REVERSE1]], ptr align 4 [[TMP21]], splat (i1 true), i32 [[TMP19]]) ; RV64-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP22]], [[INDEX]] ; RV64-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]] @@ -628,9 +654,11 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) { ; RV32-NEXT: [[TMP4:%.*]] = sub i32 0, [[TMP3]] ; RV32-NEXT: [[TMP14:%.*]] = getelementptr float, ptr [[TMP8]], i32 [[TMP4]] ; RV32-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP14]], splat (i1 true), i32 [[TMP9]]) -; RV32-NEXT: [[VP_REVERSE1:%.*]] = fadd [[VP_OP_LOAD]], splat (float 1.000000e+00) +; RV32-NEXT: [[REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP9]]) +; RV32-NEXT: [[TMP15:%.*]] = fadd [[REVERSE]], splat (float 1.000000e+00) ; RV32-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]] ; RV32-NEXT: [[TMP22:%.*]] = getelementptr float, ptr [[TMP16]], i32 [[TMP4]] +; RV32-NEXT: [[VP_REVERSE1:%.*]] = call @llvm.experimental.vp.reverse.nxv4f32( [[TMP15]], splat (i1 true), i32 [[TMP9]]) ; RV32-NEXT: call void @llvm.vp.store.nxv4f32.p0( [[VP_REVERSE1]], ptr align 4 [[TMP22]], splat (i1 true), i32 [[TMP9]]) ; RV32-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP23]], [[INDEX]] ; RV32-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP23]] @@ -667,11 +695,15 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) { ; RV64-UF2-NEXT: [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[TMP11]] ; RV64-UF2-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP18]], align 4 ; RV64-UF2-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP19]], align 4 -; RV64-UF2-NEXT: [[REVERSE3:%.*]] = fadd [[WIDE_LOAD]], splat (float 1.000000e+00) -; RV64-UF2-NEXT: [[REVERSE4:%.*]] = fadd [[WIDE_LOAD1]], splat (float 1.000000e+00) +; RV64-UF2-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv4f32( [[WIDE_LOAD]]) +; RV64-UF2-NEXT: [[REVERSE2:%.*]] = call @llvm.vector.reverse.nxv4f32( [[WIDE_LOAD1]]) +; RV64-UF2-NEXT: [[TMP20:%.*]] = fadd [[REVERSE]], splat (float 1.000000e+00) +; RV64-UF2-NEXT: [[TMP21:%.*]] = fadd [[REVERSE2]], splat (float 1.000000e+00) ; RV64-UF2-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]] ; RV64-UF2-NEXT: [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[TMP15]] ; RV64-UF2-NEXT: [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[TMP11]] +; RV64-UF2-NEXT: [[REVERSE3:%.*]] = call @llvm.vector.reverse.nxv4f32( [[TMP20]]) +; RV64-UF2-NEXT: [[REVERSE4:%.*]] = call @llvm.vector.reverse.nxv4f32( [[TMP21]]) ; RV64-UF2-NEXT: store [[REVERSE3]], ptr [[TMP27]], align 4 ; RV64-UF2-NEXT: store [[REVERSE4]], ptr [[TMP32]], align 4 ; RV64-UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll index c4ebc4f2ad9c..09e116dfb19e 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll @@ -25,9 +25,11 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt ; IF-EVL-NEXT: [[TMP6:%.*]] = sub i64 0, [[TMP4]] ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP8]], i64 [[TMP6]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-NEXT: [[VP_REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_OP_LOAD]], splat (i1 true), i32 [[TMP5]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP7]] ; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP13]], i64 [[TMP6]] -; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_OP_LOAD]], ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-NEXT: [[VP_REVERSE3:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_REVERSE]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_REVERSE3]], ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP5]]) ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP20]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]] ; IF-EVL-NEXT: [[TMP24:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 @@ -60,9 +62,11 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt ; NO-VP-NEXT: [[TMP12:%.*]] = sub i64 0, [[TMP9]] ; NO-VP-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP12]] ; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP14]], align 4 +; NO-VP-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv4i32( [[WIDE_LOAD]]) ; NO-VP-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP8]] ; NO-VP-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP12]] -; NO-VP-NEXT: store [[WIDE_LOAD]], ptr [[TMP20]], align 4 +; NO-VP-NEXT: [[REVERSE1:%.*]] = call @llvm.vector.reverse.nxv4i32( [[REVERSE]]) +; NO-VP-NEXT: store [[REVERSE1]], ptr [[TMP20]], align 4 ; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] ; NO-VP-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -129,9 +133,11 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal ; IF-EVL-NEXT: [[TMP7:%.*]] = sub i64 0, [[TMP15]] ; IF-EVL-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP16]], i64 [[TMP7]] ; IF-EVL-NEXT: [[VP_OP_LOAD4:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP20]], [[VP_REVERSE_MASK]], i32 [[TMP5]]) +; IF-EVL-NEXT: [[VP_REVERSE:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_OP_LOAD4]], splat (i1 true), i32 [[TMP5]]) ; IF-EVL-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP11]] ; IF-EVL-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP21]], i64 [[TMP7]] -; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[VP_OP_LOAD4]], ptr align 4 [[TMP25]], [[VP_REVERSE_MASK]], i32 [[TMP5]]) +; IF-EVL-NEXT: [[TMP13:%.*]] = call @llvm.experimental.vp.reverse.nxv4i32( [[VP_REVERSE]], splat (i1 true), i32 [[TMP5]]) +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP13]], ptr align 4 [[TMP25]], [[VP_REVERSE_MASK]], i32 [[TMP5]]) ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP26]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]] ; IF-EVL-NEXT: [[TMP29:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 @@ -169,9 +175,11 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal ; NO-VP-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[TMP15]], i64 [[TMP14]] ; NO-VP-NEXT: [[REVERSE:%.*]] = call @llvm.vector.reverse.nxv4i1( [[TMP10]]) ; NO-VP-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr align 4 [[TMP16]], [[REVERSE]], poison) +; NO-VP-NEXT: [[REVERSE2:%.*]] = call @llvm.vector.reverse.nxv4i32( [[WIDE_MASKED_LOAD]]) ; NO-VP-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP8]] ; NO-VP-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[TMP17]], i64 [[TMP14]] -; NO-VP-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr align 4 [[TMP22]], [[REVERSE]]) +; NO-VP-NEXT: [[REVERSE4:%.*]] = call @llvm.vector.reverse.nxv4i32( [[REVERSE2]]) +; NO-VP-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[REVERSE4]], ptr align 4 [[TMP22]], [[REVERSE]]) ; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] ; NO-VP-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll index 379d595ab85b..7fbc6a581b24 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-riscv-vector-reverse.ll @@ -37,10 +37,12 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> ; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-end-pointer ir<%arrayidx>, vp<%evl> ; CHECK-NEXT: WIDEN ir<%0> = vp.load vp<[[VP8]]>, vp<%evl> -; CHECK-NEXT: WIDEN ir<%add9> = add ir<%0>, ir<1> +; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP9:%[0-9]+]]> = call llvm.experimental.vp.reverse(ir<%0>, ir, vp<%evl>) +; CHECK-NEXT: WIDEN ir<%add9> = add vp<[[VP9]]>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> -; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-end-pointer ir<%arrayidx3>, vp<%evl> -; CHECK-NEXT: WIDEN vp.store vp<[[VP9]]>, ir<%add9>, vp<%evl> +; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-end-pointer ir<%arrayidx3>, vp<%evl> +; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP11:%[0-9]+]]> = call llvm.experimental.vp.reverse(ir<%add9>, ir, vp<%evl>) +; CHECK-NEXT: WIDEN vp.store vp<[[VP10]]>, vp<[[VP11]]>, vp<%evl> ; CHECK-NEXT: EMIT vp<%current.iteration.next> = add vp<%evl>, vp<[[VP5]]> ; CHECK-NEXT: EMIT vp<%avl.next> = sub nuw vp<%avl>, vp<%evl> ; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP0]]> diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll b/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll deleted file mode 100644 index 7e122001d042..000000000000 --- a/llvm/test/Transforms/LoopVectorize/VPlan/simplify-reverse-reverse.ll +++ /dev/null @@ -1,59 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter-out-after "middle.block:" --version 6 -; RUN: opt -p loop-vectorize -force-vector-width=4 -disable-output -vplan-print-after=simplifyReverses %s 2>&1 | FileCheck %s - -define void @reverse_copy(ptr noalias %src, ptr noalias %dst, i32 %n) { -; CHECK-LABEL: VPlan for loop in 'reverse_copy' -; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF -; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF -; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count -; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = original trip-count -; CHECK-EMPTY: -; CHECK-NEXT: ir-bb: -; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV ((-1 * (0 smin (-1 + %n))) + %n) -; CHECK-NEXT: Successor(s): scalar.ph, vector.ph -; CHECK-EMPTY: -; CHECK-NEXT: vector.ph: -; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = DERIVED-IV ir<%n> + vp<[[VP2]]> * ir<-1> -; CHECK-NEXT: Successor(s): vector loop -; CHECK-EMPTY: -; CHECK-NEXT: vector loop: { -; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV -; CHECK-EMPTY: -; CHECK-NEXT: vector.body: -; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<%n>, ir<-1>, vp<[[VP0]]> -; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = DERIVED-IV ir<%n> + vp<[[VP5]]> * ir<-1> -; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<-1>, vp<[[VP0]]> -; CHECK-NEXT: CLONE ir<%iv.next> = add nsw vp<[[VP7]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr ir<%src>, ir<%iv.next> -; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-end-pointer ir<%gep.src>, vp<[[VP0]]> -; CHECK-NEXT: WIDEN ir<%x> = load vp<[[VP8]]> -; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = reverse ir<%x> -; CHECK-NEXT: CLONE ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv.next> -; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-end-pointer ir<%gep.dst>, vp<[[VP0]]> -; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = reverse vp<[[VP9]]> -; CHECK-NEXT: WIDEN store vp<[[VP10]]>, ir<%x> -; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]> -; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]> -; CHECK-NEXT: No successors -; CHECK-NEXT: } -; CHECK-NEXT: Successor(s): middle.block -; CHECK-EMPTY: -; CHECK-NEXT: middle.block: -; -entry: - br label %loop - -loop: - %iv = phi i32 [%n, %entry], [%iv.next, %loop] - %iv.next = add nsw i32 %iv, -1 - %gep.src = getelementptr i32, ptr %src, i32 %iv.next - %x = load i32, ptr %gep.src - %gep.dst = getelementptr i32, ptr %dst, i32 %iv.next - store i32 %x, ptr %gep.dst - %ec = icmp sgt i32 %iv.next, 0 - br i1 %ec, label %loop, label %exit - -exit: - ret void -} diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll index a7271524f019..8a7b3f1c4582 100644 --- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll +++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll @@ -53,7 +53,6 @@ ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] reassociateHeaderMask ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyRecipes ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeBranchOnConst -; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] simplifyReverses ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] removeDeadRecipes ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] createAndOptimizeReplicateRegions ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] mergeBlocksIntoPredecessors diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll index 1bfe36f9f1e7..ba2da480dc90 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll @@ -1091,74 +1091,27 @@ for.end: define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr nocapture readonly %trigger) { ; AVX1-LABEL: define void @foo6( ; AVX1-SAME: ptr readonly captures(none) [[IN:%.*]], ptr captures(none) [[OUT:%.*]], i32 [[SIZE:%.*]], ptr readonly captures(none) [[TRIGGER:%.*]]) #[[ATTR0]] { -; AVX1-NEXT: [[ENTRY:.*:]] +; AVX1-NEXT: [[ENTRY:.*]]: ; AVX1-NEXT: br label %[[FOR_BODY:.*]] ; AVX1: [[FOR_BODY]]: -; AVX1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i64 32768 -; AVX1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TRIGGER]], i64 16384 -; AVX1-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[IN]], i64 32768 -; AVX1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]] -; AVX1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[TRIGGER]], [[SCEVGEP]] -; AVX1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; AVX1-NEXT: [[BOUND03:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP2]] -; AVX1-NEXT: [[BOUND14:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP]] -; AVX1-NEXT: [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]] -; AVX1-NEXT: [[CMP1:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]] -; AVX1-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC:.*]] +; AVX1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 4095, %[[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ] +; AVX1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[INDVARS_IV]] +; AVX1-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; AVX1-NEXT: [[CMP1:%.*]] = icmp sgt i32 [[TMP0]], 0 +; AVX1-NEXT: br i1 [[CMP1]], label %[[IF_THEN:.*]], label %[[FOR_INC]] +; AVX1: [[IF_THEN]]: +; AVX1-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, ptr [[IN]], i64 [[INDVARS_IV]] +; AVX1-NEXT: [[TMP1:%.*]] = load double, ptr [[ARRAYIDX3]], align 8 +; AVX1-NEXT: [[ADD:%.*]] = fadd double [[TMP1]], 5.000000e-01 +; AVX1-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVARS_IV]] +; AVX1-NEXT: store double [[ADD]], ptr [[ARRAYIDX5]], align 8 +; AVX1-NEXT: br label %[[FOR_INC]] ; AVX1: [[FOR_INC]]: -; AVX1-NEXT: br label %[[VECTOR_BODY:.*]] -; AVX1: [[VECTOR_BODY]]: -; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_INC]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; AVX1-NEXT: [[TMP0:%.*]] = sub i64 4095, [[INDEX]] -; AVX1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], i64 [[TMP0]] -; AVX1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -3 -; AVX1-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -7 -; AVX1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -11 -; AVX1-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -15 -; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4, !alias.scope [[META18:![0-9]+]] -; AVX1-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4, !alias.scope [[META18]] -; AVX1-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4, !alias.scope [[META18]] -; AVX1-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i32>, ptr [[TMP5]], align 4, !alias.scope [[META18]] -; AVX1-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> -; AVX1-NEXT: [[REVERSE9:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD6]], <4 x i32> poison, <4 x i32> -; AVX1-NEXT: [[REVERSE10:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD7]], <4 x i32> poison, <4 x i32> -; AVX1-NEXT: [[REVERSE11:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD8]], <4 x i32> poison, <4 x i32> -; AVX1-NEXT: [[TMP6:%.*]] = icmp sgt <4 x i32> [[REVERSE]], zeroinitializer -; AVX1-NEXT: [[TMP7:%.*]] = icmp sgt <4 x i32> [[REVERSE9]], zeroinitializer -; AVX1-NEXT: [[TMP8:%.*]] = icmp sgt <4 x i32> [[REVERSE10]], zeroinitializer -; AVX1-NEXT: [[TMP9:%.*]] = icmp sgt <4 x i32> [[REVERSE11]], zeroinitializer -; AVX1-NEXT: [[TMP10:%.*]] = getelementptr double, ptr [[IN]], i64 [[TMP0]] -; AVX1-NEXT: [[TMP11:%.*]] = getelementptr double, ptr [[TMP10]], i64 -3 -; AVX1-NEXT: [[TMP12:%.*]] = getelementptr double, ptr [[TMP10]], i64 -7 -; AVX1-NEXT: [[TMP13:%.*]] = getelementptr double, ptr [[TMP10]], i64 -11 -; AVX1-NEXT: [[TMP14:%.*]] = getelementptr double, ptr [[TMP10]], i64 -15 -; AVX1-NEXT: [[REVERSE12:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <4 x i32> -; AVX1-NEXT: [[REVERSE13:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> poison, <4 x i32> -; AVX1-NEXT: [[REVERSE14:%.*]] = shufflevector <4 x i1> [[TMP8]], <4 x i1> poison, <4 x i32> -; AVX1-NEXT: [[REVERSE15:%.*]] = shufflevector <4 x i1> [[TMP9]], <4 x i1> poison, <4 x i32> -; AVX1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP11]], <4 x i1> [[REVERSE12]], <4 x double> poison), !alias.scope [[META21:![0-9]+]] -; AVX1-NEXT: [[WIDE_MASKED_LOAD16:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP12]], <4 x i1> [[REVERSE13]], <4 x double> poison), !alias.scope [[META21]] -; AVX1-NEXT: [[WIDE_MASKED_LOAD17:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP13]], <4 x i1> [[REVERSE14]], <4 x double> poison), !alias.scope [[META21]] -; AVX1-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP14]], <4 x i1> [[REVERSE15]], <4 x double> poison), !alias.scope [[META21]] -; AVX1-NEXT: [[TMP15:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], splat (double 5.000000e-01) -; AVX1-NEXT: [[TMP16:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD16]], splat (double 5.000000e-01) -; AVX1-NEXT: [[TMP17:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD17]], splat (double 5.000000e-01) -; AVX1-NEXT: [[TMP18:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD18]], splat (double 5.000000e-01) -; AVX1-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[OUT]], i64 [[TMP0]] -; AVX1-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[TMP19]], i64 -3 -; AVX1-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[TMP19]], i64 -7 -; AVX1-NEXT: [[TMP22:%.*]] = getelementptr double, ptr [[TMP19]], i64 -11 -; AVX1-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP19]], i64 -15 -; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP15]], ptr align 8 [[TMP20]], <4 x i1> [[REVERSE12]]), !alias.scope [[META23:![0-9]+]], !noalias [[META25:![0-9]+]] -; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP16]], ptr align 8 [[TMP21]], <4 x i1> [[REVERSE13]]), !alias.scope [[META23]], !noalias [[META25]] -; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP17]], ptr align 8 [[TMP22]], <4 x i1> [[REVERSE14]]), !alias.scope [[META23]], !noalias [[META25]] -; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP18]], ptr align 8 [[TMP23]], <4 x i1> [[REVERSE15]]), !alias.scope [[META23]], !noalias [[META25]] -; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 -; AVX1-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 -; AVX1-NEXT: br i1 [[TMP24]], label %[[FOR_END:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] +; AVX1-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1 +; AVX1-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV]], 0 +; AVX1-NEXT: br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_BODY]] ; AVX1: [[FOR_END]]: -; AVX1-NEXT: br [[FOR_END1:label %.*]] -; AVX1: [[IF_THEN]]: +; AVX1-NEXT: ret void ; ; AVX2-LABEL: define void @foo6( ; AVX2-SAME: ptr readonly captures(none) [[IN:%.*]], ptr captures(none) [[OUT:%.*]], i32 [[SIZE:%.*]], ptr readonly captures(none) [[TRIGGER:%.*]]) #[[ATTR0]] { @@ -1207,10 +1160,14 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr ; AVX2-NEXT: [[REVERSE13:%.*]] = shufflevector <4 x i1> [[TMP7]], <4 x i1> poison, <4 x i32> ; AVX2-NEXT: [[REVERSE14:%.*]] = shufflevector <4 x i1> [[TMP8]], <4 x i1> poison, <4 x i32> ; AVX2-NEXT: [[REVERSE15:%.*]] = shufflevector <4 x i1> [[TMP9]], <4 x i1> poison, <4 x i32> -; AVX2-NEXT: [[REVERSE19:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP11]], <4 x i1> [[REVERSE12]], <4 x double> poison), !alias.scope [[META25:![0-9]+]] -; AVX2-NEXT: [[REVERSE20:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP12]], <4 x i1> [[REVERSE13]], <4 x double> poison), !alias.scope [[META25]] -; AVX2-NEXT: [[REVERSE21:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP13]], <4 x i1> [[REVERSE14]], <4 x double> poison), !alias.scope [[META25]] -; AVX2-NEXT: [[REVERSE22:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP14]], <4 x i1> [[REVERSE15]], <4 x double> poison), !alias.scope [[META25]] +; AVX2-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP11]], <4 x i1> [[REVERSE12]], <4 x double> poison), !alias.scope [[META25:![0-9]+]] +; AVX2-NEXT: [[WIDE_MASKED_LOAD16:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP12]], <4 x i1> [[REVERSE13]], <4 x double> poison), !alias.scope [[META25]] +; AVX2-NEXT: [[WIDE_MASKED_LOAD17:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP13]], <4 x i1> [[REVERSE14]], <4 x double> poison), !alias.scope [[META25]] +; AVX2-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP14]], <4 x i1> [[REVERSE15]], <4 x double> poison), !alias.scope [[META25]] +; AVX2-NEXT: [[REVERSE19:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE20:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD16]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE21:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD17]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE22:%.*]] = shufflevector <4 x double> [[WIDE_MASKED_LOAD18]], <4 x double> poison, <4 x i32> ; AVX2-NEXT: [[TMP15:%.*]] = fadd <4 x double> [[REVERSE19]], splat (double 5.000000e-01) ; AVX2-NEXT: [[TMP16:%.*]] = fadd <4 x double> [[REVERSE20]], splat (double 5.000000e-01) ; AVX2-NEXT: [[TMP17:%.*]] = fadd <4 x double> [[REVERSE21]], splat (double 5.000000e-01) @@ -1220,10 +1177,14 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr ; AVX2-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[TMP19]], i64 -7 ; AVX2-NEXT: [[TMP22:%.*]] = getelementptr double, ptr [[TMP19]], i64 -11 ; AVX2-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP19]], i64 -15 -; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP15]], ptr align 8 [[TMP20]], <4 x i1> [[REVERSE12]]), !alias.scope [[META27:![0-9]+]], !noalias [[META29:![0-9]+]] -; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP16]], ptr align 8 [[TMP21]], <4 x i1> [[REVERSE13]]), !alias.scope [[META27]], !noalias [[META29]] -; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP17]], ptr align 8 [[TMP22]], <4 x i1> [[REVERSE14]]), !alias.scope [[META27]], !noalias [[META29]] -; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[TMP18]], ptr align 8 [[TMP23]], <4 x i1> [[REVERSE15]]), !alias.scope [[META27]], !noalias [[META29]] +; AVX2-NEXT: [[REVERSE23:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE24:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE25:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: [[REVERSE26:%.*]] = shufflevector <4 x double> [[TMP18]], <4 x double> poison, <4 x i32> +; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[REVERSE23]], ptr align 8 [[TMP20]], <4 x i1> [[REVERSE12]]), !alias.scope [[META27:![0-9]+]], !noalias [[META29:![0-9]+]] +; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[REVERSE24]], ptr align 8 [[TMP21]], <4 x i1> [[REVERSE13]]), !alias.scope [[META27]], !noalias [[META29]] +; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[REVERSE25]], ptr align 8 [[TMP22]], <4 x i1> [[REVERSE14]]), !alias.scope [[META27]], !noalias [[META29]] +; AVX2-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[REVERSE26]], ptr align 8 [[TMP23]], <4 x i1> [[REVERSE15]]), !alias.scope [[META27]], !noalias [[META29]] ; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; AVX2-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 ; AVX2-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] @@ -1278,10 +1239,14 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr ; AVX512-NEXT: [[REVERSE13:%.*]] = shufflevector <8 x i1> [[TMP7]], <8 x i1> poison, <8 x i32> ; AVX512-NEXT: [[REVERSE14:%.*]] = shufflevector <8 x i1> [[TMP8]], <8 x i1> poison, <8 x i32> ; AVX512-NEXT: [[REVERSE15:%.*]] = shufflevector <8 x i1> [[TMP9]], <8 x i1> poison, <8 x i32> -; AVX512-NEXT: [[REVERSE19:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP11]], <8 x i1> [[REVERSE12]], <8 x double> poison), !alias.scope [[META37:![0-9]+]] -; AVX512-NEXT: [[REVERSE20:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP12]], <8 x i1> [[REVERSE13]], <8 x double> poison), !alias.scope [[META37]] -; AVX512-NEXT: [[REVERSE21:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP13]], <8 x i1> [[REVERSE14]], <8 x double> poison), !alias.scope [[META37]] -; AVX512-NEXT: [[REVERSE22:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP14]], <8 x i1> [[REVERSE15]], <8 x double> poison), !alias.scope [[META37]] +; AVX512-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP11]], <8 x i1> [[REVERSE12]], <8 x double> poison), !alias.scope [[META37:![0-9]+]] +; AVX512-NEXT: [[WIDE_MASKED_LOAD16:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP12]], <8 x i1> [[REVERSE13]], <8 x double> poison), !alias.scope [[META37]] +; AVX512-NEXT: [[WIDE_MASKED_LOAD17:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP13]], <8 x i1> [[REVERSE14]], <8 x double> poison), !alias.scope [[META37]] +; AVX512-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 8 [[TMP14]], <8 x i1> [[REVERSE15]], <8 x double> poison), !alias.scope [[META37]] +; AVX512-NEXT: [[REVERSE19:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE20:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD16]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE21:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD17]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE22:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD18]], <8 x double> poison, <8 x i32> ; AVX512-NEXT: [[TMP15:%.*]] = fadd <8 x double> [[REVERSE19]], splat (double 5.000000e-01) ; AVX512-NEXT: [[TMP16:%.*]] = fadd <8 x double> [[REVERSE20]], splat (double 5.000000e-01) ; AVX512-NEXT: [[TMP17:%.*]] = fadd <8 x double> [[REVERSE21]], splat (double 5.000000e-01) @@ -1291,10 +1256,14 @@ define void @foo6(ptr nocapture readonly %in, ptr nocapture %out, i32 %size, ptr ; AVX512-NEXT: [[TMP21:%.*]] = getelementptr double, ptr [[TMP19]], i64 -15 ; AVX512-NEXT: [[TMP22:%.*]] = getelementptr double, ptr [[TMP19]], i64 -23 ; AVX512-NEXT: [[TMP23:%.*]] = getelementptr double, ptr [[TMP19]], i64 -31 -; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[TMP15]], ptr align 8 [[TMP20]], <8 x i1> [[REVERSE12]]), !alias.scope [[META39:![0-9]+]], !noalias [[META41:![0-9]+]] -; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[TMP16]], ptr align 8 [[TMP21]], <8 x i1> [[REVERSE13]]), !alias.scope [[META39]], !noalias [[META41]] -; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[TMP17]], ptr align 8 [[TMP22]], <8 x i1> [[REVERSE14]]), !alias.scope [[META39]], !noalias [[META41]] -; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[TMP18]], ptr align 8 [[TMP23]], <8 x i1> [[REVERSE15]]), !alias.scope [[META39]], !noalias [[META41]] +; AVX512-NEXT: [[REVERSE23:%.*]] = shufflevector <8 x double> [[TMP15]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE24:%.*]] = shufflevector <8 x double> [[TMP16]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE25:%.*]] = shufflevector <8 x double> [[TMP17]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: [[REVERSE26:%.*]] = shufflevector <8 x double> [[TMP18]], <8 x double> poison, <8 x i32> +; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[REVERSE23]], ptr align 8 [[TMP20]], <8 x i1> [[REVERSE12]]), !alias.scope [[META39:![0-9]+]], !noalias [[META41:![0-9]+]] +; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[REVERSE24]], ptr align 8 [[TMP21]], <8 x i1> [[REVERSE13]]), !alias.scope [[META39]], !noalias [[META41]] +; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[REVERSE25]], ptr align 8 [[TMP22]], <8 x i1> [[REVERSE14]]), !alias.scope [[META39]], !noalias [[META41]] +; AVX512-NEXT: call void @llvm.masked.store.v8f64.p0(<8 x double> [[REVERSE26]], ptr align 8 [[TMP23]], <8 x i1> [[REVERSE15]]), !alias.scope [[META39]], !noalias [[META41]] ; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 ; AVX512-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096 ; AVX512-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] @@ -1398,13 +1367,13 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in ; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP27]], <4 x i1> [[TMP23]]) ; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; AVX1-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; AVX1-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] +; AVX1-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; AVX1: [[MIDDLE_BLOCK]]: ; AVX1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]] ; AVX1-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] ; AVX1: [[VEC_EPILOG_ITER_CHECK]]: ; AVX1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 -; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29:![0-9]+]] +; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19:![0-9]+]] ; AVX1: [[VEC_EPILOG_PH]]: ; AVX1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; AVX1-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4 @@ -1424,7 +1393,7 @@ define void @foo7(ptr noalias nocapture %out, ptr noalias nocapture readonly %in ; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP35]], <4 x i1> [[TMP34]]) ; AVX1-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4 ; AVX1-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]] -; AVX1-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] +; AVX1-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; AVX1: [[VEC_EPILOG_MIDDLE_BLOCK]]: ; AVX1-NEXT: [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]] ; AVX1-NEXT: br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] @@ -1719,13 +1688,13 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in ; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP27]], <4 x i1> [[TMP23]]) ; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; AVX1-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; AVX1-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] +; AVX1-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] ; AVX1: [[MIDDLE_BLOCK]]: ; AVX1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]] ; AVX1-NEXT: br i1 [[CMP_N]], [[FOR_END_LOOPEXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]] ; AVX1: [[VEC_EPILOG_ITER_CHECK]]: ; AVX1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4 -; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF29]] +; AVX1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF19]] ; AVX1: [[VEC_EPILOG_PH]]: ; AVX1-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; AVX1-NEXT: [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4 @@ -1745,7 +1714,7 @@ define void @foo8(ptr noalias nocapture %out, ptr noalias nocapture readonly %in ; AVX1-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> splat (double 5.000000e-01), ptr align 8 [[TMP35]], <4 x i1> [[TMP34]]) ; AVX1-NEXT: [[INDEX_NEXT13]] = add nuw i64 [[INDEX10]], 4 ; AVX1-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC9]] -; AVX1-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]] +; AVX1-NEXT: br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] ; AVX1: [[VEC_EPILOG_MIDDLE_BLOCK]]: ; AVX1-NEXT: [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]] ; AVX1-NEXT: br i1 [[CMP_N14]], [[FOR_END_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]] diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll index a242d9efe468..451085f04fde 100644 --- a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll +++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll @@ -173,8 +173,8 @@ define signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n) { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP12]] ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 -7 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x float>, ptr [[TMP15]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], splat (float 1.000000e+00) -; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x float> [[TMP14]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <8 x float> [[WIDE_LOAD]], <8 x float> poison, <8 x i32> +; CHECK-NEXT: [[TMP16:%.*]] = fadd fast <8 x float> [[REVERSE]], splat (float 1.000000e+00) ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: store <8 x float> [[TMP16]], ptr [[TMP17]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 @@ -201,8 +201,8 @@ define signext i32 @f2(ptr noalias %A, ptr noalias %B, i32 signext %n) { ; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP24]] ; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 -3 ; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <4 x float>, ptr [[TMP27]], align 4 -; CHECK-NEXT: [[TMP26:%.*]] = fadd fast <4 x float> [[WIDE_LOAD9]], splat (float 1.000000e+00) -; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <4 x float> [[TMP26]], <4 x float> poison, <4 x i32> +; CHECK-NEXT: [[REVERSE10:%.*]] = shufflevector <4 x float> [[WIDE_LOAD9]], <4 x float> poison, <4 x i32> +; CHECK-NEXT: [[TMP28:%.*]] = fadd fast <4 x float> [[REVERSE10]], splat (float 1.000000e+00) ; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX7]] ; CHECK-NEXT: store <4 x float> [[TMP28]], ptr [[TMP29]], align 4 ; CHECK-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX7]], 4 diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-known-true.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-known-true.ll index 98c80065c792..66c848e04582 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-check-known-true.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-check-known-true.ll @@ -39,8 +39,10 @@ define void @test_runtime_check_known_false_after_construction(ptr %start.1, ptr ; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START_2_DIFF]], i64 [[OFFSET_IDX]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[NEXT_GEP3]], i64 -3 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP14]], align 8 +; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> ; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 -3 -; CHECK-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP16]], align 8 +; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i64> [[REVERSE]], <4 x i64> poison, <4 x i32> +; CHECK-NEXT: store <4 x i64> [[REVERSE4]], ptr [[TMP16]], align 8 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -49,11 +51,11 @@ define void @test_runtime_check_known_false_after_construction(ptr %start.1, ptr ; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[START_1]], %[[ENTRY]] ], [ [[START_1]], %[[VECTOR_SCEVCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[START_2_DIFF]], %[[ENTRY]] ], [ [[START_2_DIFF]], %[[VECTOR_SCEVCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[START_2_DIFF]], %[[ENTRY]] ], [ [[START_2_DIFF]], %[[VECTOR_SCEVCHECK]] ] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] -; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ] ; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 -8 ; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 -8 ; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[PTR_IV_2]], align 8 diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll index 23ebd63d07cd..18063f09e15c 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-hoist.ll @@ -998,13 +998,16 @@ define void @decreasing_inner_iv(ptr nocapture noundef %dst, ptr nocapture nound ; CHECK-NEXT: [[TMP22:%.*]] = add nsw i64 [[TMP21]], [[TMP15]] ; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP22]] ; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 -3 -; CHECK-NEXT: [[REVERSE:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4, !alias.scope [[META46:![0-9]+]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP24]], align 4, !alias.scope [[META46:![0-9]+]] +; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP25:%.*]] = add nsw i64 [[TMP21]], [[TMP16]] ; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP25]] ; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i64 -3 -; CHECK-NEXT: [[REVERSE4:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4, !alias.scope [[META49:![0-9]+]], !noalias [[META46]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4, !alias.scope [[META49:![0-9]+]], !noalias [[META46]] +; CHECK-NEXT: [[REVERSE4:%.*]] = shufflevector <4 x i32> [[WIDE_LOAD3]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP28:%.*]] = add nsw <4 x i32> [[REVERSE4]], [[REVERSE]] -; CHECK-NEXT: store <4 x i32> [[TMP28]], ptr [[TMP27]], align 4, !alias.scope [[META49]], !noalias [[META46]] +; CHECK-NEXT: [[REVERSE5:%.*]] = shufflevector <4 x i32> [[TMP28]], <4 x i32> poison, <4 x i32> +; CHECK-NEXT: store <4 x i32> [[REVERSE5]], ptr [[TMP27]], align 4, !alias.scope [[META49]], !noalias [[META46]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP51:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/simplify-reverse-reverse.ll b/llvm/test/Transforms/LoopVectorize/simplify-reverse-reverse.ll deleted file mode 100644 index a76a01574c59..000000000000 --- a/llvm/test/Transforms/LoopVectorize/simplify-reverse-reverse.ll +++ /dev/null @@ -1,194 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 -; RUN: opt -p loop-vectorize -force-vector-width=4 < %s -S | FileCheck %s - -define void @reverse_copy(ptr noalias %src, ptr noalias %dst, i32 %n) { -; CHECK-LABEL: define void @reverse_copy( -; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[N:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -1 -; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 0) -; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[N]], [[SMIN]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[N_VEC]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[N]], [[INDEX]] -; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[TMP3]], -1 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i64 -3 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST]], i32 [[TMP4]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i64 -3 -; CHECK-NEXT: store <4 x i32> [[WIDE_LOAD]], ptr [[TMP8]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ] -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], -1 -; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[IV_NEXT]] -; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_SRC]], align 4 -; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i32 [[IV_NEXT]] -; CHECK-NEXT: store i32 [[X]], ptr [[GEP_DST]], align 4 -; CHECK-NEXT: [[EC:%.*]] = icmp sgt i32 [[IV_NEXT]], 0 -; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void -; -entry: - br label %loop - -loop: - %iv = phi i32 [%n, %entry], [%iv.next, %loop] - %iv.next = add nsw i32 %iv, -1 - %gep.src = getelementptr i32, ptr %src, i32 %iv.next - %x = load i32, ptr %gep.src - %gep.dst = getelementptr i32, ptr %dst, i32 %iv.next - store i32 %x, ptr %gep.dst - %ec = icmp sgt i32 %iv.next, 0 - br i1 %ec, label %loop, label %exit - -exit: - ret void -} - -define void @reverse_binop(ptr noalias %src, ptr noalias %dst, i32 %n) { -; CHECK-LABEL: define void @reverse_binop( -; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[N:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -1 -; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 0) -; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[N]], [[SMIN]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[N_VEC]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[N]], [[INDEX]] -; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[TMP3]], -1 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i64 -3 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i32> [[WIDE_LOAD]], splat (i32 1) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[DST]], i32 [[TMP4]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[TMP8]], i64 -3 -; CHECK-NEXT: store <4 x i32> [[TMP7]], ptr [[TMP9]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ] -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], -1 -; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i32 [[IV_NEXT]] -; CHECK-NEXT: [[X:%.*]] = load i32, ptr [[GEP_SRC]], align 4 -; CHECK-NEXT: [[Y:%.*]] = add i32 [[X]], 1 -; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i32 [[IV_NEXT]] -; CHECK-NEXT: store i32 [[Y]], ptr [[GEP_DST]], align 4 -; CHECK-NEXT: [[EC:%.*]] = icmp sgt i32 [[IV_NEXT]], 0 -; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void -; -entry: - br label %loop - -loop: - %iv = phi i32 [%n, %entry], [%iv.next, %loop] - %iv.next = add nsw i32 %iv, -1 - %gep.src = getelementptr i32, ptr %src, i32 %iv.next - %x = load i32, ptr %gep.src - %y = add i32 %x, 1 - %gep.dst = getelementptr i32, ptr %dst, i32 %iv.next - store i32 %y, ptr %gep.dst - %ec = icmp sgt i32 %iv.next, 0 - br i1 %ec, label %loop, label %exit - -exit: - ret void -} - -define void @reverse_fneg(ptr noalias %src, ptr noalias %dst, i32 %n) { -; CHECK-LABEL: define void @reverse_fneg( -; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[N:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[N]], -1 -; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 0) -; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[N]], [[SMIN]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4 -; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[N_VEC]] -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = sub i32 [[N]], [[INDEX]] -; CHECK-NEXT: [[TMP4:%.*]] = add nsw i32 [[TMP3]], -1 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr float, ptr [[SRC]], i32 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr float, ptr [[TMP5]], i64 -3 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP6]], align 4 -; CHECK-NEXT: [[REVERSE1:%.*]] = fneg <4 x float> [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[DST]], i32 [[TMP4]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i64 -3 -; CHECK-NEXT: store <4 x float> [[REVERSE1]], ptr [[TMP9]], align 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 -; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] -; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[ENTRY]] ] -; CHECK-NEXT: br label %[[LOOP:.*]] -; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], -1 -; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr float, ptr [[SRC]], i32 [[IV_NEXT]] -; CHECK-NEXT: [[X:%.*]] = load float, ptr [[GEP_SRC]], align 4 -; CHECK-NEXT: [[Y:%.*]] = fneg float [[X]] -; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr float, ptr [[DST]], i32 [[IV_NEXT]] -; CHECK-NEXT: store float [[Y]], ptr [[GEP_DST]], align 4 -; CHECK-NEXT: [[EC:%.*]] = icmp sgt i32 [[IV_NEXT]], 0 -; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]] -; CHECK: [[EXIT]]: -; CHECK-NEXT: ret void -; -entry: - br label %loop - -loop: - %iv = phi i32 [%n, %entry], [%iv.next, %loop] - %iv.next = add nsw i32 %iv, -1 - %gep.src = getelementptr float, ptr %src, i32 %iv.next - %x = load float, ptr %gep.src - %y = fneg float %x - %gep.dst = getelementptr float, ptr %dst, i32 %iv.next - store float %y, ptr %gep.dst - %ec = icmp sgt i32 %iv.next, 0 - br i1 %ec, label %loop, label %exit - -exit: - ret void -}