diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index bedf9e217c71c5..097c8cdc5fb34f 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -584,6 +584,12 @@ OPT_CONFIG_INTEGER(JitDoLoopHoisting, "JitDoLoopHoisting", 1) // Perform loop OPT_CONFIG_INTEGER(JitDoLoopInversion, "JitDoLoopInversion", 1) // Perform loop inversion on "for/while" loops RELEASE_CONFIG_INTEGER(JitLoopInversionSizeLimit, "JitLoopInversionSizeLimit", 100) // limit inversion to loops with no // more than this many tree nodes +// When set, skip inverting a loop that is already bottom-tested (has an exiting BBJ_COND latch) and +// has no recognized induction variable, unless the loop-continuation test that would be duplicated +// contains a call or a memory load whose address has no loop-varying local. +RELEASE_CONFIG_INTEGER(JitLoopInversionRequireBenefitForBottomTested, + "JitLoopInversionRequireBenefitForBottomTested", + 0) OPT_CONFIG_INTEGER(JitDoRangeAnalysis, "JitDoRangeAnalysis", 1) // Perform range check analysis OPT_CONFIG_INTEGER(JitDoVNBasedDeadStoreRemoval, "JitDoVNBasedDeadStoreRemoval", 1) // Perform VN-based dead store // removal diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 58e3310b66ebf2..5014671a718086 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -1949,15 +1949,22 @@ bool Compiler::optTryInvertWhileLoop(FlowGraphNaturalLoop* loop) return (ivTestBlock != nullptr) && (candidate == ivTestBlock); }; + bool sawExitingCondLatch = false; + for (FlowEdge* const backEdge : loop->BackEdges()) { BasicBlock* const latch = backEdge->getSourceBlock(); - if (isExitingCondLatch(latch) && isIvTest(latch)) + if (isExitingCondLatch(latch)) { - JITDUMP("No loop-inversion for " FMT_LP "; IV-test latch " FMT_BB " already makes it bottom-tested\n", - loop->GetIndex(), latch->bbNum); - return false; + sawExitingCondLatch = true; + + if (isIvTest(latch)) + { + JITDUMP("No loop-inversion for " FMT_LP "; IV-test latch " FMT_BB " already makes it bottom-tested\n", + loop->GetIndex(), latch->bbNum); + return false; + } } if (latch->KindIs(BBJ_ALWAYS) && latch->isEmpty()) @@ -1965,17 +1972,103 @@ bool Compiler::optTryInvertWhileLoop(FlowGraphNaturalLoop* loop) for (FlowEdge* const predEdge : latch->PredEdges()) { BasicBlock* const pred = predEdge->getSourceBlock(); - if (loop->ContainsBlock(pred) && isExitingCondLatch(pred) && isIvTest(pred)) + if (loop->ContainsBlock(pred) && isExitingCondLatch(pred)) { - JITDUMP("No loop-inversion for " FMT_LP "; IV-test predecessor " FMT_BB - " of canonical latch " FMT_BB " already makes it bottom-tested\n", - loop->GetIndex(), pred->bbNum, latch->bbNum); - return false; + sawExitingCondLatch = true; + + if (isIvTest(pred)) + { + JITDUMP("No loop-inversion for " FMT_LP "; IV-test predecessor " FMT_BB + " of canonical latch " FMT_BB " already makes it bottom-tested\n", + loop->GetIndex(), pred->bbNum, latch->bbNum); + return false; + } } } } } + // If the loop is already bottom-tested (has an exiting BBJ_COND latch that is not the IV test) + // and no induction variable was recognized, only invert when the block that would be duplicated + // (condBlock) contains a call or a memory load whose address has no loop-varying local. Classify + // condBlock here and record the locals appearing in its indirection address expressions; the + // size-check walk below flags whether any of them is assigned in the loop (making the load + // loop-variant). + const bool checkBenefit = sawExitingCondLatch && (ivTestBlock == nullptr) && + (JitConfig.JitLoopInversionRequireBenefitForBottomTested() != 0); + bool condHasCall = false; + bool condHasIndir = false; + bool condAddrStored = false; + BitVecTraits condTraits(lvaCount, this); + BitVec condIndirAddrLocals = BitVecOps::UninitVal(); + if (checkBenefit) + { + assert(analyzedIteration); + + condIndirAddrLocals = BitVecOps::MakeEmpty(&condTraits); + + struct CondClassifier : GenTreeVisitor + { + BitVecTraits* m_traits; + BitVec* m_addrLocals; + bool* m_hasCall; + bool* m_hasIndir; + + enum + { + DoPreOrder = true + }; + + CondClassifier(Compiler* comp, BitVecTraits* traits, BitVec* addrLocals, bool* hasCall, bool* hasIndir) + : GenTreeVisitor(comp) + , m_traits(traits) + , m_addrLocals(addrLocals) + , m_hasCall(hasCall) + , m_hasIndir(hasIndir) + { + } + + void CollectLocals(GenTree* tree) + { + if (tree->OperIsLocal()) + { + BitVecOps::AddElemD(m_traits, *m_addrLocals, tree->AsLclVarCommon()->GetLclNum()); + } + tree->VisitOperands([&](GenTree* op) -> GenTree::VisitResult { + CollectLocals(op); + return GenTree::VisitResult::Continue; + }); + } + + fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) + { + GenTree* n = *use; + if (n->IsCall()) + { + *m_hasCall = true; + return WALK_ABORT; + } + if (n->OperIsIndir()) + { + *m_hasIndir = true; + CollectLocals(n->AsIndir()->Addr()); + } + return WALK_CONTINUE; + } + }; + + CondClassifier cc(this, &condTraits, &condIndirAddrLocals, &condHasCall, &condHasIndir); + for (Statement* const stmt : condBlock->Statements()) + { + GenTree* root = stmt->GetRootNode(); + cc.WalkTree(&root, nullptr); + if (condHasCall) + { + break; + } + } + } + JITDUMP("Condition in block " FMT_BB " of loop " FMT_LP " is a candidate for duplication to invert the loop\n", condBlock->bbNum, loop->GetIndex()); @@ -2051,6 +2144,13 @@ bool Compiler::optTryInvertWhileLoop(FlowGraphNaturalLoop* loop) { *boundsCheckFlag = true; } + // Note whether any local appearing in the condition's indirection addresses is stored + // in the loop (making that load loop-variant). + if (checkBenefit && !condHasCall && tree->OperIsLocalStore() && + BitVecOps::IsMember(&condTraits, condIndirAddrLocals, tree->AsLclVarCommon()->GetLclNum())) + { + condAddrStored = true; + } loopSize++; return 1; }); @@ -2081,6 +2181,21 @@ bool Compiler::optTryInvertWhileLoop(FlowGraphNaturalLoop* loop) } } + // Skip the inversion unless the duplicated test carries a benefit: a call, or an indirection off + // an address with no loop-varying local. condAddrStored is left conservatively false if the size + // walk above was skipped or aborted early, keeping the inversion. + if (checkBenefit) + { + const bool keepInverting = condHasCall || (condHasIndir && !condAddrStored); + if (!keepInverting) + { + JITDUMP("No loop-inversion for " FMT_LP "; already bottom-tested with no recognized IV and no " + "hoistable/call benefit in the duplicated condition\n", + loop->GetIndex()); + return false; + } + } + unsigned estDupCostSz = 0; for (BasicBlock* const block : duplicatedBlocks.BottomUpOrder())