Lines Matching +full:count +full:- +full:threshold
1 //===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 // reads a profile file generated by a sampling profiler (e.g. Linux Perf -
16 // - prof: Represents branch weights. This annotation is added to branches
22 //===----------------------------------------------------------------------===//
99 #define DEBUG_TYPE "sample-profile"
100 #define CSINLINE_DEBUG DEBUG_TYPE "-inline"
123 "sample-profile-file", cl::init(""), cl::value_desc("filename"),
124 cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
130 "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
131 cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
134 "salvage-stale-profile", cl::Hidden, cl::init(false),
138 SalvageUnusedProfile("salvage-unused-profile", cl::Hidden, cl::init(false),
143 "report-profile-staleness", cl::Hidden, cl::init(false),
147 "persist-profile-staleness", cl::Hidden, cl::init(false),
152 "profile-sample-accurate", cl::Hidden, cl::init(false),
153 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
155 "un-sampled callsites and functions conservatively as unknown. "));
158 "profile-sample-block-accurate", cl::Hidden, cl::init(false),
159 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
164 "profile-accurate-for-symsinlist", cl::Hidden, cl::init(true),
166 "be accurate. It may be overriden by profile-sample-accurate. "));
169 "sample-profile-merge-inlinee", cl::Hidden, cl::init(true),
172 "only be enabled when top-down order of profile loading is "
176 "sample-profile-top-down-load", cl::Hidden, cl::init(true),
177 cl::desc("Do profile annotation and inlining for functions in top-down "
182 UseProfiledCallGraph("use-profiled-call-graph", cl::init(true), cl::Hidden,
183 cl::desc("Process functions in a top-down order "
185 "-sample-profile-top-down-load is on."));
188 "sample-profile-inline-size", cl::Hidden, cl::init(false),
193 // side effects. For instance, pre-link SCC inliner would see merged profiles
196 "disable-sample-loader-inlining", cl::Hidden, cl::init(false),
197 cl::desc("If true, artifically skip inline transformation in sample-loader "
199 "--sample-profile-merge-inlinee)."));
203 SortProfiledSCC("sort-profiled-scc-member", cl::init(true), cl::Hidden,
207 "sample-profile-inline-growth-limit", cl::Hidden, cl::init(12),
208 cl::desc("The size growth ratio limit for proirity-based sample profile "
212 "sample-profile-inline-limit-min", cl::Hidden, cl::init(100),
214 "proirity-based sample profile loader inlining."));
217 "sample-profile-inline-limit-max", cl::Hidden, cl::init(10000),
219 "proirity-based sample profile loader inlining."));
222 "sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000),
223 cl::desc("Hot callsite threshold for proirity-based sample profile loader "
227 "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45),
228 cl::desc("Threshold for inlining cold callsites"));
232 "sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25),
234 "Relative hotness percentage threshold for indirect "
235 "call promotion in proirity-based sample profile loader inlining."));
238 "sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1),
243 "hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000),
245 "total sample count is above the specified percentile"));
248 "min-functions-for-staleness-error", cl::Hidden, cl::init(50),
253 "precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80),
258 "sample-profile-prioritized-inline", cl::Hidden,
263 "sample-profile-use-preinliner", cl::Hidden,
267 "sample-profile-recursive-inline", cl::Hidden,
271 "sample-profile-remove-probe", cl::Hidden, cl::init(false),
272 cl::desc("Remove pseudo-probe after sample profile annotation."));
275 "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"),
282 "sample-profile-inline-replay-scope",
295 "sample-profile-inline-replay-fallback",
312 "sample-profile-inline-replay-format",
326 MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden,
331 "overwrite-existing-weights", cl::Hidden, cl::init(false),
335 "annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false),
337 "sample-profile inline pass name."));
370 // built in post-thin-link phase and var promotion has been done, in GUIDToFuncNameMapper()
404 FS->GUIDToFuncNameMap = Map; in SetGUIDToFuncNameMapForAll()
405 for (const auto &ICS : FS->getCallsiteSamples()) { in SetGUIDToFuncNameMapForAll()
424 // Prorated callsite count, which will be used to guide inlining. For example,
448 if (LCS->getBodySamples().size() != RCS->getBodySamples().size()) in operator ()()
449 return LCS->getBodySamples().size() > RCS->getBodySamples().size(); in operator ()()
452 return LCS->getGUID() < RCS->getGUID(); in operator ()()
463 /// -sample-profile-file and annotates every affected function with the
499 uint64_t Threshold);
535 /// is one-to-one mapping.
538 /// Map from function name to profile name generated by call-graph based
539 /// profile fuzzy matching(--salvage-unused-profile).
570 // each callee function. Initially this is just the entry count.
591 // list and -profile-accurate-for-symsinlist flag, but it can be
592 // overriden by -profile-sample-accurate or profile-sample-accurate
622 const Instruction *TI = BB->getTerminator(); in findUnlikelyJumps()
623 // Check if a block ends with InvokeInst and mark non-taken branch unlikely. in findUnlikelyJumps()
630 const Instruction *SuccTI = Succ->getTerminator(); in findUnlikelyJumps()
632 if (SuccTI->getNumSuccessors() == 0) { in findUnlikelyJumps()
644 DT->recalculate(F); in computeDominanceAndLoopInfo()
649 LI->analyze(*DT); in computeDominanceAndLoopInfo()
667 // For non-CS profile, if a direct call/invoke instruction is inlined in in getInstWeight()
668 // profile (findCalleeFunctionSamples returns non-empty result), but not in getInstWeight()
670 // call instruction should have 0 count. in getInstWeight()
671 // For CS profile, the callsite count of previously inlined callees is in getInstWeight()
672 // populated with the entry count of the callees. in getInstWeight()
675 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB)) in getInstWeight()
702 CalleeName = Callee->getName(); in findCalleeFunctionSamples()
705 return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName); in findCalleeFunctionSamples()
711 return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL), in findCalleeFunctionSamples()
712 CalleeName, Reader->getRemapper(), in findCalleeFunctionSamples()
718 /// the total call count of the indirect call in \p Sum.
730 assert(L && R && "Expect non-null FunctionSamples"); in findIndirectCallFunctionSamples()
731 if (L->getHeadSamplesEstimate() != R->getHeadSamplesEstimate()) in findIndirectCallFunctionSamples()
732 return L->getHeadSamplesEstimate() > R->getHeadSamplesEstimate(); in findIndirectCallFunctionSamples()
733 return L->getGUID() < R->getGUID(); in findIndirectCallFunctionSamples()
738 ContextTracker->getIndirectCalleeContextSamplesFor(DIL); in findIndirectCallFunctionSamples()
742 // For CSSPGO, we only use target context profile's entry count in findIndirectCallFunctionSamples()
743 // as that already includes both inlined callee and non-inlined ones.. in findIndirectCallFunctionSamples()
746 Sum += FS->getHeadSamplesEstimate(); in findIndirectCallFunctionSamples()
759 if (auto T = FS->findCallTargetMapAt(CallSite)) in findIndirectCallFunctionSamples()
762 if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) { in findIndirectCallFunctionSamples()
763 if (M->empty()) in findIndirectCallFunctionSamples()
789 it.first->second = ContextTracker->getContextSamplesFor(DIL); in findFunctionSamples()
791 it.first->second = Samples->findFunctionSamples( in findFunctionSamples()
792 DIL, Reader->getRemapper(), &FuncNameToProfNameMap); in findFunctionSamples()
794 return it.first->second; in findFunctionSamples()
799 /// If the profile count for the promotion candidate \p Candidate is
802 /// NOMORE_ICP_MAGICNUM count values in the value profile of \p Inst, we
815 if (V.Count != NOMORE_ICP_MAGICNUM) in doesHistoryAllowICP()
818 // If the promotion candidate has NOMORE_ICP_MAGICNUM count in the in doesHistoryAllowICP()
835 /// we expect to use it to update the total count in the value profile.
847 // OldSum is the existing total count in the value profile data. in updateIDTMetaData()
855 CallTargets[0].Count == NOMORE_ICP_MAGICNUM) && in updateIDTMetaData()
857 "with count being NOMORE_ICP_MAGICNUM"); in updateIDTMetaData()
860 ValueCountMap[V.Value] = V.Count; in updateIDTMetaData()
862 ValueCountMap.try_emplace(CallTargets[0].Value, CallTargets[0].Count); in updateIDTMetaData()
864 // count OldSum and reset the target's count to NOMORE_ICP_MAGICNUM. in updateIDTMetaData()
866 OldSum -= Pair.first->second; in updateIDTMetaData()
867 Pair.first->second = NOMORE_ICP_MAGICNUM; in updateIDTMetaData()
874 if (V.Count == NOMORE_ICP_MAGICNUM) in updateIDTMetaData()
875 ValueCountMap[V.Value] = V.Count; in updateIDTMetaData()
879 auto Pair = ValueCountMap.try_emplace(Data.Value, Data.Count); in updateIDTMetaData()
883 // Keep the count as NOMORE_ICP_MAGICNUM in the profile and decrease in updateIDTMetaData()
884 // Sum by Data.Count. in updateIDTMetaData()
885 assert(Sum >= Data.Count && "Sum should never be less than Data.Count"); in updateIDTMetaData()
886 Sum -= Data.Count; in updateIDTMetaData()
898 if (L.Count != R.Count) in updateIDTMetaData()
899 return L.Count > R.Count; in updateIDTMetaData()
905 annotateValueSite(*Inst.getParent()->getParent()->getParent(), Inst, in updateIDTMetaData()
922 // Bail out early if sample-loader inliner is disabled. in tryPromoteAndInlineCandidate()
930 auto CalleeFunctionName = Candidate.CalleeSamples->getFunction(); in tryPromoteAndInlineCandidate()
932 if (R == SymbolMap.end() || !R->second) in tryPromoteAndInlineCandidate()
936 if (!doesHistoryAllowICP(CI, R->second->getName())) in tryPromoteAndInlineCandidate()
940 // R->getValue() != &F is to prevent promoting a recursive call. in tryPromoteAndInlineCandidate()
946 if (!R->second->isDeclaration() && R->second->getSubprogram() && in tryPromoteAndInlineCandidate()
947 R->second->hasFnAttribute("use-sample-profile") && in tryPromoteAndInlineCandidate()
948 R->second != &F && isLegalToPromote(CI, R->second, &Reason)) { in tryPromoteAndInlineCandidate()
949 // For promoted target, set its value with NOMORE_ICP_MAGICNUM count in tryPromoteAndInlineCandidate()
952 Function::getGUID(R->second->getName()), NOMORE_ICP_MAGICNUM}}; in tryPromoteAndInlineCandidate()
956 CI, R->second, Candidate.CallsiteCount, Sum, false, ORE); in tryPromoteAndInlineCandidate()
958 Sum -= Candidate.CallsiteCount; in tryPromoteAndInlineCandidate()
960 // distribution will be used to scale down non-promoted profile target in tryPromoteAndInlineCandidate()
961 // counts later. By doing this we lose track of the real callsite count in tryPromoteAndInlineCandidate()
986 Candidate.CallInstr->getName())<< " because " in tryPromoteAndInlineCandidate()
1016 Function *CalledFunction = I->getCalledFunction(); in emitOptimizationRemarksForInlineCandidates()
1018 ORE->emit(OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(), in emitOptimizationRemarksForInlineCandidates()
1019 "InlineAttempt", I->getDebugLoc(), in emitOptimizationRemarksForInlineCandidates()
1020 I->getParent()) in emitOptimizationRemarksForInlineCandidates()
1031 DenseSet<GlobalValue::GUID> &InlinedGUIDs, uint64_t Threshold) { in findExternalInlineCandidate() argument
1040 Function::getGUID(CB->getCalledFunction()->getName())); in findExternalInlineCandidate()
1043 // Otherwise, drop the threshold to import everything that we can in findExternalInlineCandidate()
1044 Threshold = 0; in findExternalInlineCandidate()
1058 // Set threshold to zero to honor pre-inliner decision. in findExternalInlineCandidate()
1060 Threshold = 0; in findExternalInlineCandidate()
1061 Samples->findInlinedFunctions(InlinedGUIDs, SymbolMap, Threshold); in findExternalInlineCandidate()
1065 ContextTrieNode *Caller = ContextTracker->getContextNodeForProfile(Samples); in findExternalInlineCandidate()
1071 FunctionSamples *CalleeSample = Node->getFunctionSamples(); in findExternalInlineCandidate()
1078 // If pre-inliner decision is used, honor that for importing as well. in findExternalInlineCandidate()
1081 CalleeSample->getContext().hasAttribute(ContextShouldBeInlined); in findExternalInlineCandidate()
1082 if (!PreInline && CalleeSample->getHeadSamplesEstimate() < Threshold) in findExternalInlineCandidate()
1085 Function *Func = SymbolMap.lookup(CalleeSample->getFunction()); in findExternalInlineCandidate()
1087 if (!Func || Func->isDeclaration()) in findExternalInlineCandidate()
1088 InlinedGUIDs.insert(CalleeSample->getGUID()); in findExternalInlineCandidate()
1092 for (const auto &BS : CalleeSample->getBodySamples()) in findExternalInlineCandidate()
1094 if (TS.second > Threshold) { in findExternalInlineCandidate()
1096 if (!Callee || Callee->isDeclaration()) in findExternalInlineCandidate()
1103 // entry count and call target count to determine importing. in findExternalInlineCandidate()
1104 for (auto &Child : Node->getAllChildContext()) { in findExternalInlineCandidate()
1117 /// - If it is hot enough, inline the callsites and adds callsites of the callee
1121 /// - If a callsite is not inlined, merge the its profile to the outline
1122 /// version (if --sample-profile-merge-inlinee is true), or scale the
1124 /// instances (if --sample-profile-merge-inlinee is false).
1136 // Profile symbol list is ignored when profile-sample-accurate is on. in inlineHotFunctions()
1139 !F.hasFnAttribute("profile-sample-accurate"))) && in inlineHotFunctions()
1140 "ProfAccForSymsInList should be false when profile-sample-accurate " in inlineHotFunctions()
1158 assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && in inlineHotFunctions()
1161 if (FS->getHeadSamplesEstimate() > 0 || in inlineHotFunctions()
1183 Function *CalledFunction = I->getCalledFunction(); in inlineHotFunctions()
1185 0 /* dummy count */, in inlineHotFunctions()
1190 if (I->isIndirectCall()) { in inlineHotFunctions()
1196 PSI->getOrCompHotCountThreshold()); in inlineHotFunctions()
1202 Candidate = {I, FS, FS->getHeadSamplesEstimate(), 1.0}; in inlineHotFunctions()
1208 } else if (CalledFunction && CalledFunction->getSubprogram() && in inlineHotFunctions()
1209 !CalledFunction->isDeclaration()) { in inlineHotFunctions()
1217 PSI->getOrCompHotCountThreshold()); in inlineHotFunctions()
1233 // --disable-sample-loader-inlining is true. in tryInlineCandidate()
1245 ORE->emit(OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(), in tryInlineCandidate()
1262 emitInlinedIntoBasedOnCost(*ORE, DLoc, BB, *CalledFunction, *BB->getParent(), in tryInlineCandidate()
1267 InlinedCallSites->clear(); in tryInlineCandidate()
1269 InlinedCallSites->push_back(I); in tryInlineCandidate()
1273 ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples); in tryInlineCandidate()
1286 setProbeDistributionFactor(*I, Probe->Factor * in tryInlineCandidate()
1297 assert(CB && "Expect non-null call instruction"); in getInlineCandidate()
1311 Factor = Probe->Factor; in getInlineCandidate()
1314 CalleeSamples ? CalleeSamples->getHeadSamplesEstimate() * Factor : 0; in getInlineCandidate()
1323 Advice = ExternalInlineAdvisor->getAdvice(CB); in getExternalInlineAdvisorCost()
1325 if (!Advice->isInliningRecommended()) { in getExternalInlineAdvisorCost()
1326 Advice->recordUnattemptedInlining(); in getExternalInlineAdvisorCost()
1329 Advice->recordInlining(); in getExternalInlineAdvisorCost()
1347 // Adjust threshold based on call site hotness, only do this for callsite in shouldInlineCandidate()
1348 // prioritized inliner because otherwise cost-benefit check is done earlier. in shouldInlineCandidate()
1351 if (Candidate.CallsiteCount > PSI->getHotCountThreshold()) in shouldInlineCandidate()
1357 Function *Callee = Candidate.CallInstr->getCalledFunction(); in shouldInlineCandidate()
1361 // We will ignore the threshold from inline cost, so always get full cost. in shouldInlineCandidate()
1367 // when cost exceeds threshold without checking all IRs in the callee. in shouldInlineCandidate()
1377 // With CSSPGO, the preinliner in llvm-profgen can estimate global inline in shouldInlineCandidate()
1381 // aiming at better context-sensitive post-inline profile quality, assuming in shouldInlineCandidate()
1383 // we replay that inline decision under `sample-profile-use-preinliner`. in shouldInlineCandidate()
1388 // so the original context-sensitive preinliner decision should be ignored in shouldInlineCandidate()
1390 SampleContext &Context = Candidate.CalleeSamples->getContext(); in shouldInlineCandidate()
1396 // For old FDO inliner, we inline the call site if it is below hot threshold, in shouldInlineCandidate()
1403 // Otherwise only use the cost from call analyzer, but overwite threshold with in shouldInlineCandidate()
1404 // Sample PGO threshold. in shouldInlineCandidate()
1411 // Profile symbol list is ignored when profile-sample-accurate is on. in inlineHotFunctionsWithPriority()
1414 !F.hasFnAttribute("profile-sample-accurate"))) && in inlineHotFunctionsWithPriority()
1415 "ProfAccForSymsInList should be false when profile-sample-accurate " in inlineHotFunctionsWithPriority()
1434 // because with top-down inlining, we can grow inliner size significantly in inlineHotFunctionsWithPriority()
1453 Function *CalledFunction = I->getCalledFunction(); in inlineHotFunctionsWithPriority()
1457 if (I->isIndirectCall()) { in inlineHotFunctionsWithPriority()
1464 // TODO: Consider disable pre-lTO ICP for MonoLTO as well in inlineHotFunctionsWithPriority()
1467 PSI->getOrCompHotCountThreshold()); in inlineHotFunctionsWithPriority()
1471 FS->getHeadSamplesEstimate() * Candidate.CallsiteDistribution; in inlineHotFunctionsWithPriority()
1486 if (!PSI->isHotCount(EntryCountDistributed)) in inlineHotFunctionsWithPriority()
1490 // call site count for the promoted inline candidate too. in inlineHotFunctionsWithPriority()
1505 } else if (CalledFunction && CalledFunction->getSubprogram() && in inlineHotFunctionsWithPriority()
1506 !CalledFunction->isDeclaration()) { in inlineHotFunctionsWithPriority()
1520 PSI->getOrCompHotCountThreshold()); in inlineHotFunctionsWithPriority()
1546 Function *Callee = I->getCalledFunction(); in promoteMergeNotInlinedContextSamples()
1547 if (!Callee || Callee->isDeclaration()) in promoteMergeNotInlinedContextSamples()
1550 ORE->emit( in promoteMergeNotInlinedContextSamples()
1552 I->getDebugLoc(), I->getParent()) in promoteMergeNotInlinedContextSamples()
1558 if (FS->getTotalSamples() == 0 && FS->getHeadSamplesEstimate() == 0) { in promoteMergeNotInlinedContextSamples()
1563 if (FS->getContext().hasAttribute(sampleprof::ContextDuplicatedIntoBase)) in promoteMergeNotInlinedContextSamples()
1571 // callee profiles with a non-zero head sample count. in promoteMergeNotInlinedContextSamples()
1572 if (FS->getHeadSamples() == 0) { in promoteMergeNotInlinedContextSamples()
1575 const_cast<FunctionSamples *>(FS)->addHeadSamples( in promoteMergeNotInlinedContextSamples()
1576 FS->getHeadSamplesEstimate()); in promoteMergeNotInlinedContextSamples()
1580 // top-down processing of functions' annotation. in promoteMergeNotInlinedContextSamples()
1581 FunctionSamples *OutlineFS = Reader->getSamplesFor(*Callee); in promoteMergeNotInlinedContextSamples()
1586 FunctionId(FunctionSamples::getCanonicalFnName(Callee->getName()))]; in promoteMergeNotInlinedContextSamples()
1587 OutlineFS->merge(*FS, 1); in promoteMergeNotInlinedContextSamples()
1589 OutlineFS->setContextSynthetic(); in promoteMergeNotInlinedContextSamples()
1594 pair.first->second.entryCount += FS->getHeadSamplesEstimate(); in promoteMergeNotInlinedContextSamples()
1599 /// Returns the sorted CallTargetMap \p M by count in descending order.
1635 FS->findCallTargetMapAt(CallSite); in generateMDProfMetadata()
1639 // Prorate the callsite counts based on the pre-ICP distribution in generateMDProfMetadata()
1643 if (Probe->Factor < 1) in generateMDProfMetadata()
1644 T = SampleRecord::adjustCallTargets(T.get(), Probe->Factor); in generateMDProfMetadata()
1654 // inlined and non-inlined targets. in generateMDProfMetadata()
1657 FS->findFunctionSamplesMapAt(CallSite)) { in generateMDProfMetadata()
1685 Instruction *TI = BB->getTerminator(); in generateMDProfMetadata()
1686 if (TI->getNumSuccessors() == 1) in generateMDProfMetadata()
1692 DebugLoc BranchLoc = TI->getDebugLoc(); in generateMDProfMetadata()
1706 EdgeIndex.resize(TI->getNumSuccessors()); in generateMDProfMetadata()
1707 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) { in generateMDProfMetadata()
1708 const BasicBlock *Succ = TI->getSuccessor(I); in generateMDProfMetadata()
1713 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) { in generateMDProfMetadata()
1714 BasicBlock *Succ = TI->getSuccessor(I); in generateMDProfMetadata()
1719 // if needed. Sample counts in profiles are 64-bit unsigned values, in generateMDProfMetadata()
1720 // but internally branch weights are expressed as 32-bit values. in generateMDProfMetadata()
1743 MaxDestInst = Succ->getFirstNonPHIOrDbgOrLifetime(); in generateMDProfMetadata()
1751 // Only set weights if there is at least one non-zero weight. in generateMDProfMetadata()
1760 (!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) { in generateMDProfMetadata()
1761 LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n"); in generateMDProfMetadata()
1763 ORE->emit([&]() { in generateMDProfMetadata()
1770 TI->setMetadata(LLVMContext::MD_prof, nullptr); in generateMDProfMetadata()
1790 if (!ProbeManager->getDesc(F)) in emitAnnotations()
1795 if (ProbeManager->profileIsValid(F, *Samples)) { in emitAnnotations()
1834 ProfiledCG = std::make_unique<ProfiledCallGraph>(Reader->getProfiles()); in buildProfiledCallGraph()
1842 ProfiledCG->addProfiledFunction( in buildProfiledCallGraph()
1855 errs() << "WARNING: -use-profiled-call-graph ignored, should be used " in buildFunctionOrder()
1856 "together with -sample-profile-top-down-load.\n"; in buildFunctionOrder()
1863 // non-inlined inline instances, and that is not the way how in buildFunctionOrder()
1876 // Use profiled call edges to augment the top-down order. There are cases in buildFunctionOrder()
1877 // that the top-down order computed based on the static call graph doesn't in buildFunctionOrder()
1885 // an SCC may not reflect the call contexts in the context-sensitive in buildFunctionOrder()
1887 // function order in one SCC is being adjusted to a top-down order based in buildFunctionOrder()
1893 // the caller A. If any transferred edge (say A->C) is indirect, the in buildFunctionOrder()
1894 // original profiled indirect edge B->C, even if considered, would not in buildFunctionOrder()
1895 // enforce a top-down order from the caller A to the potential indirect in buildFunctionOrder()
1904 // to-be-dropped A, will have no profile to consume when its outlined in buildFunctionOrder()
1905 // version is compiled. This can lead to a profile-less prelink in buildFunctionOrder()
1911 // to enforce a top-down order for the rest of the functions. in buildFunctionOrder()
1919 // an SCC order incompatible with profile-defined one. Using strictly in buildFunctionOrder()
1934 Function *F = SymbolMap.lookup(Node->Name); in buildFunctionOrder()
1947 dbgs() << F->getName() << "\n"; in buildFunctionOrder()
1966 Reader->setSkipFlatProf(LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink); in doInitialization()
1969 Reader->setModule(&M); in doInitialization()
1970 if (std::error_code EC = Reader->read()) { in doInitialization()
1976 PSL = Reader->getProfileSymbolList(); in doInitialization()
1978 // While profile-sample-accurate is on, ignore symbol list. in doInitialization()
1984 if (auto NameTable = Reader->getNameTable()) { in doInitialization()
2006 // Apply tweaks if context-sensitive or probe-based profile is available. in doInitialization()
2007 if (Reader->profileIsCS() || Reader->profileIsPreInlined() || in doInitialization()
2008 Reader->profileIsProbeBased()) { in doInitialization()
2015 // Enable priority-base inliner and size inline by default for CSSPGO. in doInitialization()
2024 if (Reader->profileIsPreInlined()) { in doInitialization()
2029 // Enable stale profile matching by default for probe-based profile. in doInitialization()
2031 // which is currently only available for pseudo-probe mode. Removing the in doInitialization()
2034 if (Reader->profileIsProbeBased() && in doInitialization()
2039 if (!Reader->profileIsCS()) { in doInitialization()
2040 // Non-CS profile should be fine without a function size budget for the in doInitialization()
2042 // in the prevoius build or pre-computed by the preinliner with a size in doInitialization()
2051 if (Reader->profileIsCS()) { in doInitialization()
2054 Reader->getProfiles(), &GUIDToFuncNameMap); in doInitialization()
2057 // Load pseudo probe descriptors for probe-based function samples. in doInitialization()
2058 if (Reader->profileIsProbeBased()) { in doInitialization()
2060 if (!ProbeManager->moduleIsProbed(M)) { in doInitialization()
2062 "Pseudo-probe-based profile requires SampleProfileProbePass"; in doInitialization()
2079 // Note that this is a module-level check. Even if one module is errored out,
2086 // 1) The function is hot enough, which is tuned by a hotness-based
2092 "Only support for probe-based profile"); in rejectHighStalenessProfile()
2097 const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID()); in rejectHighStalenessProfile()
2101 // Use a hotness-based threshold to control the function selection. in rejectHighStalenessProfile()
2102 if (!PSI->isHotCountNthPercentile(HotFuncCutoffForStalenessError, in rejectHighStalenessProfile()
2107 if (ProbeManager->profileIsHashMismatched(*FuncDesc, FS)) in rejectHighStalenessProfile()
2115 // Finally check the mismatch percentage against the threshold. in rejectHighStalenessProfile()
2138 I->eraseFromParent(); in removePseudoProbeInsts()
2148 M.setProfileSummary(Reader->getSummary().getMD(M.getContext()), in runOnModule()
2150 PSI->refresh(); in runOnModule()
2154 rejectHighStalenessProfile(M, PSI, Reader->getProfiles())) in runOnModule()
2158 for (const auto &I : Reader->getProfiles()) in runOnModule()
2161 auto Remapper = Reader->getRemapper(); in runOnModule()
2177 r.first->second = nullptr; in runOnModule()
2182 if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) { in runOnModule()
2183 if (*MapName != OrigName && !MapName->empty()) in runOnModule()
2192 MatchingManager->runOnModule(); in runOnModule()
2193 MatchingManager->clearMatchingData(); in runOnModule()
2195 assert(SymbolMap.count(FunctionId()) == 0 && in runOnModule()
2198 "FuncNameToProfNameMap is not empty when --salvage-unused-profile is " in runOnModule()
2203 assert(!F->isDeclaration()); in runOnModule()
2223 // By default the entry count is initialized to -1, which will be treated in runOnFunction()
2227 uint64_t initialEntryCount = -1; in runOnFunction()
2230 if (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) { in runOnFunction()
2234 // profile-sample-accurate is a user assertion which has a higher precedence in runOnFunction()
2235 // than symbol list. When profile-sample-accurate is on, ignore symbol list. in runOnFunction()
2240 // PSL -- profile symbol list include all the symbols in sampled binary. in runOnFunction()
2245 // Initialize the entry count to 0 for functions in the list. in runOnFunction()
2246 if (PSL->contains(F.getName())) in runOnFunction()
2261 GUIDsInProfile.count(Function::getGUID(CanonName))) || in runOnFunction()
2262 (!FunctionSamples::UseMD5 && NamesInProfile.count(CanonName))) in runOnFunction()
2263 initialEntryCount = -1; in runOnFunction()
2266 // Initialize entry count when the function has no existing entry in runOnFunction()
2267 // count value. in runOnFunction()
2273 AM->getResult<FunctionAnalysisManagerModuleProxy>(*F.getParent()) in runOnFunction()
2282 Samples = ContextTracker->getBaseSamplesFor(F); in runOnFunction()
2284 Samples = Reader->getSamplesFor(F); in runOnFunction()
2291 Samples = &It->second; in runOnFunction()
2292 } else if (auto Remapper = Reader->getRemapper()) { in runOnFunction()
2293 if (auto RemppedName = Remapper->lookUpNameInProfile(CanonName)) { in runOnFunction()
2296 Samples = &It->second; in runOnFunction()
2302 if (Samples && !Samples->empty()) in runOnFunction()
2317 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { in run()
2320 auto GetTTI = [&](Function &F) -> TargetTransformInfo & { in run()
2323 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { in run()