Skip to content

Commit 66fe5d5

Browse files
aaupovaaupov
committed
[BOLT] Add pseudo probe inline tree to YAML profile
To be used for pseudo probe function matching (#100446). Test Plan: updated pseudoprobe-decoding-inline.test Pull Request: #107137
1 parent e0a705e commit 66fe5d5

File tree

4 files changed

+153
-39
lines changed

4 files changed

+153
-39
lines changed

‎bolt/include/bolt/Profile/ProfileYAMLMapping.h

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -95,24 +95,28 @@ template <> struct MappingTraits<bolt::SuccessorInfo> {
9595

9696
namespace bolt {
9797
struct PseudoProbeInfo {
98-
llvm::yaml::Hex64 GUID;
9998
uint64_t Index;
99+
uint32_t InlineTreeIndex;
100+
llvm::yaml::Hex32 Offset{0};
100101
uint8_t Type;
101102

102103
bool operator==(const PseudoProbeInfo &Other) const {
103-
return GUID == Other.GUID && Index == Other.Index;
104+
return InlineTreeIndex == Other.InlineTreeIndex && Index == Other.Index;
104105
}
105-
bool operator!=(const PseudoProbeInfo &Other) const {
106-
return !(*this == Other);
106+
bool operator<(const PseudoProbeInfo &Other) const {
107+
if (InlineTreeIndex == Other.InlineTreeIndex)
108+
return Index < Other.Index;
109+
return InlineTreeIndex < Other.InlineTreeIndex;
107110
}
108111
};
109112
} // end namespace bolt
110113

111114
template <> struct MappingTraits<bolt::PseudoProbeInfo> {
112115
static void mapping(IO &YamlIO, bolt::PseudoProbeInfo &PI) {
113-
YamlIO.mapRequired("guid", PI.GUID);
114116
YamlIO.mapRequired("id", PI.Index);
115117
YamlIO.mapRequired("type", PI.Type);
118+
YamlIO.mapOptional("inline_tree_id", PI.InlineTreeIndex, (uint32_t)0);
119+
YamlIO.mapOptional("offset", PI.Offset, (uint32_t)0);
116120
}
117121

118122
static const bool flow = true;
@@ -122,7 +126,7 @@ template <> struct MappingTraits<bolt::PseudoProbeInfo> {
122126

123127
LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::CallSiteInfo)
124128
LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::SuccessorInfo)
125-
LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::PseudoProbeInfo)
129+
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::bolt::PseudoProbeInfo)
126130

127131
namespace llvm {
128132
namespace yaml {
@@ -163,10 +167,35 @@ template <> struct MappingTraits<bolt::BinaryBasicBlockProfile> {
163167
}
164168
};
165169

170+
namespace bolt {
171+
struct InlineTreeInfo {
172+
uint32_t Index;
173+
uint32_t ParentIndex;
174+
uint32_t CallSiteProbe;
175+
llvm::yaml::Hex64 GUID;
176+
llvm::yaml::Hex64 Hash;
177+
bool operator==(const InlineTreeInfo &Other) const {
178+
return Index == Other.Index;
179+
}
180+
};
181+
} // end namespace bolt
182+
183+
template <> struct MappingTraits<bolt::InlineTreeInfo> {
184+
static void mapping(IO &YamlIO, bolt::InlineTreeInfo &ITI) {
185+
YamlIO.mapRequired("guid", ITI.GUID);
186+
YamlIO.mapRequired("hash", ITI.Hash);
187+
YamlIO.mapRequired("id", ITI.Index);
188+
YamlIO.mapOptional("parent", ITI.ParentIndex, (uint32_t)0);
189+
YamlIO.mapOptional("callsite", ITI.CallSiteProbe, 0);
190+
}
191+
192+
static const bool flow = true;
193+
};
166194
} // end namespace yaml
167195
} // end namespace llvm
168196

169197
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::bolt::BinaryBasicBlockProfile)
198+
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::bolt::InlineTreeInfo)
170199

171200
namespace llvm {
172201
namespace yaml {
@@ -179,8 +208,7 @@ struct BinaryFunctionProfile {
179208
llvm::yaml::Hex64 Hash{0};
180209
uint64_t ExecCount{0};
181210
std::vector<BinaryBasicBlockProfile> Blocks;
182-
llvm::yaml::Hex64 GUID{0};
183-
llvm::yaml::Hex64 PseudoProbeDescHash{0};
211+
std::vector<InlineTreeInfo> InlineTree;
184212
bool Used{false};
185213
};
186214
} // end namespace bolt
@@ -194,9 +222,8 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
194222
YamlIO.mapRequired("nblocks", BFP.NumBasicBlocks);
195223
YamlIO.mapOptional("blocks", BFP.Blocks,
196224
std::vector<bolt::BinaryBasicBlockProfile>());
197-
YamlIO.mapOptional("guid", BFP.GUID, (uint64_t)0);
198-
YamlIO.mapOptional("pseudo_probe_desc_hash", BFP.PseudoProbeDescHash,
199-
(uint64_t)0);
225+
YamlIO.mapOptional("inline_tree", BFP.InlineTree,
226+
std::vector<bolt::InlineTreeInfo>());
200227
}
201228
};
202229

‎bolt/lib/Profile/DataAggregator.cpp

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "llvm/Support/raw_ostream.h"
3535
#include <map>
3636
#include <optional>
37+
#include <queue>
3738
#include <unordered_map>
3839
#include <utility>
3940

@@ -2402,12 +2403,43 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
24022403
const unsigned BlockIndex = BlockMap.getBBIndex(BI.To.Offset);
24032404
YamlBF.Blocks[BlockIndex].ExecCount += BI.Branches;
24042405
}
2405-
if (PseudoProbeDecoder) {
2406-
if ((YamlBF.GUID = BF->getGUID())) {
2407-
const MCPseudoProbeFuncDesc *FuncDesc =
2408-
PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID);
2409-
YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash;
2406+
DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t>
2407+
InlineTreeNodeId;
2408+
if (PseudoProbeDecoder && BF->getGUID()) {
2409+
std::queue<const MCDecodedPseudoProbeInlineTree *> ITWorklist;
2410+
// FIXME: faster inline tree lookup by top-level GUID
2411+
if (const MCDecodedPseudoProbeInlineTree *InlineTree = llvm::find_if(
2412+
PseudoProbeDecoder->getDummyInlineRoot().getChildren(),
2413+
[&](const auto &InlineTree) {
2414+
return InlineTree.Guid == BF->getGUID();
2415+
})) {
2416+
ITWorklist.push(InlineTree);
2417+
InlineTreeNodeId[InlineTree] = 0;
2418+
auto Hash =
2419+
PseudoProbeDecoder->getFuncDescForGUID(BF->getGUID())->FuncHash;
2420+
YamlBF.InlineTree.emplace_back(
2421+
yaml::bolt::InlineTreeInfo{0, 0, 0, BF->getGUID(), Hash});
2422+
}
2423+
uint32_t ParentId = 0;
2424+
uint32_t NodeId = 1;
2425+
while (!ITWorklist.empty()) {
2426+
const MCDecodedPseudoProbeInlineTree *Cur = ITWorklist.front();
2427+
for (const MCDecodedPseudoProbeInlineTree &Child :
2428+
Cur->getChildren()) {
2429+
InlineTreeNodeId[&Child] = NodeId;
2430+
auto Hash =
2431+
PseudoProbeDecoder->getFuncDescForGUID(Child.Guid)->FuncHash;
2432+
YamlBF.InlineTree.emplace_back(yaml::bolt::InlineTreeInfo{
2433+
NodeId++, ParentId, std::get<1>(Child.getInlineSite()),
2434+
Child.Guid, Hash});
2435+
ITWorklist.push(&Child);
2436+
}
2437+
ITWorklist.pop();
2438+
++ParentId;
24102439
}
2440+
}
2441+
2442+
if (PseudoProbeDecoder) {
24112443
// Fetch probes belonging to all fragments
24122444
const AddressProbesMap &ProbeMap =
24132445
PseudoProbeDecoder->getAddress2ProbesMap();
@@ -2420,12 +2452,19 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
24202452
const uint32_t OutputAddress = Probe.getAddress();
24212453
const uint32_t InputOffset = BAT->translate(
24222454
FuncAddr, OutputAddress - FuncAddr, /*IsBranchSrc=*/true);
2423-
const unsigned BlockIndex = getBlock(InputOffset).second;
2455+
const auto [BlockOffset, BlockIndex] = getBlock(InputOffset);
2456+
uint32_t NodeId = InlineTreeNodeId[Probe.getInlineTreeNode()];
2457+
uint32_t Offset = InputOffset - BlockOffset;
24242458
YamlBF.Blocks[BlockIndex].PseudoProbes.emplace_back(
2425-
yaml::bolt::PseudoProbeInfo{Probe.getGuid(), Probe.getIndex(),
2459+
yaml::bolt::PseudoProbeInfo{Probe.getIndex(), NodeId, Offset,
24262460
Probe.getType()});
24272461
}
24282462
}
2463+
for (yaml::bolt::BinaryBasicBlockProfile &YamlBB : YamlBF.Blocks) {
2464+
llvm::sort(YamlBB.PseudoProbes);
2465+
YamlBB.PseudoProbes.erase(llvm::unique(YamlBB.PseudoProbes),
2466+
YamlBB.PseudoProbes.end());
2467+
}
24292468
}
24302469
// Drop blocks without a hash, won't be useful for stale matching.
24312470
llvm::erase_if(YamlBF.Blocks,

‎bolt/lib/Profile/YAMLProfileWriter.cpp

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,15 @@
1212
#include "bolt/Profile/BoltAddressTranslation.h"
1313
#include "bolt/Profile/DataAggregator.h"
1414
#include "bolt/Profile/ProfileReaderBase.h"
15+
#include "bolt/Profile/ProfileYAMLMapping.h"
1516
#include "bolt/Rewrite/RewriteInstance.h"
1617
#include "bolt/Utils/CommandLineOpts.h"
18+
#include "llvm/MC/MCPseudoProbe.h"
1719
#include "llvm/Support/CommandLine.h"
1820
#include "llvm/Support/FileSystem.h"
1921
#include "llvm/Support/raw_ostream.h"
22+
#include <deque>
23+
#include <queue>
2024

2125
#undef DEBUG_TYPE
2226
#define DEBUG_TYPE "bolt-prof"
@@ -77,13 +81,6 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
7781
YamlBF.Hash = BF.getHash();
7882
YamlBF.NumBasicBlocks = BF.size();
7983
YamlBF.ExecCount = BF.getKnownExecutionCount();
80-
if (PseudoProbeDecoder) {
81-
if ((YamlBF.GUID = BF.getGUID())) {
82-
const MCPseudoProbeFuncDesc *FuncDesc =
83-
PseudoProbeDecoder->getFuncDescForGUID(YamlBF.GUID);
84-
YamlBF.PseudoProbeDescHash = FuncDesc->FuncHash;
85-
}
86-
}
8784

8885
BinaryFunction::BasicBlockOrderType Order;
8986
llvm::copy(UseDFS ? BF.dfs() : BF.getLayout().blocks(),
@@ -92,6 +89,40 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
9289
const FunctionLayout Layout = BF.getLayout();
9390
Layout.updateLayoutIndices(Order);
9491

92+
DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t> InlineTreeNodeId;
93+
if (PseudoProbeDecoder && BF.getGUID()) {
94+
std::queue<const MCDecodedPseudoProbeInlineTree *> ITWorklist;
95+
// FIXME: faster inline tree lookup by top-level GUID
96+
if (const MCDecodedPseudoProbeInlineTree *InlineTree = llvm::find_if(
97+
PseudoProbeDecoder->getDummyInlineRoot().getChildren(),
98+
[&](const auto &InlineTree) {
99+
return InlineTree.Guid == BF.getGUID();
100+
})) {
101+
ITWorklist.push(InlineTree);
102+
InlineTreeNodeId[InlineTree] = 0;
103+
auto Hash =
104+
PseudoProbeDecoder->getFuncDescForGUID(BF.getGUID())->FuncHash;
105+
YamlBF.InlineTree.emplace_back(
106+
yaml::bolt::InlineTreeInfo{0, 0, 0, BF.getGUID(), Hash});
107+
}
108+
uint32_t ParentId = 0;
109+
uint32_t NodeId = 1;
110+
while (!ITWorklist.empty()) {
111+
const MCDecodedPseudoProbeInlineTree *Cur = ITWorklist.front();
112+
for (const MCDecodedPseudoProbeInlineTree &Child : Cur->getChildren()) {
113+
InlineTreeNodeId[&Child] = NodeId;
114+
auto Hash =
115+
PseudoProbeDecoder->getFuncDescForGUID(Child.Guid)->FuncHash;
116+
YamlBF.InlineTree.emplace_back(yaml::bolt::InlineTreeInfo{
117+
NodeId++, ParentId, std::get<1>(Child.getInlineSite()), Child.Guid,
118+
Hash});
119+
ITWorklist.push(&Child);
120+
}
121+
ITWorklist.pop();
122+
++ParentId;
123+
}
124+
}
125+
95126
for (const BinaryBasicBlock *BB : Order) {
96127
yaml::bolt::BinaryBasicBlockProfile YamlBB;
97128
YamlBB.Index = BB->getLayoutIndex();
@@ -198,10 +229,18 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS,
198229
const uint64_t FuncAddr = BF.getAddress();
199230
const std::pair<uint64_t, uint64_t> &BlockRange =
200231
BB->getInputAddressRange();
201-
for (const MCDecodedPseudoProbe &Probe : ProbeMap.find(
202-
FuncAddr + BlockRange.first, FuncAddr + BlockRange.second))
232+
const std::pair<uint64_t, uint64_t> BlockAddrRange = {
233+
FuncAddr + BlockRange.first, FuncAddr + BlockRange.second};
234+
for (const MCDecodedPseudoProbe &Probe :
235+
ProbeMap.find(BlockAddrRange.first, BlockAddrRange.second)) {
236+
uint32_t NodeId = InlineTreeNodeId[Probe.getInlineTreeNode()];
237+
uint32_t Offset = Probe.getAddress() - BlockAddrRange.first;
203238
YamlBB.PseudoProbes.emplace_back(yaml::bolt::PseudoProbeInfo{
204-
Probe.getGuid(), Probe.getIndex(), Probe.getType()});
239+
Probe.getIndex(), NodeId, Offset, Probe.getType()});
240+
}
241+
llvm::sort(YamlBB.PseudoProbes);
242+
YamlBB.PseudoProbes.erase(llvm::unique(YamlBB.PseudoProbes),
243+
YamlBB.PseudoProbes.end());
205244
}
206245

207246
YamlBF.Blocks.emplace_back(YamlBB);

‎bolt/test/X86/pseudoprobe-decoding-inline.test

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,29 +14,38 @@
1414
# RUN: FileCheck --input-file %t.yaml2 %s --check-prefix CHECK-YAML
1515
# CHECK-YAML: name: bar
1616
# CHECK-YAML: - bid: 0
17-
# CHECK-YAML: pseudo_probes: [ { guid: 0xE413754A191DB537, id: 1, type: 0 }, { guid: 0xE413754A191DB537, id: 4, type: 0 } ]
18-
# CHECK-YAML: guid: 0xE413754A191DB537
19-
# CHECK-YAML: pseudo_probe_desc_hash: 0x10E852DA94
17+
# CHECK-YAML: pseudo_probes:
18+
# CHECK-YAML-NEXT: - { id: 1, type: 0
19+
# CHECK-YAML-NEXT: - { id: 4, type: 0
20+
# CHECK-YAML: inline_tree:
21+
# CHECK-YAML-NEXT: - { guid: 0xE413754A191DB537, hash: 0x10E852DA94, id: 0 }
2022
#
2123
# CHECK-YAML: name: foo
2224
# CHECK-YAML: - bid: 0
23-
# CHECK-YAML: pseudo_probes: [ { guid: 0x5CF8C24CDB18BDAC, id: 1, type: 0 }, { guid: 0x5CF8C24CDB18BDAC, id: 2, type: 0 } ]
24-
# CHECK-YAML: guid: 0x5CF8C24CDB18BDAC
25-
# CHECK-YAML: pseudo_probe_desc_hash: 0x200205A19C5B4
25+
# CHECK-YAML: pseudo_probes:
26+
# CHECK-YAML-NEXT: - { id: 1, type: 0 }
27+
# CHECK-YAML-NEXT: - { id: 2, type: 0 }
28+
# CHECK-YAML: inline_tree:
29+
# CHECK-YAML-NEXT: - { guid: 0x5CF8C24CDB18BDAC, hash: 0x200205A19C5B4, id: 0 }
30+
# CHECK-YAML-NEXT: - { guid: 0xE413754A191DB537, hash: 0x10E852DA94, id: 1, callsite: 8 }
2631
#
2732
# CHECK-YAML: name: main
2833
# CHECK-YAML: - bid: 0
29-
# CHECK-YAML: pseudo_probes: [ { guid: 0xDB956436E78DD5FA, id: 1, type: 0 }, { guid: 0x5CF8C24CDB18BDAC, id: 1, type: 0 }, { guid: 0x5CF8C24CDB18BDAC, id: 2, type: 0 } ]
30-
# CHECK-YAML: guid: 0xDB956436E78DD5FA
31-
# CHECK-YAML: pseudo_probe_desc_hash: 0x10000FFFFFFFF
34+
# CHECK-YAML: pseudo_probes:
35+
# CHECK-YAML-NEXT: - { id: 1, type: 0 }
36+
# CHECK-YAML-NEXT: - { id: 1, type: 0, inline_tree_id: 1 }
37+
# CHECK-YAML-NEXT: - { id: 2, type: 0, inline_tree_id: 1 }
38+
# CHECK-YAML: inline_tree:
39+
# CHECK-YAML-NEXT: - { guid: 0xDB956436E78DD5FA, hash: 0x10000FFFFFFFF, id: 0 }
40+
# CHECK-YAML-NEXT: - { guid: 0x5CF8C24CDB18BDAC, hash: 0x200205A19C5B4, id: 1, callsite: 2 }
41+
# CHECK-YAML-NEXT: - { guid: 0xE413754A191DB537, hash: 0x10E852DA94, id: 2, parent: 1, callsite: 8 }
3242
#
3343
## Check that without --profile-write-pseudo-probes option, no pseudo probes are
3444
## generated
3545
# RUN: perf2bolt %S/../../../llvm/test/tools/llvm-profgen/Inputs/inline-cs-pseudoprobe.perfbin -p %t.preagg --pa -w %t.yaml -o %t.fdata
3646
# RUN: FileCheck --input-file %t.yaml %s --check-prefix CHECK-NO-OPT
3747
# CHECK-NO-OPT-NOT: pseudo_probes
38-
# CHECK-NO-OPT-NOT: guid
39-
# CHECK-NO-OPT-NOT: pseudo_probe_desc_hash
48+
# CHECK-NO-OPT-NOT: inline_tree
4049

4150
CHECK: Report of decoding input pseudo probe binaries
4251

0 commit comments

Comments
 (0)