Skip to content

Commit 866c421

Browse files
iohk-bors[bot]deepfire
iohk-bors[bot]
and
deepfire
authored
Merge #4208
4208: workbench: template-based benchmark reporting & fixes r=deepfire a=deepfire 1. Fix `aws-get` & speed up `fetch-analysis` 2. Templated benchmark reporting: `wb analyse compare RUN1 RUN...` Co-authored-by: Kosyrev Serge <[email protected]>
2 parents d6d2dc7 + eb4f35d commit 866c421

File tree

18 files changed

+516
-164
lines changed

18 files changed

+516
-164
lines changed

Diff for: ‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ ps: ## Plain-text list of profiles
6868
##
6969
PROFILES_BASE := default plutus oldtracing
7070
PROFILES_STARTSTOP := startstop startstop-p2p startstop-plutus startstop-notracer startstop-oldtracing
71-
PROFILES_CI_TEST := ci-test ci-test-p2p ci-test-plutus ci-test-notracer
71+
PROFILES_CI_TEST := ci-test ci-test-p2p ci-test-plutus ci-test-notracer ci-test-dense10
7272
PROFILES_CI_BENCH := ci-bench ci-bench-p2p ci-bench-plutus ci-bench-notracer
7373
PROFILES_10 := 10 10-p2p 10-plutus 10-notracer
7474
PROFILES_FORGE_STRESS := forge-stress forge-stress-p2p forge-stress-plutus forge-stress-plutus-singleton forge-stress-notracer

Diff for: ‎bench/locli/locli.cabal

+6
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ common base
4747

4848
build-depends: base
4949

50+
if os(windows)
51+
buildable: False
52+
5053
library
5154
import: base
5255
hs-source-dirs: src
@@ -56,6 +59,7 @@ library
5659
Data.DataDomain
5760

5861
Cardano.Command
62+
Cardano.Report
5963
Cardano.TopHandler
6064
Cardano.Util
6165

@@ -91,6 +95,7 @@ library
9195
, containers
9296
, deepseq
9397
, directory
98+
, ede
9499
, extra
95100
, filepath
96101
, file-embed
@@ -114,6 +119,7 @@ library
114119
, trace-resources
115120
, transformers
116121
, transformers-except
122+
, unix
117123
, unordered-containers
118124
, utf8-string
119125
, vector

Diff for: ‎bench/locli/src/Cardano/Analysis/API.hs

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ import Cardano.Util
3737
-- | Results of block propagation analysis.
3838
data BlockProp f
3939
= BlockProp
40-
{ bpVersion :: !Version
40+
{ bpVersion :: !Cardano.Analysis.Version.Version
4141
, bpDomainSlots :: !(DataDomain SlotNo)
4242
, bpDomainBlocks :: !(DataDomain BlockNo)
4343
, bpForgerChecks :: !(CDF f NominalDiffTime)
@@ -148,7 +148,7 @@ data BPErrorKind
148148
-- | The top-level representation of the machine timeline analysis results.
149149
data MachPerf f
150150
= MachPerf
151-
{ sVersion :: !Version
151+
{ sVersion :: !Cardano.Analysis.Version.Version
152152
, sDomainSlots :: !(DataDomain SlotNo)
153153
-- distributions
154154
, sMissCDF :: !(CDF f Double)

Diff for: ‎bench/locli/src/Cardano/Analysis/Context.hs

+67-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{-# LANGUAGE DeriveAnyClass #-}
2+
{-# LANGUAGE GeneralisedNewtypeDeriving #-}
23
{-# LANGUAGE StrictData #-}
34
module Cardano.Analysis.Context (module Cardano.Analysis.Context) where
45

56
import Cardano.Prelude
67

7-
import Data.Aeson (FromJSON, ToJSON)
8+
import Data.Aeson (FromJSON (..), ToJSON (..), withObject, object, (.:), (.:?), (.=))
9+
import Data.Text qualified as T
810
import Data.Time.Clock (UTCTime, NominalDiffTime)
911

1012

@@ -52,10 +54,74 @@ data GeneratorProfile
5254
}
5355
deriving (Generic, Show, FromJSON, ToJSON)
5456

57+
newtype Commit = Commit { unCommit :: Text } deriving newtype (Show, FromJSON, ToJSON)
58+
newtype Branch = Branch { unBranch :: Text } deriving newtype (Show, FromJSON, ToJSON)
59+
newtype Version = Version { unVersion :: Text } deriving newtype (Show, FromJSON, ToJSON)
60+
61+
unsafeShortenCommit :: Int -> Commit -> Commit
62+
unsafeShortenCommit n (Commit c) = Commit (T.take n c)
63+
64+
data Manifest
65+
= Manifest
66+
{ mNode :: !Commit
67+
, mNodeApproxVer :: !Version
68+
, mNodeBranch :: !Branch
69+
, mNodeStatus :: !Text
70+
, mNetwork :: !Commit
71+
, mLedger :: !Commit
72+
, mPlutus :: !Commit
73+
, mCrypto :: !Commit
74+
, mBase :: !Commit
75+
, mPrelude :: !Commit
76+
}
77+
deriving (Generic, Show)
78+
79+
unsafeShortenManifest :: Int -> Manifest -> Manifest
80+
unsafeShortenManifest n m@Manifest{..} =
81+
m { mNode = unsafeShortenCommit n mNode
82+
, mNetwork = unsafeShortenCommit n mNetwork
83+
, mLedger = unsafeShortenCommit n mLedger
84+
, mPlutus = unsafeShortenCommit n mPlutus
85+
, mCrypto = unsafeShortenCommit n mCrypto
86+
, mBase = unsafeShortenCommit n mBase
87+
, mPrelude = unsafeShortenCommit n mPrelude
88+
}
89+
90+
instance FromJSON Manifest where
91+
parseJSON = withObject "Manifest" $ \v -> do
92+
mNode <- v .: "cardano-node"
93+
mNodeBranch <- v .:? "cardano-node-branch" <&> fromMaybe (Branch "unknown")
94+
mNodeApproxVer <- v .:? "cardano-node-version" <&> fromMaybe (Version "unknown")
95+
mNodeStatus <- v .: "cardano-node-status"
96+
mNetwork <- v .: "ouroboros-network"
97+
mLedger <- v .: "cardano-ledger"
98+
mPlutus <- v .: "plutus"
99+
mCrypto <- v .: "cardano-crypto"
100+
mBase <- v .: "cardano-base"
101+
mPrelude <- v .: "cardano-prelude"
102+
pure Manifest{..}
103+
104+
instance ToJSON Manifest where
105+
toJSON Manifest{..} =
106+
object
107+
[ "cardano-node" .= mNode
108+
, "cardano-node-branch" .= mNodeBranch
109+
, "cardano-node-version" .= mNodeApproxVer
110+
, "cardano-node-status" .= mNodeStatus
111+
, "ouroboros-network" .= mNetwork
112+
, "cardano-ledger" .= mLedger
113+
, "plutus" .= mPlutus
114+
, "cardano-crypto" .= mCrypto
115+
, "cardano-base" .= mBase
116+
, "cardano-prelude" .= mPrelude
117+
]
118+
55119
data Metadata
56120
= Metadata
57121
{ tag :: Text
122+
, batch :: Text
58123
, profile :: Text
59124
, era :: Text
125+
, manifest :: Manifest
60126
}
61127
deriving (Generic, Show, FromJSON, ToJSON)

Diff for: ‎bench/locli/src/Cardano/Analysis/Ground.hs

+23-115
Original file line numberDiff line numberDiff line change
@@ -82,28 +82,21 @@ data HostDeduction
8282
---
8383
--- Files
8484
---
85-
newtype JsonGenesisFile
86-
= JsonGenesisFile { unJsonGenesisFile :: FilePath }
87-
deriving (Show, Eq)
88-
89-
newtype JsonRunMetafile
90-
= JsonRunMetafile { unJsonRunMetafile :: FilePath }
91-
deriving (Show, Eq)
92-
93-
newtype JsonDomainFile
94-
= JsonDomainFile { unJsonDomainFile :: FilePath }
85+
newtype InputDir
86+
= InputDir { unInputDir :: FilePath }
9587
deriving (Show, Eq)
88+
deriving newtype (NFData)
9689

9790
newtype JsonLogfile
9891
= JsonLogfile { unJsonLogfile :: FilePath }
9992
deriving (Show, Eq)
10093
deriving newtype (NFData)
10194

102-
newtype JsonInputFile
95+
newtype JsonInputFile (a :: Type)
10396
= JsonInputFile { unJsonInputFile :: FilePath }
10497
deriving (Show, Eq)
10598

106-
newtype JsonOutputFile
99+
newtype JsonOutputFile (a :: Type)
107100
= JsonOutputFile { unJsonOutputFile :: FilePath }
108101
deriving (Show, Eq)
109102

@@ -115,6 +108,10 @@ newtype OrgOutputFile
115108
= OrgOutputFile { unOrgOutputFile :: FilePath }
116109
deriving (Show, Eq)
117110

111+
newtype TextInputFile
112+
= TextInputFile { unTextInputFile :: FilePath }
113+
deriving (Show, Eq)
114+
118115
newtype TextOutputFile
119116
= TextOutputFile { unTextOutputFile :: FilePath }
120117
deriving (Show, Eq)
@@ -127,59 +124,15 @@ newtype OutputFile
127124
= OutputFile { unOutputFile :: FilePath }
128125
deriving (Show, Eq)
129126

130-
data MachineTimelineOutputFiles
131-
= MachineTimelineOutputFiles
132-
{ mtofSlotStats :: Maybe JsonOutputFile
133-
, mtofAnalysis :: Maybe JsonOutputFile
134-
, mtofFullStatsPretty :: Maybe TextOutputFile
135-
, mtofReportStatsPretty :: Maybe TextOutputFile
136-
, mtofTimelinePretty :: Maybe TextOutputFile
137-
, mtofTimelineCsv :: Maybe CsvOutputFile
138-
, mtofFullStatsCsv :: Maybe CsvOutputFile
139-
, mtofDerivedVectors0Csv :: Maybe CsvOutputFile
140-
, mtofDerivedVectors1Csv :: Maybe CsvOutputFile
141-
}
142-
deriving (Show)
143-
144-
data BlockPropOutputFiles
145-
= BlockPropOutputFiles
146-
{ bpofForgerPretty :: Maybe TextOutputFile
147-
, bpofPeersPretty :: Maybe TextOutputFile
148-
, bpofPropagationPretty :: Maybe TextOutputFile
149-
, bpofFullStatsPretty :: Maybe TextOutputFile
150-
, bpofMachViews :: Maybe JsonOutputFile
151-
, bpofChainPretty :: Maybe TextOutputFile
152-
, bpofChain :: Maybe JsonOutputFile
153-
, bpofChainRaw :: Maybe JsonOutputFile
154-
, bpofAnalysis :: Maybe JsonOutputFile
155-
}
156-
deriving (Show)
157-
158127
---
159128
--- Parsers
160129
---
161-
optJsonGenesisFile :: String -> String -> Parser JsonGenesisFile
162-
optJsonGenesisFile optname desc =
163-
fmap JsonGenesisFile $
130+
optInputDir :: String -> String -> Parser InputDir
131+
optInputDir optname desc =
132+
fmap InputDir $
164133
Opt.option Opt.str
165134
$ long optname
166-
<> metavar "GENESIS-FILE"
167-
<> help desc
168-
169-
optJsonRunMetafile :: String -> String -> Parser JsonRunMetafile
170-
optJsonRunMetafile optname desc =
171-
fmap JsonRunMetafile $
172-
Opt.option Opt.str
173-
$ long optname
174-
<> metavar "RUN-METAFILE"
175-
<> help desc
176-
177-
optJsonDomainFile :: String -> String -> Parser JsonDomainFile
178-
optJsonDomainFile optname desc =
179-
fmap JsonDomainFile $
180-
Opt.option Opt.str
181-
$ long optname
182-
<> metavar "DOMAINFILE"
135+
<> metavar "DIR"
183136
<> help desc
184137

185138
optJsonLogfile :: String -> String -> Parser JsonLogfile
@@ -195,15 +148,15 @@ argJsonLogfile =
195148
JsonLogfile <$>
196149
Opt.argument Opt.str (Opt.metavar "LOGFILE")
197150

198-
optJsonInputFile :: String -> String -> Parser JsonInputFile
151+
optJsonInputFile :: String -> String -> Parser (JsonInputFile a)
199152
optJsonInputFile optname desc =
200153
fmap JsonInputFile $
201154
Opt.option Opt.str
202155
$ long optname
203156
<> metavar "JSON-FILE"
204157
<> help desc
205158

206-
optJsonOutputFile :: String -> String -> Parser JsonOutputFile
159+
optJsonOutputFile :: String -> String -> Parser (JsonOutputFile a)
207160
optJsonOutputFile optname desc =
208161
fmap JsonOutputFile $
209162
Opt.option Opt.str
@@ -219,6 +172,14 @@ optGnuplotOutputFile optname desc =
219172
<> metavar "CDF-OUTFILE"
220173
<> help desc
221174

175+
optTextInputFile :: String -> String -> Parser TextInputFile
176+
optTextInputFile optname desc =
177+
fmap TextInputFile $
178+
Opt.option Opt.str
179+
$ long optname
180+
<> metavar "TEXT-INFILE"
181+
<> help desc
182+
222183
optTextOutputFile :: String -> String -> Parser TextOutputFile
223184
optTextOutputFile optname desc =
224185
fmap TextOutputFile $
@@ -279,56 +240,3 @@ optWord optname desc def =
279240
<> metavar "INT"
280241
<> help desc
281242
<> value def
282-
283-
parseMachineTimelineOutputFiles :: Parser MachineTimelineOutputFiles
284-
parseMachineTimelineOutputFiles =
285-
MachineTimelineOutputFiles
286-
<$> optional
287-
(optJsonOutputFile "slotstats-json"
288-
"Per-slot performance summaries")
289-
<*> optional
290-
(optJsonOutputFile "analysis-json"
291-
"Write analysis JSON to this file, if specified -- otherwise print to stdout.")
292-
<*> optional
293-
(optTextOutputFile "fullstats-text"
294-
"Full performance statistics breakdown")
295-
<*> optional
296-
(optTextOutputFile "reportstats-text"
297-
"Report performance statistics breakdown")
298-
<*> optional
299-
(optTextOutputFile "timeline-text"
300-
"Dump pretty timeline of extracted slot leadership summaries, as a side-effect of log analysis")
301-
<*> optional
302-
(optCsvOutputFile "timeline-csv"
303-
"Dump CSV of the timeline")
304-
<*> optional
305-
(optCsvOutputFile "stats-csv"
306-
"Dump CSV of the timeline statistics")
307-
<*> optional
308-
(optCsvOutputFile "derived-vectors-0-csv"
309-
"Dump CSV of vectors derived from the timeline")
310-
<*> optional
311-
(optCsvOutputFile "derived-vectors-1-csv"
312-
"Dump CSV of vectors derived from the timeline")
313-
314-
parseBlockPropOutputFiles :: Parser BlockPropOutputFiles
315-
parseBlockPropOutputFiles =
316-
BlockPropOutputFiles
317-
<$> optional
318-
(optTextOutputFile "forger-text" "Forger stats")
319-
<*> optional
320-
(optTextOutputFile "peers-text" "Peers stats")
321-
<*> optional
322-
(optTextOutputFile "propagation-text" "Propagation stats")
323-
<*> optional
324-
(optTextOutputFile "fullstats-text" "Full (forger+peers+propagation) stats")
325-
<*> optional
326-
(optJsonOutputFile "mach-views-json" "Machine chain views as JSON")
327-
<*> optional
328-
(optTextOutputFile "chain-text" "Timeline of chain evolution, one line per block")
329-
<*> optional
330-
(optJsonOutputFile "chain-raw-json" "Unfiltered chain as JSON")
331-
<*> optional
332-
(optJsonOutputFile "chain-json" "Chain as JSON")
333-
<*> optional
334-
(optJsonOutputFile "analysis-json" "Analysis as JSON")

0 commit comments

Comments
 (0)