Skip to content

Commit 9ec03aa

Browse files
Merge #4021
4021: Reduce memory usage of create staked command r=newhoggy a=newhoggy The problem arises we build a very large in-memory JSON value and then write it out as `genesis.json`. The amount of memory used by the in-memory JSON value can be so large as to use up all the memory. This PR reduces the severity of the memory usage in two ways. 1. It introduces a `ListMap` type, which is almost like `Map`, but has a `[(k, v)]` as its internal representation. This avoids the serialisation cost of constructing a map only to convert it back into a list. 2. Uses Lazy IO so that generated stuffed utxos are created on demand rather than upfront an all in memory. 3. Introduces a new `LazyToJson` type class which doesn't have the memory retention problems of `aeson` library. 4. For evaluation of fields to `WHNF` to allow parent object to be GCed which allows large fields that have already be serialised to be collected as well. 5. Writes delegations to a single `delegations.jsonl` file which is a newline delimited JSON file. This file is streamed multiple times so that generation of the `genesis.json` file does not retain memory unnecessarily. This PR also changes the command to no longer generate payment keys and stake keys. If we want to have the ability optionally output these files, there is additional work to do. Addresses #3938 Co-authored-by: John Ky <[email protected]>
2 parents 78a13b0 + cc24684 commit 9ec03aa

File tree

15 files changed

+359
-396
lines changed

15 files changed

+359
-396
lines changed

cabal.project

+6
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,12 @@ source-repository-package
330330
tag: ee59880f47ab835dbd73bea0847dab7869fc20d8
331331
--sha256: 1lrzknw765pz2j97nvv9ip3l1mcpf2zr4n56hwlz0rk7wq7ls4cm
332332

333+
source-repository-package
334+
type: git
335+
location: https://github.com/input-output-hk/aeson
336+
tag: be4774468e651d1d512edad278cca7276e978034
337+
--sha256: 12fr5xnr3ax0r5gzwbf4v49yirppgprmvzlfj1ldx4zhcrdf5j7j
338+
333339
constraints:
334340
hedgehog >= 1.0
335341
, bimap >= 0.4.0

cardano-api/cardano-api.cabal

+2-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,6 @@ library
117117
, cardano-protocol-tpraos
118118
, cardano-slotting
119119
, cborg
120-
, vector-map
121120
, contra-tracer
122121
, containers
123122
, cryptonite
@@ -141,6 +140,7 @@ library
141140
, plutus-ledger-api
142141
, prettyprinter
143142
, prettyprinter-configurable
143+
, random
144144
, scientific
145145
, serialise
146146
, small-steps
@@ -155,6 +155,7 @@ library
155155
, typed-protocols
156156
, unordered-containers >= 0.2.11
157157
, vector
158+
, vector-map
158159
, yaml
159160

160161
library gen

cardano-api/src/Cardano/Api.hs

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ module Cardano.Api (
4848
castVerificationKey,
4949
castSigningKey,
5050
generateSigningKey,
51+
generateInsecureSigningKey,
5152

5253
-- ** Hashes
5354
-- | In Cardano most keys are identified by their hash, and hashes are

cardano-api/src/Cardano/Api/Address.hs

+12
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ import Cardano.Api.Script
106106
import Cardano.Api.SerialiseBech32
107107
import Cardano.Api.SerialiseRaw
108108
import Cardano.Api.Utils
109+
import Control.DeepSeq (NFData(..), deepseq)
109110

110111

111112

@@ -192,6 +193,10 @@ deriving instance Eq (Address addrtype)
192193
deriving instance Ord (Address addrtype)
193194
deriving instance Show (Address addrtype)
194195

196+
instance NFData (Address addrtype) where
197+
rnf = \case
198+
ByronAddress address -> deepseq address ()
199+
ShelleyAddress n pc sr -> deepseq (deepseq (deepseq n pc) sr) ()
195200

196201
instance HasTypeProxy addrtype => HasTypeProxy (Address addrtype) where
197202
data AsType (Address addrtype) = AsAddress (AsType addrtype)
@@ -337,6 +342,9 @@ data AddressInEra era where
337342
-> Address addrtype
338343
-> AddressInEra era
339344

345+
instance NFData (AddressInEra era) where
346+
rnf (AddressInEra t a) = deepseq (deepseq t a) ()
347+
340348
instance IsCardanoEra era => ToJSON (AddressInEra era) where
341349
toJSON = Aeson.String . serialiseAddress
342350

@@ -387,6 +395,10 @@ data AddressTypeInEra addrtype era where
387395

388396
deriving instance Show (AddressTypeInEra addrtype era)
389397

398+
instance NFData (AddressTypeInEra addrtype era) where
399+
rnf = \case
400+
ByronAddressInAnyEra -> ()
401+
ShelleyAddressInEra sbe -> deepseq sbe ()
390402

391403
instance HasTypeProxy era => HasTypeProxy (AddressInEra era) where
392404
data AsType (AddressInEra era) = AsAddressInEra (AsType era)

cardano-api/src/Cardano/Api/Eras.hs

+9
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ import Ouroboros.Consensus.Shelley.Eras as Consensus (StandardAllegra,
5555
StandardBabbage, StandardMary, StandardShelley)
5656

5757
import Cardano.Api.HasTypeProxy
58+
import Control.DeepSeq (NFData(..))
5859

5960

6061
-- | A type used as a tag to distinguish the Byron era.
@@ -306,6 +307,14 @@ data ShelleyBasedEra era where
306307
ShelleyBasedEraAlonzo :: ShelleyBasedEra AlonzoEra
307308
ShelleyBasedEraBabbage :: ShelleyBasedEra BabbageEra
308309

310+
instance NFData (ShelleyBasedEra era) where
311+
rnf = \case
312+
ShelleyBasedEraShelley -> ()
313+
ShelleyBasedEraAllegra -> ()
314+
ShelleyBasedEraMary -> ()
315+
ShelleyBasedEraAlonzo -> ()
316+
ShelleyBasedEraBabbage -> ()
317+
309318
deriving instance Eq (ShelleyBasedEra era)
310319
deriving instance Ord (ShelleyBasedEra era)
311320
deriving instance Show (ShelleyBasedEra era)

cardano-api/src/Cardano/Api/Key.hs

+14
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
module Cardano.Api.Key
66
( Key(..)
77
, generateSigningKey
8+
, generateInsecureSigningKey
89
, CastVerificationKeyRole(..)
910
, CastSigningKeyRole(..)
1011
, AsType(AsVerificationKey, AsSigningKey)
@@ -21,7 +22,9 @@ import Cardano.Api.Hash
2122
import Cardano.Api.HasTypeProxy
2223
import Cardano.Api.SerialiseRaw
2324
import Cardano.Api.SerialiseTextEnvelope
25+
import System.Random (StdGen)
2426

27+
import qualified System.Random as Random
2528

2629
-- | An interface for cryptographic keys used for signatures with a 'SigningKey'
2730
-- and a 'VerificationKey' key.
@@ -67,6 +70,17 @@ generateSigningKey keytype = do
6770
seedSize = deterministicSigningKeySeedSize keytype
6871

6972

73+
generateInsecureSigningKey
74+
:: (Key keyrole, SerialiseAsRawBytes (SigningKey keyrole))
75+
=> StdGen
76+
-> AsType keyrole
77+
-> IO (SigningKey keyrole, StdGen)
78+
generateInsecureSigningKey g keytype = do
79+
let (bs, g') = Random.genByteString (fromIntegral $ deterministicSigningKeySeedSize keytype) g
80+
case deserialiseFromRawBytes (AsSigningKey keytype) bs of
81+
Just key -> return (key, g')
82+
Nothing -> error "generateInsecureSigningKey: Unable to generate insecure key"
83+
7084
instance HasTypeProxy a => HasTypeProxy (VerificationKey a) where
7185
data AsType (VerificationKey a) = AsVerificationKey (AsType a)
7286
proxyToAsType _ = AsVerificationKey (proxyToAsType (Proxy :: Proxy a))

cardano-api/src/Cardano/Api/KeysShelley.hs

+3-2
Original file line numberDiff line numberDiff line change
@@ -323,8 +323,9 @@ instance HasTypeProxy StakeKey where
323323

324324
instance Key StakeKey where
325325

326-
newtype VerificationKey StakeKey =
327-
StakeVerificationKey (Shelley.VKey Shelley.Staking StandardCrypto)
326+
newtype VerificationKey StakeKey = StakeVerificationKey
327+
{ unStakeVerificationKey :: Shelley.VKey Shelley.Staking StandardCrypto
328+
}
328329
deriving stock (Eq)
329330
deriving newtype (ToCBOR, FromCBOR)
330331
deriving anyclass SerialiseAsCBOR

cardano-api/src/Cardano/Api/TxBody.hs

+2-2
Original file line numberDiff line numberDiff line change
@@ -2680,8 +2680,8 @@ getByronTxBodyContent (Annotated Byron.UnsafeTx{txInputs, txOutputs} _) =
26802680
txScriptValidity = TxScriptValidityNone
26812681
}
26822682

2683-
makeShelleyTransactionBody :: ()
2684-
=> ShelleyBasedEra era
2683+
makeShelleyTransactionBody
2684+
:: ShelleyBasedEra era
26852685
-> TxBodyContent BuildTx era
26862686
-> Either TxBodyError (TxBody era)
26872687
makeShelleyTransactionBody era@ShelleyBasedEraShelley

cardano-cli/cardano-cli.cabal

+4
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ library
7070
Cardano.CLI.Byron.UpdateProposal
7171
Cardano.CLI.Byron.Vote
7272

73+
Cardano.CLI.IO.Lazy
74+
7375
Cardano.CLI.Shelley.Commands
7476
Cardano.CLI.Shelley.Key
7577
Cardano.CLI.Shelley.Orphans
@@ -135,6 +137,7 @@ library
135137
, ouroboros-network
136138
, parsec
137139
, prettyprinter
140+
, random
138141
, cardano-ledger-shelley
139142
, set-algebra
140143
, split
@@ -143,6 +146,7 @@ library
143146
, time
144147
, transformers
145148
, transformers-except
149+
, unliftio-core
146150
, utf8-string
147151
, vector
148152
, yaml
+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
{-# LANGUAGE BangPatterns #-}
2+
{-# LANGUAGE ScopedTypeVariables #-}
3+
4+
module Cardano.CLI.IO.Lazy
5+
( replicateM
6+
, sequenceM
7+
, traverseM
8+
, traverseStateM
9+
, forM
10+
, forStateM
11+
) where
12+
13+
import Control.Applicative (Applicative((<*>), pure), (<$>))
14+
import Control.Monad (Monad(..))
15+
import Control.Monad.IO.Unlift (MonadIO(liftIO), MonadUnliftIO, askUnliftIO, UnliftIO(unliftIO))
16+
import Data.Function (($), (.), flip)
17+
import Data.Int (Int)
18+
import System.IO (IO)
19+
20+
import qualified Data.List as L
21+
import qualified System.IO.Unsafe as IO
22+
23+
replicateM :: MonadUnliftIO m => Int -> m a -> m [a]
24+
replicateM n f = sequenceM (L.replicate n f)
25+
26+
sequenceM :: MonadUnliftIO m => [m a] -> m [a]
27+
sequenceM as = do
28+
f <- askUnliftIO
29+
liftIO $ sequenceIO (L.map (unliftIO f) as)
30+
31+
-- | Traverses the function over the list and produces a lazy list in a
32+
-- monadic context.
33+
--
34+
-- It is intended to be like the "standard" 'traverse' except
35+
-- that the list is generated lazily.
36+
traverseM :: MonadUnliftIO m => (a -> m b) -> [a] -> m [b]
37+
traverseM f as = do
38+
u <- askUnliftIO
39+
liftIO $ IO.unsafeInterleaveIO (go u as)
40+
where
41+
go _ [] = pure []
42+
go !u (v:vs) = do
43+
!res <- unliftIO u (f v)
44+
rest <- IO.unsafeInterleaveIO (go u vs)
45+
pure (res:rest)
46+
47+
traverseStateM :: forall m s a b. MonadUnliftIO m => s -> (s -> a -> m (s, b)) -> [a] -> m [b]
48+
traverseStateM s f as = do
49+
u <- askUnliftIO
50+
liftIO $ IO.unsafeInterleaveIO (go s u as)
51+
where
52+
go :: s -> UnliftIO m -> [a] -> IO [b]
53+
go _ _ [] = pure []
54+
go t !u (v:vs) = do
55+
(t', !res) <- unliftIO u (f t v)
56+
rest <- IO.unsafeInterleaveIO (go t' u vs)
57+
pure (res:rest)
58+
59+
forM :: MonadUnliftIO m => [a] -> (a -> m b) -> m [b]
60+
forM = flip traverseM
61+
62+
forStateM :: MonadUnliftIO m => s -> [a] -> (s -> a -> m (s, b)) -> m [b]
63+
forStateM s as f = traverseStateM s f as
64+
65+
-- Internal
66+
sequenceIO :: [IO a] -> IO [a]
67+
sequenceIO = IO.unsafeInterleaveIO . go
68+
where go :: [IO a] -> IO [a]
69+
go [] = return []
70+
go (fa:fas) = (:) <$> fa <*> IO.unsafeInterleaveIO (go fas)

cardano-cli/src/Cardano/CLI/Shelley/Key.hs

+7
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ module Cardano.CLI.Shelley.Key
3434

3535
, PaymentVerifier(..)
3636
, StakeVerifier(..)
37+
38+
, generateKeyPair
3739
) where
3840

3941
import Cardano.Prelude
@@ -481,3 +483,8 @@ readVerificationKeyOrHashOrTextEnvFile asType verKeyOrHashOrFile =
481483
eitherVk <- readVerificationKeyOrTextEnvFile asType vkOrFile
482484
pure (verificationKeyHash <$> eitherVk)
483485
VerificationKeyHash vkHash -> pure (Right vkHash)
486+
487+
generateKeyPair :: Key keyrole => AsType keyrole -> IO (VerificationKey keyrole, SigningKey keyrole)
488+
generateKeyPair asType = do
489+
skey <- generateSigningKey asType
490+
return (getVerificationKey skey, skey)

0 commit comments

Comments
 (0)