From 5bdc918234cf44acdd866425b6f5e57ee2925f28 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Giraudeau Date: Tue, 17 Jan 2023 12:07:55 +0100 Subject: [PATCH 1/4] nix: minimaly filter projetBase for tests to avoid rebuilds --- flake.lock | 451 ++++++++++++++++++++++- flake.nix | 5 + nix/haskell.nix | 149 ++++---- nix/workbench/backend/supervisor-run.nix | 2 +- 4 files changed, 540 insertions(+), 67 deletions(-) diff --git a/flake.lock b/flake.lock index 71aae9b6e42..4625647b39d 100644 --- a/flake.lock +++ b/flake.lock @@ -305,6 +305,21 @@ "type": "github" } }, + "blank": { + "locked": { + "lastModified": 1625557891, + "narHash": "sha256-O8/MWsPBGhhyPoPLHZAuoZiiHo9q6FLlEeIDEXuj6T4=", + "owner": "divnix", + "repo": "blank", + "rev": "5a5d2684073d9f563072ed07c871d577a6c614a8", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "blank", + "type": "github" + } + }, "cabal-32": { "flake": false, "locked": { @@ -2272,6 +2287,60 @@ "type": "github" } }, + "devshell": { + "inputs": { + "flake-utils": [ + "tullia", + "std", + "flake-utils" + ], + "nixpkgs": [ + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1663445644, + "narHash": "sha256-+xVlcK60x7VY1vRJbNUEAHi17ZuoQxAIH4S4iUFUGBA=", + "owner": "numtide", + "repo": "devshell", + "rev": "e3dc3e21594fe07bdb24bdf1c8657acaa4cb8f66", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "dmerge": { + "inputs": { + "nixlib": [ + "tullia", + "std", + "nixpkgs" + ], + "yants": [ + "tullia", + "std", + "yants" + ] + }, + "locked": { + "lastModified": 1659548052, + "narHash": "sha256-fzI2gp1skGA8mQo/FBFrUAtY0GQkAIAaV/V127TJPyY=", + "owner": "divnix", + "repo": "data-merge", + "rev": "d160d18ce7b1a45b88344aa3f13ed1163954b497", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "data-merge", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -2390,6 +2459,22 @@ "type": "github" } }, + "flake-compat_8": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1644229661, @@ -2540,6 +2625,21 @@ "type": "github" } }, + "flake-utils_19": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "flake-utils_2": { "locked": { "lastModified": 1644229661, @@ -2555,6 +2655,21 @@ "type": "github" } }, + "flake-utils_20": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "flake-utils_3": { "locked": { "lastModified": 1644229661, @@ -2966,6 +3081,25 @@ "type": "github" } }, + "gomod2nix": { + "inputs": { + "nixpkgs": "nixpkgs_16", + "utils": "utils_18" + }, + "locked": { + "lastModified": 1655245309, + "narHash": "sha256-d/YPoQ/vFn1+GTmSdvbSBSTOai61FONxB4+Lt6w/IVI=", + "owner": "tweag", + "repo": "gomod2nix", + "rev": "40d32f82fc60d66402eb0972e6e368aeab3faf58", + "type": "github" + }, + "original": { + "owner": "tweag", + "repo": "gomod2nix", + "type": "github" + } + }, "hackage": { "flake": false, "locked": { @@ -4317,6 +4451,28 @@ "type": "indirect" } }, + "incl": { + "inputs": { + "nixlib": [ + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1669263024, + "narHash": "sha256-E/+23NKtxAqYG/0ydYgxlgarKnxmDbg6rCMWnOBqn9Q=", + "owner": "divnix", + "repo": "incl", + "rev": "ce7bebaee048e4cd7ebdb4cee7885e00c4e2abca", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "incl", + "type": "github" + } + }, "iohkNix": { "inputs": { "nixpkgs": [ @@ -5203,6 +5359,33 @@ "type": "github" } }, + "n2c": { + "inputs": { + "flake-utils": [ + "tullia", + "std", + "flake-utils" + ], + "nixpkgs": [ + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1665039323, + "narHash": "sha256-SAh3ZjFGsaCI8FRzXQyp56qcGdAqgKEfJWPCQ0Sr7tQ=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "b008fe329ffb59b67bf9e7b08ede6ee792f2741a", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nix": { "inputs": { "lowdown-src": "lowdown-src", @@ -5224,6 +5407,38 @@ "type": "github" } }, + "nix-nomad": { + "inputs": { + "flake-compat": "flake-compat_8", + "flake-utils": [ + "tullia", + "nix2container", + "flake-utils" + ], + "gomod2nix": "gomod2nix", + "nixpkgs": [ + "tullia", + "nixpkgs" + ], + "nixpkgs-lib": [ + "tullia", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1658277770, + "narHash": "sha256-T/PgG3wUn8Z2rnzfxf2VqlR1CBjInPE0l1yVzXxPnt0=", + "owner": "tristanpemble", + "repo": "nix-nomad", + "rev": "054adcbdd0a836ae1c20951b67ed549131fd2d70", + "type": "github" + }, + "original": { + "owner": "tristanpemble", + "repo": "nix-nomad", + "type": "github" + } + }, "nix-tools": { "flake": false, "locked": { @@ -5496,6 +5711,25 @@ "type": "github" } }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_19", + "nixpkgs": "nixpkgs_17" + }, + "locked": { + "lastModified": 1658567952, + "narHash": "sha256-XZ4ETYAMU7XcpEeAFP3NOl9yDXNuZAen/aIJ84G+VgA=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "60bb43d405991c1378baf15a40b5811a53e32ffa", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nixTools": { "flake": false, "locked": { @@ -5554,6 +5788,38 @@ "type": "github" } }, + "nixago": { + "inputs": { + "flake-utils": [ + "tullia", + "std", + "flake-utils" + ], + "nixago-exts": [ + "tullia", + "std", + "blank" + ], + "nixpkgs": [ + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1661824785, + "narHash": "sha256-/PnwdWoO/JugJZHtDUioQp3uRiWeXHUdgvoyNbXesz8=", + "owner": "nix-community", + "repo": "nixago", + "rev": "8c1f9e5f1578d4b2ea989f618588d62a335083c3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixago", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1642336556, @@ -6865,6 +7131,69 @@ "type": "indirect" } }, + "nixpkgs_16": { + "locked": { + "lastModified": 1653581809, + "narHash": "sha256-Uvka0V5MTGbeOfWte25+tfRL3moECDh1VwokWSZUdoY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "83658b28fe638a170a19b8933aa008b30640fbd1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_17": { + "locked": { + "lastModified": 1654807842, + "narHash": "sha256-ADymZpr6LuTEBXcy6RtFHcUZdjKTBRTMYwu19WOx17E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "fc909087cc3386955f21b4665731dbdaceefb1d8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_18": { + "locked": { + "lastModified": 1674407282, + "narHash": "sha256-2qwc8mrPINSFdWffPK+ji6nQ9aGnnZyHSItVcYDZDlk=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "ab1254087f4cdf4af74b552d7fc95175d9bdbb49", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_19": { + "locked": { + "lastModified": 1665087388, + "narHash": "sha256-FZFPuW9NWHJteATOf79rZfwfRn5fE0wi9kRzvGfDHPA=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "95fda953f6db2e9496d2682c4fc7b82f959878f7", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1632864508, @@ -7164,6 +7493,21 @@ "type": "github" } }, + "nosys": { + "locked": { + "lastModified": 1667881534, + "narHash": "sha256-FhwJ15uPLRsvaxtt/bNuqE/ykMpNAPF0upozFKhTtXM=", + "owner": "divnix", + "repo": "nosys", + "rev": "2d0d5207f6a230e9d0f660903f8db9807b54814f", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "nosys", + "type": "github" + } + }, "old-ghc-nix": { "flake": false, "locked": { @@ -7869,7 +8213,12 @@ "node-process": "node-process_2", "node-snapshot": "node-snapshot_2", "plutus-apps": "plutus-apps_4", - "utils": "utils_18" + "std": [ + "tullia", + "std" + ], + "tullia": "tullia", + "utils": "utils_19" } }, "stackage": { @@ -8160,6 +8509,69 @@ "type": "github" } }, + "std": { + "inputs": { + "arion": [ + "tullia", + "std", + "blank" + ], + "blank": "blank", + "devshell": "devshell", + "dmerge": "dmerge", + "flake-utils": "flake-utils_20", + "incl": "incl", + "makes": [ + "tullia", + "std", + "blank" + ], + "microvm": [ + "tullia", + "std", + "blank" + ], + "n2c": "n2c", + "nixago": "nixago", + "nixpkgs": "nixpkgs_19", + "nosys": "nosys", + "yants": "yants" + }, + "locked": { + "lastModified": 1674526466, + "narHash": "sha256-tMTaS0bqLx6VJ+K+ZT6xqsXNpzvSXJTmogkraBGzymg=", + "owner": "divnix", + "repo": "std", + "rev": "516387e3d8d059b50e742a2ff1909ed3c8f82826", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "std", + "type": "github" + } + }, + "tullia": { + "inputs": { + "nix-nomad": "nix-nomad", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs_18", + "std": "std" + }, + "locked": { + "lastModified": 1675695930, + "narHash": "sha256-B7rEZ/DBUMlK1AcJ9ajnAPPxqXY6zW2SBX+51bZV0Ac=", + "owner": "input-output-hk", + "repo": "tullia", + "rev": "621365f2c725608f381b3ad5b57afef389fd4c31", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "tullia", + "type": "github" + } + }, "utils": { "locked": { "lastModified": 1648297722, @@ -8296,6 +8708,21 @@ } }, "utils_18": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "utils_19": { "locked": { "lastModified": 1623875721, "narHash": "sha256-A8BU7bjS5GirpAUv4QA+QnJ4CceLHkcXdRp4xITDB0s=", @@ -8429,6 +8856,28 @@ "repo": "flake-utils", "type": "github" } + }, + "yants": { + "inputs": { + "nixpkgs": [ + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1667096281, + "narHash": "sha256-wRRec6ze0gJHmGn6m57/zhz/Kdvp9HS4Nl5fkQ+uIuA=", + "owner": "divnix", + "repo": "yants", + "rev": "d18f356ec25cb94dc9c275870c3a7927a10f8c3c", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "yants", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index f757b1f1a61..e580b3c86af 100644 --- a/flake.nix +++ b/flake.nix @@ -63,6 +63,9 @@ }; cardano-mainnet-mirror.url = "github:input-output-hk/cardano-mainnet-mirror/nix"; + + tullia.url = "github:input-output-hk/tullia"; + std.follows = "tullia/std"; }; outputs = @@ -79,6 +82,7 @@ , node-measured , node-process , cardano-node-workbench + , std , ... }@input: let @@ -156,6 +160,7 @@ project = (import ./nix/haskell.nix { inherit (pkgs) haskell-nix gitrev; + inherit (std) incl; inputMap = { "https://input-output-hk.github.io/cardano-haskell-packages" = CHaP; }; diff --git a/nix/haskell.nix b/nix/haskell.nix index faaaf47274c..d7b2a6bcce2 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -5,6 +5,7 @@ , # Version info (git revision) gitrev , inputMap +, incl }: let @@ -20,32 +21,7 @@ let }: { inherit inputMap; name = "cardano-node"; - # We clean-up src to avoid rebuild for unrelated changes for tests that use $CARDANO_NODE_SRC: - src = - let - src = haskellLib.cleanSourceWith { - src = ../.; - name = "cardano-node-src"; - filter = path: type: - let - relPath = lib.removePrefix "${src.origSrcSubDir}/" path; - in - # excludes top-level directories not part of cabal project (or used by tests): - (type != "directory" - || (builtins.match ".*/.*" relPath != null) - || (!(lib.elem relPath [ - "nix" - "doc" - "ci" - ]) - && !(lib.hasPrefix "." relPath))) - # exclude ".gitignore" files - && !(lib.hasSuffix ".gitignore" relPath) - # only keep cabal.project from files at root: - && (type == "directory" || builtins.match ".*/.*" relPath != null || (relPath == "cabal.project")); - }; - in - src; + src = ../.; compiler-nix-name = "ghc8107"; cabalProjectLocal = '' allow-newer: terminfo:base @@ -122,45 +98,88 @@ let }) projectPackagesExes; }) - ({ pkgs, config, ... }: { - # Packages we wish to ignore version bounds of. - # This is similar to jailbreakCabal, however it - # does not require any messing with cabal files. - packages.katip.doExactConfig = true; - # split data output for ekg to reduce closure size - packages.ekg.components.library.enableSeparateDataOutput = true; - # cardano-cli-test depends on cardano-cli - # TODO: do not depend on the whole `src` just to access a few files. - packages.cardano-cli.preCheck = " - export CARDANO_CLI=${config.hsPkgs.cardano-cli.components.exes.cardano-cli}/bin/cardano-cli${pkgs.stdenv.hostPlatform.extensions.executable} - export CARDANO_NODE_SRC=${src} - "; - packages.cardano-node-chairman.components.tests.chairman-tests.build-tools = - lib.mkForce [ - pkgs.lsof - config.hsPkgs.cardano-node.components.exes.cardano-node - config.hsPkgs.cardano-cli.components.exes.cardano-cli - config.hsPkgs.cardano-node-chairman.components.exes.cardano-node-chairman - ]; - # cardano-node-chairman depends on cardano-node and cardano-cli - # TODO: do not depend on the whole `src` just to access a few files. - packages.cardano-node-chairman.preCheck = " - export CARDANO_CLI=${config.hsPkgs.cardano-cli.components.exes.cardano-cli}/bin/cardano-cli${pkgs.stdenv.hostPlatform.extensions.executable} - export CARDANO_NODE=${config.hsPkgs.cardano-node.components.exes.cardano-node}/bin/cardano-node${pkgs.stdenv.hostPlatform.extensions.executable} - export CARDANO_NODE_CHAIRMAN=${config.hsPkgs.cardano-node-chairman.components.exes.cardano-node-chairman}/bin/cardano-node-chairman${pkgs.stdenv.hostPlatform.extensions.executable} - export CARDANO_NODE_SRC=${src} - "; - # cardano-testnet needs access to the git repository source - # TODO: do not depend on the whole `src` just to access a few files. - packages.cardano-testnet.preCheck = " - export CARDANO_CLI=${config.hsPkgs.cardano-cli.components.exes.cardano-cli}/bin/cardano-cli${pkgs.stdenv.hostPlatform.extensions.executable} - export CARDANO_NODE=${config.hsPkgs.cardano-node.components.exes.cardano-node}/bin/cardano-node${pkgs.stdenv.hostPlatform.extensions.executable} - export CARDANO_SUBMIT_API=${config.hsPkgs.cardano-submit-api.components.exes.cardano-submit-api}/bin/cardano-submit-api${pkgs.stdenv.hostPlatform.extensions.executable} - ${lib.optionalString (!pkgs.stdenv.hostPlatform.isWindows) '' - ''} - export CARDANO_NODE_SRC=${src} - "; - }) + ({ pkgs, config, ... }: + let + exportCliPath = "export CARDANO_CLI=${config.hsPkgs.cardano-cli.components.exes.cardano-cli}/bin/cardano-cli${pkgs.stdenv.hostPlatform.extensions.executable}"; + exportNodePath = "export CARDANO_NODE=${config.hsPkgs.cardano-node.components.exes.cardano-node}/bin/cardano-node${pkgs.stdenv.hostPlatform.extensions.executable}"; + mainnetConfigFiles = [ + "configuration/cardano/mainnet-config.yaml" + "configuration/cardano/mainnet-config.json" + "configuration/cardano/mainnet-byron-genesis.json" + "configuration/cardano/mainnet-shelley-genesis.json" + "configuration/cardano/mainnet-alonzo-genesis.json" + ]; + in + { + # Packages we wish to ignore version bounds of. + # This is similar to jailbreakCabal, however it + # does not require any messing with cabal files. + packages.katip.doExactConfig = true; + # split data output for ekg to reduce closure size + packages.ekg.components.library.enableSeparateDataOutput = true; + # cardano-cli tests depend on cardano-cli and some config files: + packages.cardano-cli.components.tests.cardano-cli-golden.preCheck = + let + # This define files included in the directory that will be passed to `H.getProjectBase` for this test: + filteredProjectBase = incl ../. [ + "scripts/plutus/scripts/v1/custom-guess-42-datum-42.plutus" + ]; + in + '' + ${exportCliPath} + export CARDANO_NODE_SRC=${filteredProjectBase} + ''; + packages.cardano-cli.components.tests.cardano-cli-test.preCheck = + let + # This define files included in the directory that will be passed to `H.getProjectBase` for this test: + filteredProjectBase = incl ../. mainnetConfigFiles; + in + '' + ${exportCliPath} + export CARDANO_NODE_SRC=${filteredProjectBase} + ''; + packages.cardano-node-chairman.components.tests.chairman-tests.build-tools = + lib.mkForce [ + pkgs.lsof + config.hsPkgs.cardano-node.components.exes.cardano-node + config.hsPkgs.cardano-cli.components.exes.cardano-cli + config.hsPkgs.cardano-node-chairman.components.exes.cardano-node-chairman + ]; + # cardano-node-chairman depends on cardano-node and cardano-cli, and some config files + packages.cardano-node-chairman.preCheck = + let + # This define files included in the directory that will be passed to `H.getProjectBase` for this test: + filteredProjectBase = incl ../. [ + "configuration/chairman" + "configuration/defaults/byron-mainnet" + "cardano-cli/test/data/golden/alonzo/genesis.alonzo.spec.json" + "scripts/protocol-params.json" + ]; + in + '' + ${exportCliPath} + ${exportNodePath} + export CARDANO_NODE_CHAIRMAN=${config.hsPkgs.cardano-node-chairman.components.exes.cardano-node-chairman}/bin/cardano-node-chairman${pkgs.stdenv.hostPlatform.extensions.executable} + export CARDANO_NODE_SRC=${filteredProjectBase} + ''; + # cardano-testnet depends on cardano-node, cardano-cli, cardano-submit-api and some config files + packages.cardano-testnet.preCheck = + let + # This define files included in the directory that will be passed to `H.getProjectBase` for this test: + filteredProjectBase = incl ../. (mainnetConfigFiles ++ [ + "configuration/cardano/mainnet-topology.json" + "configuration/defaults/byron-mainnet" + "cardano-cli/test/data/golden/alonzo/genesis.alonzo.spec.json" + "scripts/babbage/alonzo-babbage-test-genesis.json" + ]); + in + '' + ${exportCliPath} + ${exportNodePath} + export CARDANO_SUBMIT_API=${config.hsPkgs.cardano-submit-api.components.exes.cardano-submit-api}/bin/cardano-submit-api${pkgs.stdenv.hostPlatform.extensions.executable} + export CARDANO_NODE_SRC=${filteredProjectBase} + ''; + }) ({ pkgs, ... }: lib.mkIf (!pkgs.stdenv.hostPlatform.isDarwin) { # Needed for profiled builds to fix an issue loading recursion-schemes part of makeBaseFunctor # that is missing from the `_p` output. See https://gitlab.haskell.org/ghc/ghc/-/issues/18320 diff --git a/nix/workbench/backend/supervisor-run.nix b/nix/workbench/backend/supervisor-run.nix index 422d490d5bc..4e1de877029 100644 --- a/nix/workbench/backend/supervisor-run.nix +++ b/nix/workbench/backend/supervisor-run.nix @@ -108,7 +108,7 @@ in --genesis-cache-entry ${genesis} --batch-name smoke-test --base-port ${toString basePort} - --node-source ${pkgs.cardanoNodeProject.args.src.origSrc} + --node-source ${pkgs.cardanoNodeProject.args.src} --node-rev ${cardano-node-rev} --cache-dir ./cache ) From c810b87a0827bc672e57c62db65c55234c19a32d Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Giraudeau Date: Wed, 25 Jan 2023 09:49:45 +0100 Subject: [PATCH 2/4] tullia jobs: use generic gitrev for PRs to avoid unnecessary builds of big artifact, like dockerImages. Goal: bors ci build should not build anything if branch is up-to-date and ci pass. Some other minor clean-ups. --- flake.nix | 375 +++++++++++++++++++++++------------------------- nix/haskell.nix | 7 +- 2 files changed, 181 insertions(+), 201 deletions(-) diff --git a/flake.nix b/flake.nix index e580b3c86af..85cf0bf7fd0 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,11 @@ { description = "Cardano Node"; + nixConfig = { + extra-substituters = [ "https://cache.iog.io" ]; + extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" ]; + }; + inputs = { # IMPORTANT: report any change to nixpkgs channel in nix/default.nix: nixpkgs.follows = "haskellNix/nixpkgs-unstable"; @@ -10,9 +15,9 @@ flake = false; }; nixTools = { - url = "github:input-output-hk/nix-tools"; - flake = false; - }; + url = "github:input-output-hk/nix-tools"; + flake = false; + }; haskellNix = { url = "github:input-output-hk/haskell.nix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -133,246 +138,224 @@ inherit (project.pkgs) system; }).plutus-apps.haskell.packages.plutus-example.components.exes) plutus-example; - in project.exes // (with project.hsPkgs; { - inherit (ouroboros-consensus-byron.components.exes) db-converter; + in + project.exes // (with project.hsPkgs; { + inherit (ouroboros-consensus-byron.components.exes) db-converter; inherit (ouroboros-consensus-cardano.components.exes) db-analyser; inherit (ouroboros-consensus-cardano-tools.components.exes) db-synthesizer; - inherit (bech32.components.exes) bech32; - } // lib.optionalAttrs hostPlatform.isUnix { - inherit (network-mux.components.exes) cardano-ping; - inherit plutus-example; - }); + inherit (bech32.components.exes) bech32; + } // lib.optionalAttrs hostPlatform.isUnix { + inherit (network-mux.components.exes) cardano-ping; + inherit plutus-example; + }); mkCardanoNodePackages = project: (collectExes project) // { inherit (project.pkgs) cardanoLib; }; - flake = eachSystem supportedSystems (system: - let - pkgs = import nixpkgs { - inherit system overlays; - inherit (haskellNix) config; - }; - inherit (pkgs.haskell-nix) haskellLib; - inherit (haskellLib) collectChecks' collectComponents'; - inherit (pkgs.commonLib) eachEnv environments mkSupervisordCluster; - inherit (project.pkgs.stdenv) hostPlatform; + mkFlakeAttrs = pkgs: rec { + inherit (pkgs) system; + inherit (pkgs.haskell-nix) haskellLib; + inherit (haskellLib) collectChecks' collectComponents'; + inherit (pkgs.commonLib) eachEnv environments mkSupervisordCluster; + inherit (pkgs.stdenv) hostPlatform; + project = pkgs.cardanoNodeProject; - project = (import ./nix/haskell.nix { - inherit (pkgs) haskell-nix gitrev; - inherit (std) incl; - inputMap = { - "https://input-output-hk.github.io/cardano-haskell-packages" = CHaP; - }; - }).appendModule customConfig.haskellNix; + # This is used by `nix develop .` to open a devShell + devShells = let shell = import ./shell.nix { inherit pkgs customConfig cardano-mainnet-mirror; }; in { + inherit (shell) devops; + default = shell.dev; + cluster = shell; + profiled = project.profiled.shell; + }; - pinned-workbench = cardano-node-workbench.workbench.${system}; + # NixOS tests run a node and submit-api and validate it listens + nixosTests = import ./nix/nixos/tests { + inherit pkgs; + }; - shell = import ./shell.nix { inherit pkgs customConfig cardano-mainnet-mirror; }; - devShells = { - inherit (shell) devops; - default = shell.dev; - cluster = shell; - profiled = project.profiled.shell; + checks = flattenTree project.checks // + # Linux only checks: + (optionalAttrs hostPlatform.isLinux ( + prefixNamesWith "nixosTests/" (mapAttrs (_: v: v.${system} or v) nixosTests) + )) + # checks run on default system only; + // (optionalAttrs (system == defaultSystem) { + hlint = pkgs.callPackage pkgs.hlintCheck { + inherit (project.args) src; }; + }); - # NixOS tests run a node and submit-api and validate it listens - nixosTests = import ./nix/nixos/tests { - inherit pkgs; + exes = (collectExes project) // { + inherit (pkgs) cabalProjectRegenerate checkCabalProject; + "dockerImages/push" = import ./.buildkite/docker-build-push.nix { + hostPkgs = import hostNixpkgs { inherit system; }; + inherit (pkgs) dockerImage submitApiDockerImage; }; - - checks = flattenTree project.checks // - # Linux only checks: - (optionalAttrs hostPlatform.isLinux ( - prefixNamesWith "nixosTests/" (mapAttrs (_: v: v.${system} or v) nixosTests) - )) - # checks run on default system only; - // (optionalAttrs (system == defaultSystem) { - hlint = pkgs.callPackage pkgs.hlintCheck { - inherit (project.args) src; - }; - }); - - exes = (collectExes project) // { - inherit (pkgs) cabalProjectRegenerate checkCabalProject; - "dockerImages/push" = import ./.buildkite/docker-build-push.nix { - hostPkgs = import hostNixpkgs { inherit system; }; - inherit (pkgs) dockerImage submitApiDockerImage; - }; - "dockerImage/node/load" = pkgs.writeShellScript "load-docker-image" '' - docker load -i ${pkgs.dockerImage} $@ - ''; - "dockerImage/submit-api/load" = pkgs.writeShellScript "load-submit-docker-image" '' - docker load -i ${pkgs.submitApiDockerImage} $@ - ''; - } // flattenTree (pkgs.scripts // { - # `tests` are the test suites which have been built. - inherit (project) tests; - # `benchmarks` (only built, not run). - inherit (project) benchmarks; - }); - - inherit (pkgs) workbench all-profiles-json supervisord-workbench-nix supervisord-workbench-for-profile; - - packages = - let - supervisord-workbench = - pkgs.callPackage supervisord-workbench-nix { workbench = pinned-workbench; }; - in - exes - # Linux only packages: - // optionalAttrs (system == "x86_64-linux") rec { - "dockerImage/node" = pkgs.dockerImage; - "dockerImage/submit-api" = pkgs.submitApiDockerImage; - # workbench-smoke-test = - # (pkgs.supervisord-workbench-for-profile - # { - # inherit supervisord-workbench; - # profileName = "smoke-alzo"; - # } - # ).profile-run { trace = true; }; - # workbench-ci-test = - # (pkgs.supervisord-workbench-for-profile - # { - # inherit supervisord-workbench; - # profileName = "k6-600slots-1000kU-1000kD-64kbs-10tps-fixed-loaded-alzo"; - # } - # ).profile-run { }; - # workbench-smoke-analysis = workbench-smoke-test.analysis; - # workbench-ci-analysis = workbench-ci-test.analysis; - all-profiles-json = pkgs.all-profiles-json; - } - # Add checks to be able to build them individually - // (prefixNamesWith "checks/" checks); - - apps = lib.mapAttrs (n: p: { type = "app"; program = p.exePath or (if (p.executable or false) then "${p}" else "${p}/bin/${p.name or n}"); }) exes; - - ciJobs = let ciJobs = { - cardano-deployment = pkgs.cardanoLib.mkConfigHtml { inherit (pkgs.cardanoLib.environments) mainnet testnet; }; - } // optionalAttrs (system == "x86_64-linux") { - native = packages // { - shells = devShells; - internal = { - roots.project = project.roots; - plan-nix.project = project.plan-nix; - }; - profiled = lib.genAttrs [ "cardano-node" "tx-generator" "locli" ] (n: - packages.${n}.passthru.profiled - ); - asserted = lib.genAttrs [ "cardano-node" ] (n: - packages.${n}.passthru.asserted - ); - }; - musl = - let - muslProject = project.projectCross.musl64; - projectExes = collectExes muslProject; - in - projectExes // { - cardano-node-linux = import ./nix/binary-release.nix { - inherit pkgs; - inherit (exes.cardano-node.identifier) version; - platform = "linux"; - exes = lib.collect lib.isDerivation projectExes; + "dockerImage/node/load" = pkgs.writeShellScript "load-docker-image" '' + docker load -i ${pkgs.dockerImage} $@ + ''; + "dockerImage/submit-api/load" = pkgs.writeShellScript "load-submit-docker-image" '' + docker load -i ${pkgs.submitApiDockerImage} $@ + ''; + } // flattenTree (pkgs.scripts // { + # `tests` are the test suites which have been built. + inherit (project) tests; + # `benchmarks` (only built, not run). + inherit (project) benchmarks; + }); + + inherit (pkgs) workbench; + + packages = + exes + # Linux only packages: + // optionalAttrs (system == "x86_64-linux") rec { + "dockerImage/node" = pkgs.dockerImage; + "dockerImage/submit-api" = pkgs.submitApiDockerImage; + } + # Add checks to be able to build them individually + // (prefixNamesWith "checks/" checks); + + apps = lib.mapAttrs (n: p: { type = "app"; program = p.exePath or (if (p.executable or false) then "${p}" else "${p}/bin/${p.name or n}"); }) exes; + + ciJobs = + let + ciJobs = { + cardano-deployment = pkgs.cardanoLib.mkConfigHtml { inherit (pkgs.cardanoLib.environments) mainnet testnet; }; + } // optionalAttrs (system == "x86_64-linux") { + native = packages // { + shells = devShells; + internal = { + roots.project = project.roots; + plan-nix.project = project.plan-nix; }; - internal.roots.project = muslProject.roots; + profiled = lib.genAttrs [ "cardano-node" "tx-generator" "locli" ] (n: + packages.${n}.passthru.profiled + ); + asserted = lib.genAttrs [ "cardano-node" ] (n: + packages.${n}.passthru.asserted + ); }; - windows = - let - windowsProject = project.projectCross.mingwW64; - projectExes = collectExes windowsProject; - in - projectExes - // (removeRecurse { - inherit (windowsProject) checks tests benchmarks; - cardano-node-win64 = import ./nix/binary-release.nix { + musl = + let + muslProject = project.projectCross.musl64; + projectExes = collectExes muslProject; + in + projectExes // { + cardano-node-linux = import ./nix/binary-release.nix { + inherit pkgs; + inherit (exes.cardano-node.identifier) version; + platform = "linux"; + exes = lib.collect lib.isDerivation projectExes; + }; + internal.roots.project = muslProject.roots; + }; + windows = + let + windowsProject = project.projectCross.mingwW64; + projectExes = collectExes windowsProject; + in + projectExes + // (removeRecurse { + inherit (windowsProject) checks tests benchmarks; + cardano-node-win64 = import ./nix/binary-release.nix { + inherit pkgs; + inherit (exes.cardano-node.identifier) version; + platform = "win64"; + exes = lib.collect lib.isDerivation projectExes; + }; + internal.roots.project = windowsProject.roots; + }); + } // optionalAttrs (system == "x86_64-darwin") { + native = lib.filterAttrs + (n: _: + # only build docker images once on linux: + !(lib.hasPrefix "dockerImage" n)) + packages // { + cardano-node-macos = import ./nix/binary-release.nix { inherit pkgs; inherit (exes.cardano-node.identifier) version; - platform = "win64"; - exes = lib.collect lib.isDerivation projectExes; + platform = "macos"; + exes = lib.collect lib.isDerivation (collectExes project); + }; + shells = removeAttrs devShells [ "profiled" ]; + internal = { + roots.project = project.roots; + plan-nix.project = project.plan-nix; }; - internal.roots.project = windowsProject.roots; - }); - } // optionalAttrs (system == "x86_64-darwin") { - native = lib.filterAttrs (n: _: - # only build docker images once on linux: - !(lib.hasPrefix "dockerImage" n)) packages // { - cardano-node-macos = import ./nix/binary-release.nix { - inherit pkgs; - inherit (exes.cardano-node.identifier) version; - platform = "macos"; - exes = lib.collect lib.isDerivation (collectExes project); - }; - shells = removeAttrs devShells [ "profiled" ]; - internal = { - roots.project = project.roots; - plan-nix.project = project.plan-nix; }; }; - }; - defaultNonRequiredPaths = [ "windows.checks.cardano-tracer.cardano-tracer-test" ] ++ + nonRequiredPaths = [ "windows.checks.cardano-tracer.cardano-tracer-test" + ] ++ lib.optionals (system == "x86_64-darwin") [ #FIXME: ExceptionInLinkedThread (ThreadId 253) pokeSockAddr: path is too long "native.checks/cardano-testnet/cardano-testnet-tests" ]; - in pkgs.callPackages iohkNix.utils.ciJobsAggregates { - inherit ciJobs; - nonRequiredPaths = map lib.hasPrefix defaultNonRequiredPaths; - } // { - pr = pkgs.callPackages iohkNix.utils.ciJobsAggregates { + in + pkgs.callPackages iohkNix.utils.ciJobsAggregates + { inherit ciJobs; - nonRequiredPaths = map lib.hasPrefix (defaultNonRequiredPaths ++ [ - "native.membenches" - ]); + nonRequiredPaths = map lib.hasPrefix nonRequiredPaths; + } // ciJobs // { + pr = { + # We use a generic gitrev for PR CI to avoid unecessary rebuilds: + inherit ((mkFlakeAttrs (pkgs.extend (prev: final: { gitrev = "0000000000000000000000000000000000000000"; }))).ciJobs) + required nonrequired; }; - } // ciJobs; + }; + }; + flake = eachSystem supportedSystems (system: + let + pkgs = import nixpkgs { + inherit system overlays; + inherit (haskellNix) config; + }; + inherit (mkFlakeAttrs pkgs) environments packages checks apps project ciJobs devShells workbench; in { - inherit environments packages checks apps project ciJobs; + inherit environments checks project ciJobs devShells workbench; legacyPackages = pkgs // { - # allows access to hydraJobs with specifying : + # allows access to hydraJobs without specifying : hydraJobs = ciJobs; }; - # Built by `nix build .` - defaultPackage = packages.cardano-node; + packages = packages // { + # Built by `nix build .` + default = packages.cardano-node; + }; # Run by `nix run .` - defaultApp = apps.cardano-node; - - # This is used by `nix develop .` to open a devShell - inherit devShells; - - # The parametrisable workbench. - inherit workbench; + apps = apps // { + default = apps.cardano-node; + }; } ); - hydraJobs = flake.ciJobs // { - # TODO: remove nex 3 lines once tullia config reference architecture directly: - inherit (flake.ciJobs.${defaultSystem}) cardano-deployment; - linux.required = flake.ciJobs.x86_64-linux.required or {}; - macos.required = flake.ciJobs.x86_64-darwin.required or {}; - }; - # TODO: also hydraJobsPr remove once tullia config reference hydraJobs..pr: - hydraJobsPr = { - inherit (flake.ciJobs.${defaultSystem}) cardano-deployment; - linux.required = flake.ciJobs.x86_64-linux.pr.required or {}; - macos.required = flake.ciJobs.x86_64-darwin.pr.required or {}; - }; - in removeAttrs flake ["ciJobs"] // { + in + removeAttrs flake [ "ciJobs" ] // { - inherit hydraJobs hydraJobsPr; + hydraJobs = flake.ciJobs; # allows precise paths (avoid fallbacks) with nix build/eval: outputs = self; overlay = final: prev: { - cardanoNodeProject = flake.project.${final.system}; + cardanoNodeProject = (import ./nix/haskell.nix { + inherit (final) haskell-nix; + inherit (std) incl; + }).appendModule [ + { + inputMap = { + "https://input-output-hk.github.io/cardano-haskell-packages" = CHaP; + }; + } + customConfig.haskellNix + ]; cardanoNodePackages = mkCardanoNodePackages final.cardanoNodeProject; inherit (final.cardanoNodePackages) cardano-node cardano-cli cardano-submit-api bech32 plutus-example; diff --git a/nix/haskell.nix b/nix/haskell.nix index d7b2a6bcce2..541f2b3ba19 100644 --- a/nix/haskell.nix +++ b/nix/haskell.nix @@ -2,9 +2,6 @@ # Builds Haskell packages with Haskell.nix ############################################################################ { haskell-nix -, # Version info (git revision) - gitrev -, inputMap , incl }: let @@ -19,7 +16,6 @@ let , buildProject , ... }: { - inherit inputMap; name = "cardano-node"; src = ../.; compiler-nix-name = "ghc8107"; @@ -225,7 +221,8 @@ let }); in project.appendOverlays (with haskellLib.projectOverlays; [ projectComponents - (final: prev: let inherit (final.pkgs) lib; in { + (final: prev: + let inherit (final.pkgs) lib gitrev; in { profiled = final.appendModule { modules = [{ enableLibraryProfiling = true; From 62ad457acda69ee4aa567adcee383ac091160679 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Giraudeau Date: Tue, 7 Feb 2023 09:49:12 +0100 Subject: [PATCH 3/4] Remove usage of nixWrapped. it was introduced for adding flake flags by default and also as a work around for nix flake show/check: https://github.com/NixOS/nix/issues/4265 But the added complexity is not worth it, given it breaks other workflows with nix run (since the wrapper script does not use `exec`). Also nowadays everyone (should) have flake activated by default in the nix config. --- nix/README.md | 11 ++--------- nix/update-haskellNix.sh | 4 ---- nix/update-iohkNix.sh | 4 ---- nix/workbench/analyse.nix | 2 +- nix/workbench/backend/supervisor-run.nix | 2 +- nix/workbench/supervisor-run.nix | 2 +- shell.nix | 4 ++-- 7 files changed, 7 insertions(+), 22 deletions(-) delete mode 100755 nix/update-haskellNix.sh delete mode 100755 nix/update-iohkNix.sh diff --git a/nix/README.md b/nix/README.md index 1ed82d24e0b..adb27469298 100644 --- a/nix/README.md +++ b/nix/README.md @@ -1,19 +1,12 @@ # Nix dependencies -The nix build use the new flake format to manage dependencies. A flake-compatible nix command is provided from within `nix-shell`. To add flake support to your native nix setup please see https://nixos.wiki/wiki/Flakes. +The nix build use the new flake format to manage dependencies. To add flake support to your native nix setup please see https://nixos.wiki/wiki/Flakes. Cardano-node nix build depends primarily on [haskell.nix](https://github.com/input-output-hk/haskell.nix) and secondarily, for some utilities, on [iohk-nix](https://github.com/input-output-hk/iohk-nix/). -Both can be updated from within a cardano-node `nix-shell` with: +Both can be updated with: ``` nix flake lock --update-input haskellNix nix flake lock --update-input iohkNix ``` - -Or from outside the `nix-shell` with the scripts: - -``` -./nix/update-haskellNix.sh -./nix/update-iohkNix.sh -``` diff --git a/nix/update-haskellNix.sh b/nix/update-haskellNix.sh deleted file mode 100755 index dfc317c169c..00000000000 --- a/nix/update-haskellNix.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -cd "$(dirname "$0")/.." -nix-shell -I nixpkgs=./nix -p nixWrapped \ - --run "nix flake lock --update-input haskellNix" diff --git a/nix/update-iohkNix.sh b/nix/update-iohkNix.sh deleted file mode 100755 index 7d938a144cf..00000000000 --- a/nix/update-iohkNix.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -cd "$(dirname "$0")/.." -nix-shell -I nixpkgs=./nix -p nixWrapped \ - --run "nix flake lock --update-input iohkNix" diff --git a/nix/workbench/analyse.nix b/nix/workbench/analyse.nix index 3518e1b39c7..9b39520a095 100644 --- a/nix/workbench/analyse.nix +++ b/nix/workbench/analyse.nix @@ -7,7 +7,7 @@ pkgs.runCommand "workbench-run-analysis-${profileNix.name}" { requiredSystemFeatures = [ "benchmark" ]; nativeBuildInputs = with pkgs.haskellPackages; with pkgs; - [ bash coreutils gnused jq moreutils nixWrapped workbench.workbench ]; + [ bash coreutils gnused jq moreutils nix workbench.workbench ]; } '' echo "analysing run: ${run}" diff --git a/nix/workbench/backend/supervisor-run.nix b/nix/workbench/backend/supervisor-run.nix index 4e1de877029..d43c2c6273b 100644 --- a/nix/workbench/backend/supervisor-run.nix +++ b/nix/workbench/backend/supervisor-run.nix @@ -83,7 +83,7 @@ in gnused jq moreutils - nixWrapped + nix pstree python3Packages.supervisor workbench.workbench diff --git a/nix/workbench/supervisor-run.nix b/nix/workbench/supervisor-run.nix index b75bb9dc9e5..54712b8488b 100644 --- a/nix/workbench/supervisor-run.nix +++ b/nix/workbench/supervisor-run.nix @@ -89,7 +89,7 @@ let gnused jq moreutils - nixWrapped + nix pstree python3Packages.supervisor workbench.workbench diff --git a/shell.nix b/shell.nix index 9b579fde46e..c1d6e3d1242 100644 --- a/shell.nix +++ b/shell.nix @@ -74,7 +74,7 @@ let haskellBuildUtils pkgs.graphviz weeder - nixWrapped + nix pkgconfig profiteur profiterole @@ -132,7 +132,7 @@ let packages = _: []; nativeBuildInputs = with cardanoNodePackages; [ - nixWrapped + nix cardano-cli bech32 cardano-ping From c8c825b3b7481e956536d2562ebc13b1866f0dbf Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Giraudeau Date: Fri, 17 Feb 2023 16:41:10 +0100 Subject: [PATCH 4/4] devops shell: don't include build tools for all hasPkgs because this lead to duplicates PATH entries for cardano-cli, cardano-node, etc. with one having gitrev set and the other not set. --- shell.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shell.nix b/shell.nix index c1d6e3d1242..a5878345936 100644 --- a/shell.nix +++ b/shell.nix @@ -149,6 +149,9 @@ let pkgs.time ]; + # build tools for all hasPkgs not needed (would include duplicates for cardano-cli, cardano-node, etc.) + allToolDeps = false; + shellHook = '' echo "DevOps Tools" \ | ${figlet}/bin/figlet -f banner -c \