Merge pull request #3662 from input-output-hk/rvl/cabal-new-build

[DEVOPS-1061] Add support for cabal new-build

Author: disassembler

.gitignore

@@ -1,6 +1,9 @@
 # Cabal & Stack
 .stack-work/
 */dist
+dist-newstyle/
+.ghc.environment.*
+cabal.project.local
 
 # From daedalus-bridge
 node_modules/*

cabal.project

@@ -0,0 +1,8 @@
+packages:
+  */*.cabal
+  */test/*.cabal
+
+-- not sure why the cabal solver wants to download and build a newer beam-sqlite
+constraints: beam-sqlite == 0.3.2.1
+-- allow-newer: all
+-- allow-older: all

docs/how-to/build-cardano-sl-and-daedalus-from-source-code.md

@@ -167,6 +167,45 @@ Run the building script:
     $ cd cardano-sl
     [~/cardano-sl]$ ./scripts/build/cardano-sl.sh
 
+## `cabal new-build` and Nix (experimental, for developers)
+
+This type of build gives you a multi-package project with incremental
+builds, where the all of the build dependencies (Haskell packages and
+system libraries) are downloaded from the binary cache and available
+in the shell.
+
+See the previous section on how to set up the IOHK binary cache.
+
+Before you start, install a recent version of `cabal` into your
+profile. The package set used by cardano-sl (nixpkgs 18.03) only has
+cabal 2.0.0.1 which doesn't work.
+
+    [nix-shell:~]$ nix-env -f channel:nixos-18.09 -iA pkgs.cabal-install
+
+Enter the `nix-shell`:
+
+    [nix-shell:~/cardano-sl]$ nix-shell
+
+Let cabal find its dependencies (already provided by the `nix-shell`):
+
+    [nix-shell:~/cardano-sl]$ cabal new-configure
+
+After that, to build all cardano-sl packages:
+
+    [nix-shell:~/cardano-sl]$ cabal new-build all
+
+To start a GHCi session for a component (wallet-new for example), run:
+
+    [nix-shell:~/cardano-sl]$ cabal new-repl cardano-sl-wallet-new
+
+### Known issues
+
+ - `cabal-install` is not provided by the Nix shell environment. You
+   have to install it yourself.
+ - There needs to be a package version override in `cabal.project` for
+   some unknown reason.
+
+
 ## Daedalus Wallet
 
 Let's proceed with building the wallet.

node/cardano-sl-node.cabal

@@ -91,6 +91,7 @@ test-suite property-tests
                      , cardano-sl-infra
                      , cardano-sl-networking
                      , cardano-sl-util
+                     , cardano-sl-utxo
                      , cardano-sl-wallet-new
                      , constraints
                      , containers
@@ -103,7 +104,6 @@ test-suite property-tests
                      , safe-exceptions >= 0.1
                      , text
                      , universum >= 0.1.11
-                     , utxo
                      , validation
   ghc-options:
                        -threaded

pkgs/default.nix

@@ -17128,6 +17128,7 @@ license = stdenv.lib.licenses.mit;
 , cardano-sl-infra
 , cardano-sl-networking
 , cardano-sl-util
+, cardano-sl-utxo
 , cardano-sl-wallet-new
 , constraints
 , containers
@@ -17144,7 +17145,6 @@ license = stdenv.lib.licenses.mit;
 , stdenv
 , text
 , universum
-, utxo
 , validation
 }:
 mkDerivation {
@@ -17184,6 +17184,7 @@ cardano-sl-db
 cardano-sl-infra
 cardano-sl-networking
 cardano-sl-util
+cardano-sl-utxo
 cardano-sl-wallet-new
 constraints
 containers
@@ -17198,7 +17199,6 @@ reflection
 safe-exceptions
 text
 universum
-utxo
 validation
 ];
 doHaddock = false;
@@ -17662,6 +17662,82 @@ doHaddock = false;
 description = "Cardano SL - general utilities (tests)";
 license = stdenv.lib.licenses.mit;
 
+}) {};
+"cardano-sl-utxo" = callPackage
+({
+  mkDerivation
+, base
+, cardano-crypto
+, cardano-sl
+, cardano-sl-binary
+, cardano-sl-chain
+, cardano-sl-chain-test
+, cardano-sl-client
+, cardano-sl-core
+, cardano-sl-core-test
+, cardano-sl-crypto
+, cardano-sl-crypto-test
+, cardano-sl-db
+, cardano-sl-util
+, constraints
+, containers
+, cryptonite
+, data-default
+, formatting
+, lens
+, log-warper
+, mtl
+, QuickCheck
+, reflection
+, safe-exceptions
+, safecopy
+, serokell-util
+, stdenv
+, universum
+, unordered-containers
+, vector
+}:
+mkDerivation {
+
+pname = "cardano-sl-utxo";
+version = "0.1.0.0";
+src = ./../utxo;
+libraryHaskellDepends = [
+base
+cardano-crypto
+cardano-sl
+cardano-sl-binary
+cardano-sl-chain
+cardano-sl-chain-test
+cardano-sl-client
+cardano-sl-core
+cardano-sl-core-test
+cardano-sl-crypto
+cardano-sl-crypto-test
+cardano-sl-db
+cardano-sl-util
+constraints
+containers
+cryptonite
+data-default
+formatting
+lens
+log-warper
+mtl
+QuickCheck
+reflection
+safe-exceptions
+safecopy
+serokell-util
+universum
+unordered-containers
+vector
+];
+doHaddock = false;
+homepage = "https://github.com/input-output-hk/cardano-sl/#readme";
+description = "Abstract definitions of UTxO based accounting";
+license = stdenv.lib.licenses.mit;
+
 }) {};
 "cardano-sl-wallet" = callPackage
 ({
@@ -17899,6 +17975,7 @@ license = stdenv.lib.licenses.mit;
 , cardano-sl-node-ipc
 , cardano-sl-util
 , cardano-sl-util-test
+, cardano-sl-utxo
 , cardano-sl-wallet
 , cardano-sl-x509
 , cassava
@@ -17969,7 +18046,6 @@ license = stdenv.lib.licenses.mit;
 , unliftio
 , unliftio-core
 , unordered-containers
-, utxo
 , vector
 , wai
 , wai-middleware-throttle
@@ -18018,6 +18094,7 @@ cardano-sl-infra
 cardano-sl-networking
 cardano-sl-node-ipc
 cardano-sl-util
+cardano-sl-utxo
 cardano-sl-wallet
 cardano-sl-x509
 cereal
@@ -18073,7 +18150,6 @@ universum
 unliftio
 unliftio-core
 unordered-containers
-utxo
 vector
 wai
 wai-middleware-throttle
@@ -18134,6 +18210,7 @@ cardano-sl-crypto-test
 cardano-sl-db
 cardano-sl-util
 cardano-sl-util-test
+cardano-sl-utxo
 cardano-sl-wallet
 cereal
 conduit
@@ -18170,7 +18247,6 @@ time
 time-units
 universum
 unordered-containers
-utxo
 vector
 ];
 benchmarkHaskellDepends = [
@@ -83006,82 +83082,6 @@ doCheck = false;
 description = "Various small helper functions for Lists, Maybes, Tuples, Functions";
 license = stdenv.lib.licenses.bsd3;
 
-}) {};
-"utxo" = callPackage
-({
-  mkDerivation
-, base
-, cardano-crypto
-, cardano-sl
-, cardano-sl-binary
-, cardano-sl-chain
-, cardano-sl-chain-test
-, cardano-sl-client
-, cardano-sl-core
-, cardano-sl-core-test
-, cardano-sl-crypto
-, cardano-sl-crypto-test
-, cardano-sl-db
-, cardano-sl-util
-, constraints
-, containers
-, cryptonite
-, data-default
-, formatting
-, lens
-, log-warper
-, mtl
-, QuickCheck
-, reflection
-, safe-exceptions
-, safecopy
-, serokell-util
-, stdenv
-, universum
-, unordered-containers
-, vector
-}:
-mkDerivation {
-
-pname = "utxo";
-version = "0.1.0.0";
-src = ./../utxo;
-libraryHaskellDepends = [
-base
-cardano-crypto
-cardano-sl
-cardano-sl-binary
-cardano-sl-chain
-cardano-sl-chain-test
-cardano-sl-client
-cardano-sl-core
-cardano-sl-core-test
-cardano-sl-crypto
-cardano-sl-crypto-test
-cardano-sl-db
-cardano-sl-util
-constraints
-containers
-cryptonite
-data-default
-formatting
-lens
-log-warper
-mtl
-QuickCheck
-reflection
-safe-exceptions
-safecopy
-serokell-util
-universum
-unordered-containers
-vector
-];
-doHaddock = false;
-homepage = "https://github.com/input-output-hk/cardano-sl/#readme";
-description = "Abstract definitions of UTxO based accounting";
-license = stdenv.lib.licenses.mit;
-
 }) {};
 "uuid" = callPackage
 ({

shell.nix

@@ -16,24 +16,40 @@ in
 }:
 with pkgs;
 let
-  hsPkgs = haskell.packages.ghc822;
+  getCardanoSLDeps = with lib;
+    ps: filter (drv: !(localLib.isCardanoSL drv.name))
+    (concatMap haskell.lib.getHaskellBuildInputs
+     (attrValues (filterAttrs isWantedDep ps)));
+  isWantedDep = name: drv: localLib.isCardanoSL name && !(drv ? "gitrev");
+  ghc = cardanoPkgs.ghc.withPackages getCardanoSLDeps;
+
+  stackDeps = [
+    zlib openssh autoreconfHook openssl
+    gmp rocksdb git bsdiff ncurses lzma
+    perl bash
+  ];
+  # TODO: add cabal-install (2.0.0.1 won't work)
+  devTools = [ hlint cardanoPkgs.stylish-haskell ];
+
   cardanoSL = haskell.lib.buildStackProject {
-     name = "cardano-sl-env";
-     ghc = hsPkgs.ghc;
-     buildInputs = [
-       zlib openssh autoreconfHook openssl
-       gmp rocksdb git bsdiff ncurses
-       hsPkgs.happy hsPkgs.cpphs lzma
-       perl bash
-       cardanoPkgs.stylish-haskell
-       hlint
-     # cabal-install and stack pull in lots of dependencies on OSX so skip them
-     # See https://github.com/NixOS/nixpkgs/issues/21200
-     ] ++ (lib.optionals stdenv.isLinux [ cabal-install stack ])
-       ++ (lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ Cocoa CoreServices libcxx libiconv ]));
+    inherit ghc;
+    name = "cardano-sl-env";
+
+    buildInputs = devTools ++ stackDeps
+      # cabal-install and stack pull in lots of dependencies on OSX so skip them
+      # See https://github.com/NixOS/nixpkgs/issues/21200
+      ++ (lib.optionals stdenv.isLinux [ stack ])
+      ++ (lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ Cocoa CoreServices libcxx libiconv ]));
+
+    shellHook = lib.optionalString lib.inNixShell ''
+      eval "$(egrep ^export ${ghc}/bin/ghc)"
+      export PATH=${ghc}/bin:$PATH
+    '';
+
     phases = ["nobuildPhase"];
     nobuildPhase = "mkdir -p $out";
   };
+
   fixStylishHaskell = stdenv.mkDerivation {
     name = "fix-stylish-haskell";
     buildInputs = [ cardanoPkgs.stylish-haskell git ];

utxo/utxo.cabal renamed to utxo/cardano-sl-utxo.cabal

@@ -1,4 +1,4 @@
-name:                utxo
+name:                cardano-sl-utxo
 version:             0.1.0.0
 synopsis:            Abstract definitions of UTxO based accounting
 -- description: