Remove internal deps on default ghc and stackage

.buildkite/pipeline.yml

@@ -1,6 +1,6 @@
 steps:
-  - label: 'Run tests'
-    command: "./test/tests.sh"
+  - label: 'Run tests with ghc883'
+    command: "./test/tests.sh ghc883"
     agents:
       system: x86_64-linux
 
@@ -14,7 +14,7 @@ steps:
     agents:
       system: x86_64-linux
 
-  - label: 'Check closure size'
+  - label: 'Check closure size with ghc883'
     command:
       - nix-build build.nix -A maintainer-scripts.check-closure-size -o check-closure-size.sh
       - echo "+++ Closure size (MB)"
@@ -24,7 +24,7 @@ steps:
 
   - label: 'Update docs'
     command:
-      - nix-build build.nix -A maintainer-scripts.update-docs -o update-docs.sh
+      - nix-build build.nix -A maintainer-scripts.update-docs --argstr compiler-nix-name ghc883 -o update-docs.sh
       - ./update-docs.sh
     agents:
       system: x86_64-linux
@@ -36,5 +36,5 @@ steps:
 
   - label: 'Make sure non store paths like can be used as src'
     command:
-      - nix-build build.nix -A maintainer-scripts.check-path-support -o check-path-support.sh
+      - nix-build build.nix -A maintainer-scripts.check-path-support --argstr compiler-nix-name ghc883 -o check-path-support.sh
       - ./check-path-support.sh

build.nix

@@ -9,7 +9,7 @@ in
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , pkgs ? import nixpkgs nixpkgsArgs
 , ifdLevel ? 1000
-, compiler-nix-name ? "ghc865"
+, compiler-nix-name ? throw "No `compiler-nix-name` passed to build.nix"
 }:
 
 let
@@ -33,8 +33,21 @@ in rec {
   # as not all of them can work in restricted eval mode (as they
   # are not pure).
   maintainer-scripts = pkgs.dontRecurseIntoAttrs {
-    update-hackage = haskell.callPackage ./scripts/update-hackage.nix {};
-    update-stackage = haskell.callPackage ./scripts/update-stackage.nix {};
+    update-hackage = import ./scripts/update-hackage.nix {
+      inherit (pkgs) stdenv writeScript coreutils glibc git
+        openssh nix-prefetch-git gawk bash curl findutils;
+      # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
+      nix-tools = haskell.internal-nix-tools;
+      cabal-install = haskell.internal-cabal-install;
+      inherit (haskell) update-index-state-hashes;
+    };
+    update-stackage = haskell.callPackage ./scripts/update-stackage.nix {
+      inherit (pkgs) stdenv writeScript coreutils glibc git
+        openssh nix-prefetch-git gawk bash curl findutils;
+      # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
+      nix-tools = haskell.internal-nix-tools;
+      cabal-install = haskell.internal-cabal-install;
+    };
     update-pins = haskell.callPackage ./scripts/update-pins.nix {};
     update-docs = pkgs.buildPackages.callPackage ./scripts/update-docs.nix {
       generatedOptions = pkgs.callPackage ./scripts/options-doc.nix { };

lib/call-cabal-project-to-nix.nix

@@ -21,8 +21,8 @@ in
 , compiler-nix-name
 , ghc           ? null # Deprecated in favour of `compiler-nix-name`
 , ghcOverride   ? null # Used when we need to set ghc explicitly during bootstrapping
-, nix-tools     ? evalPackages.haskell-nix.nix-tools.${compiler-nix-name}
-, cabal-install ? evalPackages.haskell-nix.cabal-install.${compiler-nix-name}
+, nix-tools     ? evalPackages.haskell-nix.nix-tools.${compiler-nix-name}     # When building cabal projects we use the nix-tools
+, cabal-install ? evalPackages.haskell-nix.cabal-install.${compiler-nix-name} # and cabal-install compiled with matching ghc version
 , configureArgs ? "" # Extra arguments to pass to `cabal v2-configure`.
                      # `--enable-tests --enable-benchmarks` are included by default.
                      # If the tests and benchmarks are not needed and they

lib/call-stack-to-nix.nix

@@ -15,7 +15,7 @@
 , resolverSha256 ? null
 , materialized ? null # Location of a materialized copy of the nix files
 , checkMaterialization ? null # If true the nix files will be generated used to check plan-sha256 and material
-, nix-tools ? pkgs.haskell-nix.internal-nix-tools
+, nix-tools ? pkgs.haskell-nix.internal-nix-tools # When building stack projects we use the internal nix-tools (compiled with a fixed GHC version)
 , ... }:
 let
   inherit (haskellLib.fetchResolver {

lib/stack-cache-generator.nix

@@ -14,7 +14,7 @@
     then { location, tag, ...}: sha256map."${location}"."${tag}"
     else _: null
 , resolverSha256 ? null
-, nix-tools ? pkgs.haskell-nix.internal-nix-tools
+, nix-tools ? pkgs.haskell-nix.internal-nix-tools # When building stack projects we use the internal nix-tools (compiled with a fixed GHC version)
 , ...
 }:
 let

overlays/bootstrap.nix

@@ -394,12 +394,6 @@ in {
         nix-tools = final.evalPackages.haskell-nix.nix-tools-unchecked.${compiler-nix-name};
         materialized = ../materialized + "/${compiler-nix-name}/cabal-install";
       } // args)).components.exes.cabal;
-    cabal-install = final.lib.mapAttrs (compiler-nix-name: _:
-      final.haskell-nix.cabal-install-tool { inherit compiler-nix-name; }) final.haskell-nix.compiler;
-    cabal-install-unchecked = final.lib.mapAttrs (compiler-nix-name: _:
-      final.haskell-nix.cabal-install-tool { inherit compiler-nix-name; checkMaterialization = false; }) final.haskell-nix.compiler;
-    # Use this where we still have not good way to choose GHC version
-    internal-cabal-install = final.haskell-nix.cabal-install.ghc883;
     nix-tools-set = { compiler-nix-name, ... }@args:
       let
         project =
@@ -453,11 +447,36 @@ in {
           done
         '';
       };
+
+    # Memoize the cabal-install and nix-tools derivations by adding:
+    #   haskell-nix.cabal-install.ghcXXX
+    #   haskell-nix.cabal-install-unchecked.ghcXXX
+    #   haskell-nix.nix-tools.ghcXXX
+    #   haskell-nix.nix-tools-unchecked.ghcXXX
+    # Using these avoids unecessary calls to mkDerivation.
+    # For cabal projects we match the versions used to the compiler
+    # selected for the project to avoid the chance of a dependency
+    # another GHC version (particularly useful on macOS where
+    # executables are dynamically linked to GHC itself).
+    cabal-install = final.lib.mapAttrs (compiler-nix-name: _:
+      final.haskell-nix.cabal-install-tool { inherit compiler-nix-name; }) final.haskell-nix.compiler;
+    cabal-install-unchecked = final.lib.mapAttrs (compiler-nix-name: _:
+      final.haskell-nix.cabal-install-tool { inherit compiler-nix-name; checkMaterialization = false; }) final.haskell-nix.compiler;
     nix-tools = final.lib.mapAttrs (compiler-nix-name: _:
       final.haskell-nix.nix-tools-set { inherit compiler-nix-name; }) final.haskell-nix.compiler;
     nix-tools-unchecked = final.lib.mapAttrs (compiler-nix-name: _:
       final.haskell-nix.nix-tools-set { inherit compiler-nix-name; checkMaterialization = false; }) final.haskell-nix.compiler;
-    # Use this where we still have not good way to choose GHC version
+
+    # These `internal` versions are used for:
+    # * `nix-tools` for stack projects (since we use `nix-tools` to process
+    #   the `stack.yaml` file we cannot match the ghc of the project the
+    #   way we do for cabal projects).
+    # * Scripts are used to update stackage and hackage
+    # Updating the version of GHC selected here should be fairly safe as
+    # there should be no difference in the behaviour of these tools.
+    # (stack projects on macOS may see a significant change in the
+    # closure size of their build dependencies due to dynamic linking).
+    internal-cabal-install = final.haskell-nix.cabal-install.ghc883;
     internal-nix-tools = final.haskell-nix.nix-tools.ghc883;
 
     # WARN: The `import ../. {}` will prevent

overlays/haskell.nix

@@ -252,8 +252,10 @@ final: prev: {
         };
 
         update-index-state-hashes = import ../scripts/update-index-state-hashes.nix {
-            inherit (final.haskell-nix) indexStateHashesPath internal-nix-tools;
+            inherit (final.haskell-nix) indexStateHashesPath;
             inherit (final) coreutils nix writeShellScriptBin stdenv curl;
+            # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
+            nix-tools = final.haskell-nix.internal-nix-tools;
         };
 
         # Function to call stackToNix
@@ -267,6 +269,8 @@ final: prev: {
         # to produce the nix representation of it.
         callCabalToNix = { name, src, cabal-file ? "${name}.cabal" }:
             final.buildPackages.pkgs.runCommand "${name}.nix" {
+                # This function is only used when building stack projects (via mkCacheLine and mkCacheFile)
+                # When building stack projects we use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
                 nativeBuildInputs = [ final.buildPackages.haskell-nix.internal-nix-tools ];
 
                 LOCALE_ARCHIVE = final.lib.optionalString (final.stdenv.buildPlatform.libc == "glibc") "${final.buildPackages.glibcLocales}/lib/locale/locale-archive";

scripts/update-external.nix

@@ -1,5 +1,5 @@
 { stdenv, writeScript, glibc, coreutils, git, openssh
-, internal-nix-tools, internal-cabal-install, nix-prefetch-git
+, nix-tools, cabal-install, nix-prefetch-git
 , bash, curl, findutils, gawk }:
 
 { name, script }:
@@ -16,7 +16,7 @@ in
 
     set -euo pipefail
 
-    export PATH="${makeBinPath ([ coreutils curl findutils gawk bash git openssh internal-nix-tools internal-cabal-install nix-prefetch-git ] ++ optional stdenv.isLinux glibc)}"
+    export PATH="${makeBinPath ([ coreutils curl findutils gawk bash git openssh nix-tools cabal-install nix-prefetch-git ] ++ optional stdenv.isLinux glibc)}"
 
     ${script}
 

scripts/update-hackage.nix

@@ -1,5 +1,5 @@
 { stdenv, writeScript, coreutils, glibc, git, openssh
-, internal-nix-tools, internal-cabal-install, nix-prefetch-git
+, nix-tools, cabal-install, nix-prefetch-git
 , gawk, bash, curl, findutils
 , update-index-state-hashes }@args:
 

scripts/update-index-state-hashes.nix

@@ -1,8 +1,8 @@
-{ indexStateHashesPath, internal-nix-tools, coreutils, nix, writeShellScriptBin, stdenv, curl }:
+{ indexStateHashesPath, nix-tools, coreutils, nix, writeShellScriptBin, stdenv, curl }:
 with builtins;
 with stdenv.lib;
 writeShellScriptBin "update-index-state-hashes" ''
-   export PATH="${makeBinPath [ coreutils internal-nix-tools nix curl ]}"
+   export PATH="${makeBinPath [ coreutils nix-tools nix curl ]}"
 
    # We'll take the last element from the indexStatesHashes file via nix and get the name.
    # This is the last timestamp recorded in the file (implicit assumption: the file is

scripts/update-stackage.nix

@@ -1,5 +1,5 @@
 { stdenv, writeScript, coreutils, glibc, git, openssh
-, internal-nix-tools, internal-cabal-install, nix-prefetch-git
+, nix-tools, cabal-install, nix-prefetch-git
 , gawk, bash, curl, findutils }@args:
 
 import ./update-external.nix args {

test/default.nix

@@ -3,7 +3,7 @@
 , nixpkgs ? haskellNix.sources.nixpkgs
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , ifdLevel ? 1000
-, compiler-nix-name ? "ghc865"
+, compiler-nix-name
 , checkMaterialization ? false
 }:
 

test/tests.sh

@@ -7,6 +7,13 @@ NIX_BUILD_ARGS="${NIX_BUILD_ARGS:-}"
 
 cd $(dirname $0)
 
+if [ "$#" != "1" ]; then
+  echo "Please pass a compiler-nix-name to use.  For example: ./test/test.sh ghc883"
+  exit 1
+fi
+
+GHC=$1
+
 printf "*** Cleaning package build directories..." >& 2
 rm -rvf */cabal.project.local */.ghc.environment* */dist */dist-newstyle */.stack-work
 echo >& 2
@@ -16,11 +23,12 @@ nix build $NIX_BUILD_ARGS \
    -I . -I .. \
    --option restrict-eval true \
    --option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
-   --no-link --keep-going -f default.nix
+   --no-link --keep-going -f default.nix \
+   --argstr compiler-nix-name $GHC
 echo >& 2
 
 printf "*** Running the unit tests... " >& 2
-res=$(nix-instantiate --eval --json --strict ./default.nix -A unit.tests)
+res=$(nix-instantiate --eval --json --strict ./default.nix --argstr compiler-nix-name $GHC -A unit.tests)
 num_failed=$(jq length <<< "$res")
 if [ $num_failed -eq 0 ]; then
   printf "PASSED\n" >& 2
@@ -33,20 +41,23 @@ fi
 printf "*** Checking that a nix-shell works for runghc...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC \
     -A with-packages.test-shell \
     --run 'runghc with-packages/Point.hs'
 echo >& 2
 
 printf "*** Checking that a nix-shell works for cabal...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC \
     -A with-packages.test-shell \
     --run 'echo CABAL_CONFIG=$CABAL_CONFIG && type -p ghc && cd with-packages && cabal new-build'
 echo >& 2
 
 printf "*** Checking that a nix-shell works for cabal (doExactConfig component)...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC \
     -A with-packages.test-shell-dec \
     --run 'echo CABAL_CONFIG=$CABAL_CONFIG && echo GHC_ENVIRONMENT=$GHC_ENVIRONMENT && cd with-packages && cabal new-build'
 echo >& 2
@@ -55,6 +66,7 @@ printf "*** Checking that a nix-shell works for a project with test-suite build-
 printf "!!! This is expected to fail until https://github.com/input-output-hk/haskell.nix/issues/231 is resolved! \n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC \
     -A cabal-22.shell \
     --run 'cd cabal-22 && cabal new-build all --enable-tests --enable-benchmarks' \
     || true
@@ -63,35 +75,39 @@ echo >& 2
 printf "*** Checking that a nix-shell works for a multi-target project...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC \
     -A cabal-simple.test-shell \
     --run 'cd cabal-simple && cabal new-build'
 echo >& 2
 
 printf "*** Checking shellFor works for a cabal project, multiple packages...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC
     -A shell-for.env \
     --run 'cd shell-for && cabal new-build all'
 echo >& 2
 
 printf "*** Checking shellFor works for a cabal project, single package...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC \
     -A shell-for.envPkga \
     --run 'cd shell-for && cabal new-build --project=single.project all'
 echo >& 2
 
 printf "*** Checking shellFor has a working hoogle index...\n" >& 2
 nix-shell $NIX_BUILD_ARGS \
     --pure ./default.nix \
+    --argstr compiler-nix-name $GHC
     -A shell-for.env \
     --run 'hoogle ConduitT | grep Data.Conduit'
 echo >& 2
 
 printf "*** Checking shellFor does not depend on given packages...\n" >& 2
-drva=$(nix-instantiate ./default.nix -A shell-for.env)
+drva=$(nix-instantiate ./default.nix --argstr compiler-nix-name $GHC -A shell-for.env)
 echo "-- hello" >> shell-for/pkga/PkgA.hs
-drvb=$(nix-instantiate ./default.nix -A shell-for.env)
+drvb=$(nix-instantiate ./default.nix --argstr compiler-nix-name $GHC -A shell-for.env)
 sed -i -e '/-- hello/d' shell-for/pkga/PkgA.hs
 if [ "$drva" != "$drvb" ]; then
     printf "FAIL\nShell derivations\n$drva\n$drvb\n are not identical.\n" >& 2
@@ -106,13 +122,13 @@ echo >& 2
 
 printf "*** Checking that plan construction works with extra Hackages...\n" >& 2
 nix build $NIX_BUILD_ARGS --no-link \
-    -f ./default.nix \
+    -f ./default.nix --argstr compiler-nix-name $GHC \
     extra-hackage.run.project.plan-nix
 echo >& 2
 
 printf "*** Checking that package with extra Hackages can be build...\n" >& 2
 nix build $NIX_BUILD_ARGS --no-link \
-    -f ./default.nix \
+    -f ./default.nix --argstr compiler-nix-name $GHC \
     extra-hackage.run.project.hsPkgs.external-package-user.components.all
 echo >& 2