nix: refactor Gradle deps to be more generic

Refactoring the derivation that fetches all the POMs, JARs,
and AARs in order to make it more generic and easier to extend.
The main change is adding `files` key in `deps.json` which contains
a dict of all the files reletad to given package.

This way we can more easily include other files that might be available
for download, like AARs with ASC suffix, or `nodeps` JARs.

This is also necessary for the React Native upgrade:
#15203

Signed-off-by: Jakub Sokołowski <[email protected]>

Author: jakubgs

nix/deps/gradle/README.md

@@ -24,6 +24,7 @@ Generating scripts:
 - `get_deps.sh` - Calls Gradle to get all the dependencies of sub-projects.
 - `gradle_parser.awk` - An AWK script that parses above Gradle output.
 - `url2json.sh` - Converts the list of URLs into a format consumable by Nix.
+- `add_package.sh` - Allows for adding a missing package manually.
 
 Finally we have the Nix derivation in `default.nix` which produces a derivation with all of the Gradle project dependencies:
 ```
@@ -66,9 +67,14 @@ You can use the `add_package.sh` script to add missing Gradle dependencies:
  > make shell TARGET=gradle
 Configuring Nix shell for target 'gradle'...
 
-[nix-shell:~/status-mobile]$ nix/deps/gradle/add_package.sh com.android.tools.build:gradle:3.4.0
+[nix-shell:~/work/status-mobile]$ nix/deps/gradle/add_package.sh com.android.tools.build:gradle:3.5.3
+Changes made:
+ nix/deps/gradle/deps.urls | 24 ++++++++++++++++++++++++
+ 1 file changed, 24 insertions(+)
+
 Regenerating Nix files...
-Found 548 direct dependencies...
-Successfully added: com.android.tools.build:gradle:3.4.0
+Successfully added: com.android.tools.build:gradle:3.5.3
+
+NOTICE: Running 'make nix-update-gradle' in a new shell is recommended.
 ```
-This may take a bit longer than normal `make nix-update-gradle` but should add the missing package.
+Keep in mind that the changes made by this script do not affect the already spawned shell.

nix/deps/gradle/add_package.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 
-set -Eeu
+# This script allows for adding a package by providing a Maven full name.
+# Such name consists of 3 sections separated by the colon character.
+# Example: com.android.tools.build:gradle:3.5.3
+
+set -Eeuo pipefail
 
 if [[ $# -ne 1 ]]; then
     echo "Usage: add_package.sh <package>" >&2
@@ -18,12 +22,15 @@ source "${GIT_ROOT}/scripts/colors.sh"
 echo "${1}" | go-maven-resolver >> nix/deps/gradle/deps.urls
 
 # Remove duplicates and sort.
-sort -uo nix/deps/gradle/deps.urls nix/deps/gradle/deps.urls
+sort -uVo nix/deps/gradle/deps.urls nix/deps/gradle/deps.urls
+
+echo -e "${GRN}Changes made:${RST}" >&2
+git diff --stat nix/deps/gradle/deps.urls
+echo
 
 # Re-generate dependencies JSON.
 "${GIT_ROOT}/nix/deps/gradle/generate.sh" gen_deps_json
 
-# Re-generate dependencies list.
-"${GIT_ROOT}/nix/deps/gradle/generate.sh" gen_deps_list
-
 echo -e "${GRN}Successfully added:${RST} ${BLD}${1}${RST}" >&2
+echo
+echo -e "${YLW}NOTICE:${RST} Running '${BLD}make nix-update-gradle${RST}' in a new shell is recommended."

nix/deps/gradle/default.nix

@@ -1,82 +1,44 @@
 { stdenv, lib, pkgs, fetchurl, writeShellScriptBin }:
 
 let
-  inherit (lib)
-    removeSuffix optionalString splitString concatMapStrings
-    attrByPath attrValues last makeOverridable importJSON;
+  inherit (lib) concatStrings concatMapStrings mapAttrsToList makeOverridable;
 
-  inherit (pkgs) aapt2;
-
-  deps = importJSON ./deps.json;
-
-  # some .jar files have an `-aot` suffix that doesn't work for .pom files
-  getPOM = jarUrl: "${removeSuffix "-aot" jarUrl}.pom";
+  deps = lib.importJSON ./deps.json;
 
   script = writeShellScriptBin "create-local-maven-repo" (''
     mkdir -p $out
     cd $out
   '' +
-  # TODO: Generalize this section to not repeat the same code.
-  (concatMapStrings (dep: 
-    let
-      url = "${dep.host}/${dep.path}";
-      pom = {
-        sha1 = attrByPath [ "pom" "sha1" ] "" dep;
-        sha256 = attrByPath [ "pom" "sha256" ] "" dep;
-      };
-      pom-download = optionalString (pom.sha256 != "") (
-        fetchurl { url = getPOM url; inherit (pom) sha256; }
-      );
-      jar = {
-        sha1 = attrByPath [ "jar" "sha1" ] "" dep;
-        sha256 = attrByPath [ "jar" "sha256" ] "" dep;
-      };
-      jar-download = optionalString (jar.sha256 != "") (
-        fetchurl { url = "${url}.${dep.type}"; inherit (jar) sha256; }
-      );
-      nodeps = {
-        sha1 = attrByPath [ "nodeps" "sha1" ] "" dep;
-        sha256 = attrByPath [ "nodeps" "sha256" ] "" dep;
-      };
-      nodeps-download = optionalString (nodeps.sha256 != "") (
-        fetchurl { url = "${url}-nodeps.jar"; inherit (nodeps) sha256; }
-      );
-      fileName = last (splitString "/" dep.path);
-      directory = removeSuffix fileName dep.path;
-    in
-      ''
-        mkdir -p ${directory}
-
-        ${optionalString (pom-download != "") ''
-        ln -s "${pom-download}" "${getPOM dep.path}"
-        ''}
-        ${optionalString (pom.sha1 != "") ''
-        echo "${pom.sha1}" > "${getPOM dep.path}.sha1"
-        ''}
-        ${optionalString (jar-download != "") ''
-        ln -s "${jar-download}" "${dep.path}.${dep.type}"
-        ''}
-        ${optionalString (jar.sha1 != "") ''
-        echo "${jar.sha1}" > "${dep.path}.${dep.type}.sha1"
-        ''}
-        ${optionalString (nodeps-download != "") ''
-        ln -s "${nodeps-download}" "${dep.path}-nodeps.jar"
-        ''}
-        ${optionalString (nodeps.sha1 != "") ''
-        echo "${nodeps.sha1}" > "${dep.path}.${dep.type}.sha1"
-        ''}
+    # For each dependency in deps.json.
+    (concatMapStrings (dep: concatStrings
+      # And for each file in the 'files' dict of given dependency.
+      (mapAttrsToList (filename: hashes: let
+        # Download given file(POM, JAR, nodeps JAR, or AAR).
+        download = fetchurl {
+          url = "${dep.repo}/${dep.path}/${filename}";
+          inherit (hashes) sha256;
+        };
+        # And symlink it in the correct folder along with SHA1.
+      in ''
+        mkdir -p "${dep.path}"
+        ln -s "${download}" "${dep.path}/${filename}"
+        echo "${hashes.sha1}" > "${dep.path}/${filename}.sha1"
       '')
-    deps));
+      dep.files)
+    ) deps)
+  );
 
 in makeOverridable stdenv.mkDerivation {
   name = "status-mobile-maven-deps";
-  buildInputs = [ aapt2 ];
+
+  buildInputs = [ pkgs.aapt2 ];
+
   phases = [ "buildPhase" "patchPhase" ];
   buildPhase = "${script}/bin/create-local-maven-repo";
-  # Patched AAPT2 
+  # Replace AAPT2 package only with our patched version from overlay.
   patchPhase = ''
-    aapt2_dir=$out/com/android/tools/build/aapt2/${aapt2.version}
+    aapt2_dir=$out/com/android/tools/build/aapt2/${pkgs.aapt2.version}
     mkdir -p $aapt2_dir
-    ln -sf ${aapt2}/* $aapt2_dir
+    ln -sf ${pkgs.aapt2}/* $aapt2_dir
   '';
 }