Update cabal_build_tests scripts to allow testing builds at commits.

jorisdral · jorisdral · commit f61f23825d4e · 2022-06-24T14:25:39.000+02:00
The script previously was only aimed at build-testing `cardano-node`
tags. The script can now also build-test specific commit hashes.

Other changes:
* Include installation of missing package dependencies (openssl, lmdb).
* Include Secp256k1 installation.
* "Parameterise" the script with cabal and ghc versions.
* Installing of `cardano-cli` and `cardano-node` executables simplified.
*
diff --git a/cabal_build_tests/install-node.sh b/cabal_build_tests/install-node.sh
@@ -1,43 +1,63 @@
 #!/bin/bash
 
-### Builds cardano-node on Ubuntu or Fedora using Cabal and verifys successful installation
+### Builds cardano-node on Ubuntu or Fedora using Cabal and verifies successful installation
 ### Please note: sudo is not used because user is root
 ### Please note: 'source ~/.bashrc' cmd is not used because Docker runs this script as subscript
 
 echo ""
 
-if [[ $TAGGED_VERSION == "null" ]]
+# Check that environment variables are correctly set up.
+if [[ -z "${GIT_OBJECT}" ]]
 then
-	printf "Please specify TAGGED_VERSION on docker run.\ne.g. '-e TAGGED_VERSION=1.23.0'"
+	>&2 printf "Please specify GIT_OBJECT_TYPE on docker run.\ne.g. '-e GIT_OBJECT_TYPE=tag' or '-e GIT_OBJECT_TYPE=commit'"
+elif [[ -z "${GIT_OBJECT}" ]]
+then
+	>&2 printf "Please specify GIT_OBJECT on docker run.\ne.g. '-e GIT_OBJECT=1.23.0' for tags, or '-e GIT_OBJECT=78wagy3aw87ef' for commits."
+	exit 1
 fi
 
-echo "Using tagged version $TAGGED_VERSION"
+echo "Using git object '$GIT_OBJECT' of type '$GIT_OBJECT_TYPE'"
 cd ~ || exit 1
 
+# Set up GHC and cabal-install versions
+GHC_VERSION="8.10.7"
+GHC="ghc-$GHC_VERSION"
+GHC_SUFFIX="x86_64-deb9-linux.tar.xz"
+GHC_FULL="$GHC-$GHC_SUFFIX"
+
+echo "Using GHC version $GHC_VERSION"
+
+CABAL_VERSION="3.6.2.0"
+CABAL="cabal-install-$CABAL_VERSION"
+CABAL_SUFFIX="x86_64-linux-deb10.tar.xz"
+CABAL_FULL="$CABAL-$CABAL_SUFFIX"
+
+echo "Using cabal-install version $CABAL_VERSION"
+
 # Install dependencies
 echo "Install dependencies"
 if [[ $(cat /etc/os-release) == *"fedora"* ]]
 then
 	echo "Running on Fedora"
 	yum update -y
 	yum install git gcc gcc-c++ tmux gmp-devel make tar xz wget zlib-devel libtool autoconf -y
-	yum install systemd-devel ncurses-devel ncurses-compat-libs -y
+	yum install systemd-devel ncurses-devel ncurses-compat-libs openssl-devel lmdb-devel -y
 elif [[ $(cat /etc/os-release) == *"ubuntu"* ]]
 then
 	echo "Running on Ubuntu"
 	apt-get update -y
-	apt-get install automake build-essential pkg-config libffi-dev libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev make g++ tmux git jq wget libncursesw5 libtool autoconf -y
+	apt-get install automake build-essential pkg-config libffi-dev libgmp-dev libssl-dev libtinfo-dev libsystemd-dev zlib1g-dev make g++ tmux git jq wget libncursesw5 libtool autoconf liblmdb-dev -y
 else
-	echo "/etc/os-relase does not contain 'fedora' or 'ubuntu'"
-	cat /etc/os-release
+	>&2 echo "/etc/os-relase does not contain 'fedora' or 'ubuntu'"
+	>&2 cat /etc/os-release
 	exit 1
 fi
 
 # Download, unpack, install and update Cabal
 echo "Download, unpack, install and update Cabal"
-wget https://downloads.haskell.org/~cabal/cabal-install-3.2.0.0/cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz
-tar -xf cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz
-rm cabal-install-3.2.0.0-x86_64-unknown-linux.tar.xz cabal.sig
+wget https://downloads.haskell.org/~cabal/$CABAL/$CABAL_FULL
+tar -xf $CABAL_FULL
+rm $CABAL_FULL
 mkdir -p ~/.local/bin
 mv cabal ~/.local/bin/
 
@@ -54,28 +74,36 @@ fi
 
 cabal update
 
-if [[ $(cabal --version) != *"cabal-install version 3.2.0.0"* ]]
+if [[ $(cabal --version) != *"cabal-install version $CABAL_VERSION"* ]]
 then
-        echo "cabal version $(cabal --version) is not 3.2.0.0"
+	>&2 echo "cabal version $(cabal --version) is not $CABAL_VERSION"
 	exit 1
 else
-	echo "cabal version 3.2.0.0 is correct"
+	echo "cabal version $CABAL_VERSION is correct"
 fi
 
-# Download and install GHC
+# Download, unpack and install GHC
 echo "Download and install GHC"
 
 mkdir -p ~/src || exit 1
 cd ~/src || exit 1
-wget https://downloads.haskell.org/ghc/8.10.2/ghc-8.10.2-x86_64-deb9-linux.tar.xz
-tar -xf ghc-8.10.2-x86_64-deb9-linux.tar.xz
-rm ghc-8.10.2-x86_64-deb9-linux.tar.xz
-cd ghc-8.10.2 || exit 1
+wget https://downloads.haskell.org/ghc/$GHC_VERSION/$GHC_FULL
+tar -xf $GHC_FULL
+rm $GHC_FULL
+cd $GHC || exit 1
 ./configure
 make install
 
+if [[ $(ghc --version) != *"version $GHC_VERSION"* ]]
+then
+	>&2 echo "ghc version $(ghc --version) is not $GHC_VERSION"
+	exit 1
+else
+	echo "ghc version $GHC_VERSION is correct"
+fi
+
 # Install Libsodium
-echo "Install Lobsodium"
+echo "Install Libsodium"
 
 mkdir -p ~/src || exit 1
 cd ~/src || exit 1
@@ -90,6 +118,19 @@ make install
 export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
 export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH"
 
+# Install Secp256k1
+echo "Install Secp256k1"
+
+mkdir -p ~/src || exit 1
+cd ~/src || exit 1
+git clone https://github.com/bitcoin-core/secp256k1
+cd secp256k1 || exit 1
+git checkout ac83be33
+./autogen.sh
+./configure --enable-module-schnorrsig --enable-experimental
+make
+make install
+
 # Download the source code for cardano-node
 echo "Download the source code for cardano-node"
 
@@ -99,42 +140,101 @@ git clone https://github.com/input-output-hk/cardano-node.git
 cd cardano-node || exit 1
 git fetch --all --recurse-submodules --tags
 
-if [[ $(git tag) != *"$TAGGED_VERSION"* ]]
+# Checkout with prechecks for git object and git object type
+if [[ $(git cat-file -t "$GIT_OBJECT") != "commit" ]]
 then
-	echo "$(git tag) does not contain $TAGGED_VERSION"
+	>&2 echo "$(git cat-file -t "$GIT_OBJECT") does not refer to a commit/tag."
 	exit 1
 fi
 
-git checkout "tags/$TAGGED_VERSION"
+case $GIT_OBJECT_TYPE in
+	tag)	
+		if [[ $(git tag) != *"$GIT_OBJECT" ]]
+		then
+			>&2 echo "$(git tag) does not contain $GIT_OBJECT"
+			exit 1
+		fi
+		;;
+	commit) 
+		:
+		;;
+	*) 
+		exit 1
+		;;
+esac
+
+git checkout "$GIT_OBJECT"
 
 # Configure and build options
 echo "Configure and build options"
-cabal configure --with-compiler=ghc-8.10.2
+cabal configure --with-compiler=$GHC
 echo "package cardano-crypto-praos" >>  cabal.project.local
 echo "  flags: -external-libsodium-vrf" >>  cabal.project.local
 
 # Build and install the node
 echo "Build and install the node"
 cabal build all
-cabal install all --bindir ~/.local/bin
-
-if [[ ! -d ~/.local/bin/dist-newstyle/build/x86_64-linux/ghc-8.10.2/cardano-cli-$TAGGED_VERSION/x/ ]]
+cabal install --bindir=~/.local/bin cardano-node cardano-cli 
+
+# This is a bit of ugly indirection, and can probably be done better. The
+# problem is that the path is a glob, and so might match more than 1 
+# path
+INSTALLED=false
+for d in ~/.local/bin/dist-newstyle/build/x86_64-linux/"$GHC"/cardano-cli-*/x/
+do
+	echo "Found installation directory: $d"
+	INSTALLED=true
+done
+
+if ! $INSTALLED
 then
 	echo "Cabal build's --binddir did not work! Manually copying to ~/.local/bin/"
-	cp -p "dist-newstyle/build/x86_64-linux/ghc-8.10.2/cardano-node-$TAGGED_VERSION/x/cardano-node/build/cardano-node/cardano-node" ~/.local/bin/
-	cp -p "dist-newstyle/build/x86_64-linux/ghc-8.10.2/cardano-cli-$TAGGED_VERSION/x/cardano-cli/build/cardano-cli/cardano-cli" ~/.local/bin/
-fi
-
-# Verify node is installed
-echo "Verify node is installed"
-if [[ $(cardano-cli --version) == *"$TAGGED_VERSION"* ]]
-then
-	cardano-cli --version
-else
-	echo "$(cardano-cli --version) does not contain $TAGGED_VERSION"
-	exit 1
+	cp -p "$(./scripts/bin-path.sh cardano-node)" ~/.local/bin/
+	cp -p "$(./scripts/bin-path.sh cardano-cli)" ~/.local/bin/
 fi
 
+# Verify installation
+case $GIT_OBJECT_TYPE in
+	tag)
+		GIT_TAG=$GIT_OBJECT
+
+		# Verify cli is installed
+		echo "Verify cli is installed"
+		if [[ $(cardano-cli --version) == *"$GIT_TAG"* ]]
+		then
+			cardano-cli --version
+		else
+			>&2 echo "$(cardano-cli --version) does not contain $GIT_TAG"
+			exit 1
+		fi
+
+		# Verify node is installed
+		echo "Verify node is installed"
+		if [[ $(cardano-node --version) == *"$GIT_TAG"* ]]
+		then
+			cardano-node --version
+		else
+			>&2 echo "$(cardano-node --version) does not contain $GIT_TAG"
+			exit 1
+		fi
+
+		;;
+	commit)
+		# Verify cli is installed
+		echo "Verify cli is installed"
+		cardano-cli --version
+
+		# Verify node is installed
+		echo "Verify node is installed"
+		cardano-node --version
+
+		;;
+	*)
+		exit 1
+		;;
+esac
+
+# Succes message
 printf "\nSuccess\n\n"
 echo "You can use 'docker exec -it <container-id> /bin/bash' in a separate session to play in the environment."
 echo "Press any key to stop container."
diff --git a/cabal_build_tests/readme.md b/cabal_build_tests/readme.md
@@ -6,10 +6,14 @@
 ## Run tests
 
 ### To test cardano-node Cabal build on Ubuntu:
-`./run-ubuntu.sh <cardano-node tag>`
+`./run-ubuntu.sh -t tag -o <cardano-node tag>`
+
+`./run-ubuntu.sh -t commit -o <cardano-node commit>`
 
 ### To test cardano-node Cabal build on Fedora:
 
-`./run-fedora.sh <cardano-node tag>`
+`./run-fedora.sh  -t tag -o <cardano-node tag>`
+
+`./run-fedora.sh -t commit -o <cardano-node commit>`
 
 If "Success" is shown at the end and exit with code 0 then build has completed successfully and verified with cardano-cli.
diff --git a/cabal_build_tests/run-fedora.sh b/cabal_build_tests/run-fedora.sh
@@ -1,4 +1,17 @@
 #!/bin/bash
 
+GIT_OBJECT_TYPE=""
+GIT_OBJECT=""
+
+while getopts t:o: flag
+do
+    case "${flag}" in
+        t) GIT_OBJECT_TYPE=${OPTARG};;
+		o) GIT_OBJECT=${OPTARG};;
+        *) echo "Error in command line parsing" >&2
+		   exit 1
+    esac
+done
+
 docker build . -f fedora/Dockerfile -t cardano-node-fedora && \
-	docker run --security-opt label:disable -it -e TAGGED_VERSION="$1" cardano-node-fedora
+	docker run --security-opt label:disable -it -e GIT_OBJECT_TYPE="$GIT_OBJECT_TYPE" -e GIT_OBJECT="$GIT_OBJECT" cardano-node-fedora
diff --git a/cabal_build_tests/run-ubuntu.sh b/cabal_build_tests/run-ubuntu.sh
@@ -1,4 +1,17 @@
 #!/bin/bash
 
+GIT_OBJECT_TYPE=""
+GIT_OBJECT=""
+
+while getopts t:o: flag
+do
+    case "${flag}" in
+        t) GIT_OBJECT_TYPE=${OPTARG};;
+		o) GIT_OBJECT=${OPTARG};;
+        *) echo "Error in command line parsing" >&2
+		   exit 1
+    esac
+done
+
 docker build . -f ubuntu/Dockerfile -t cardano-node-ubuntu && \
-	docker run --security-opt label:disable -it -e TAGGED_VERSION="$1" cardano-node-ubuntu
+	docker run --security-opt label:disable -it -e GIT_OBJECT_TYPE="$GIT_OBJECT_TYPE" -e GIT_OBJECT="$GIT_OBJECT" cardano-node-ubuntu
