chore: add docker image for hermetic build scripts (#2493)

Similar changes to those of
#2298 by @mpeddada1

This setup uses two triggers:
- The first one is
[library-generation-presubmit-sdk-platform-java](https://pantheon.corp.google.com/cloud-build/triggers;region=global/edit/5783744f-0820-419f-bc5e-abbbece4be0b?e=13803378&mods=monitoring_api_prod&project=cloud-devrel-kokoro-resources),
which runs on each pull request. It builds a docker image with the
contents of `library_generation` at HEAD and runs
`library_generation/integration_tests.py` with such image
- The second one is
[library-generation-sdk-platform-java](https://pantheon.corp.google.com/cloud-build/triggers;region=global/edit/e3373892-82a2-4eac-a1f2-95523966df70?e=13803378&mods=monitoring_api_prod&project=cloud-devrel-kokoro-resources),
which:
   - is triggered upon a commit pushed to the `main` branch, then
   - builds a docker image with two tags
- `latest`, which will be constantly updated to match the latest build
     - a tag based on the branch `${COMMIT_SHA}`
   - then the image is pushed with both tags



### tasks

- [x] create dockerfile
- [x] create cloudbuild.yaml for testing
- [x] create cloudbuild test infra
- [x] create cloudbuild.yaml for releasing the image
- [x] create cloudbuild release infra

---------

Co-authored-by: Joe Wang <[email protected]>

Author: 2 people

.cloudbuild/cloudbuild-library-generation.yaml

@@ -0,0 +1,33 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 7200s # 2 hours
+substitutions:
+  _IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation"
+  _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}"
+  _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest"
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: [
+      "build",
+      "-t", "${_SHA_IMAGE_ID}",
+      "-t", "${_LATEST_IMAGE_ID}",
+      "--file", ".cloudbuild/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+
+images:
+  - ${_SHA_IMAGE_ID}
+  - ${_LATEST_IMAGE_ID}

.cloudbuild/cloudbuild-test-library-generation.yaml

@@ -0,0 +1,31 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 7200s # 2 hours
+substitutions:
+  _TEST_IMAGE_ID: 'gcr.io/cloud-devrel-public-resources/java-library-generation:${COMMIT_SHA}'
+
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: ["build", "-t", "${_TEST_IMAGE_ID}", "--file", ".cloudbuild/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+  - name: ${_TEST_IMAGE_ID}
+    entrypoint: bash
+    args: [ './library_generation/test/container_integration_tests.sh' ]
+    waitFor: [ "library-generation-build" ]
+    env:
+      - 'TEST_IMAGE_ID=${_TEST_IMAGE_ID}'
+

.cloudbuild/library_generation.Dockerfile

@@ -0,0 +1,40 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# build from the root of this repo:
+FROM gcr.io/cloud-devrel-public-resources/python
+
+# install tools
+RUN apt-get update && apt-get install -y \
+	unzip openjdk-17-jdk rsync maven \
+	&& apt-get clean
+
+COPY library_generation /src
+
+RUN rm $(which python3)
+RUN ln -s $(which python3.11) /usr/local/bin/python
+RUN ln -s $(which python3.11) /usr/local/bin/python3
+RUN python -m pip install --upgrade pip
+RUN cd /src && python -m pip install -r requirements.in
+RUN cd /src && python -m pip install .
+
+# set dummy git credentials for empty commit used in postprocessing
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Cloud Java Bot"
+
+WORKDIR /workspace
+RUN chmod 750 /workspace
+RUN chmod 750 /src/generate_repo.py
+
+CMD [ "/src/generate_repo.py" ]

library_generation/dockerignore

@@ -0,0 +1,6 @@
+README.md
+**/__pycache__/
+**/*.egg-info/
+**/output/
+**/build/
+**/google-cloud-java/

library_generation/generate_repo.py

@@ -61,22 +61,48 @@ fi
 
 # we determine the location of the .OwlBot.yaml file by checking if the target
 # folder is a monorepo folder or not
-if [[ "${postprocessing_target}" == *google-cloud-java* ]]; then
+if [[ "${is_monorepo}" == "true" ]]; then
   owlbot_yaml_relative_path=".OwlBot.yaml"
 else
   owlbot_yaml_relative_path=".github/.OwlBot.yaml"
 fi
 
+# Default values for running copy-code directly from host
+repo_binding="${postprocessing_target}"
+repo_workspace="/repo"
+preprocessed_libraries_binding="${owlbot_cli_source_folder}"
+
+# When running docker inside docker, we run into the issue of volume bindings
+# being mapped from the host machine to the child container (instead of the
+# parent container to child container) because we bind the `docker.sock` socket
+# to the parent container (i.e. docker calls use the host's filesystem context)
+# see https://serverfault.com/a/819371
+# We solve this by referencing environment variables that will be
+# set to produce the correct volume mapping.
+#
+# The workflow is: to check if we are in a docker container (via passed env var)
+# and use managed volumes (docker volume create) instead of bindings
+# (-v /path:/other-path). The volume names are also received as env vars.
+
+if [[ -n "${RUNNING_IN_DOCKER}" ]]; then
+  set -u # temporarily fail on unset variables
+  repo_binding="${REPO_BINDING_VOLUME}"
+  set +u
+  if [[ "${is_monorepo}" == "true" ]]; then
+    repo_workspace="/repo/$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev)"
+  fi
+fi
+
 docker run --rm \
   --user "$(id -u)":"$(id -g)" \
-  -v "${postprocessing_target}:/repo" \
-  -v "${owlbot_cli_source_folder}:/pre-processed-libraries" \
-  -w /repo \
+  -v "${repo_binding}:/repo" \
+  -v "/tmp:/tmp" \
+  -w "${repo_workspace}" \
   --env HOME=/tmp \
   gcr.io/cloud-devrel-public-resources/owlbot-cli@"${owlbot_cli_image_sha}" \
   copy-code \
   --source-repo-commit-hash=none \
-  --source-repo=/pre-processed-libraries \
+  --source-repo="${preprocessed_libraries_binding}" \
   --config-file="${owlbot_yaml_relative_path}"
 
 # we clone the synthtool library and manually build it
@@ -86,6 +112,7 @@ pushd /tmp/synthtool
 if [ ! -d "synthtool" ]; then
   git clone https://github.com/googleapis/synthtool.git
 fi
+git config --global --add safe.directory /tmp/synthtool/synthtool
 pushd "synthtool"
 
 git reset --hard "${synthtool_commitish}"

library_generation/postprocess_library.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# This is a wrapper of integration_tests.py that sets up the environment to run
+# the script in a docker container
+
+set -xe
+if [[ -z "${TEST_IMAGE_ID}" ]]; then
+  echo "required environemnt variable TEST_IMAGE_ID is not set"
+  exit 1
+fi
+
+if [[ ! -d google-cloud-java ]]; then
+  git clone https://github.com/googleapis/google-cloud-java
+fi
+pushd google-cloud-java
+git reset --hard main
+popd
+
+# We use a volume to hold the google-cloud-java repository used in the
+# integration tests. This is because the test container creates a child
+# container using the host machine's docker socket, meaning that we can only
+# reference volumes created from within the host machine (i.e. the machine
+# running this script)
+#
+# To summarize, we create a special volume that can be referenced both in the
+# main container and in any child containers created by this one.
+if [[ $(docker volume inspect repo) != '[]' ]]; then
+  docker volume rm repo
+fi
+docker volume create --name "repo" --opt "type=none" --opt "device=$(pwd)/google-cloud-java" --opt "o=bind"
+
+docker run --rm \
+  -v repo:/workspace \
+  -v /tmp:/tmp \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e "RUNNING_IN_DOCKER=true" \
+  -e "REPO_BINDING_VOLUME=repo" \
+  -w "/src" \
+  "${TEST_IMAGE_ID}" \
+  python -m unittest /src/test/integration_tests.py

library_generation/setup.py

@@ -26,8 +26,11 @@
 from library_generation.generate_repo import generate_from_yaml
 from library_generation.model.generation_config import from_yaml
 from library_generation.test.compare_poms import compare_xml
-from library_generation.utilities import get_library_name
-from library_generation.utilities import sh_util as shell_call
+from library_generation.utilities import (
+    get_library_name,
+    sh_util as shell_call,
+    run_process_and_print_output,
+)
 
 config_name = "generation_config.yaml"
 script_dir = os.path.dirname(os.path.realpath(__file__))
@@ -48,8 +51,9 @@ def test_generate_repo(self):
         config_files = self.__get_config_files(config_dir)
         i = 0
         for repo, config_file in config_files.items():
-            repo_dest = f"{output_folder}/{repo}"
-            self.__pull_repo_to(Path(repo_dest), repo, committish_list[i])
+            repo_dest = self.__pull_repo_to(
+                Path(f"{golden_dir}/{repo}"), repo, committish_list[i]
+            )
             library_names = self.__get_library_names_from_config(config_file)
             # prepare golden files
             for library_name in library_names:
@@ -109,11 +113,31 @@ def test_generate_repo(self):
             i += 1
 
     @classmethod
-    def __pull_repo_to(cls, dest: Path, repo: str, committish: str):
-        repo_url = f"{repo_prefix}/{repo}"
-        print(f"Cloning repository {repo_url}")
-        repo = Repo.clone_from(repo_url, dest)
+    def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str:
+        if "RUNNING_IN_DOCKER" in os.environ:
+            # the docker image expects the repo to be in /workspace
+            dest_in_docker = "/workspace"
+            run_process_and_print_output(
+                [
+                    "git",
+                    "config",
+                    "--global",
+                    "--add",
+                    "safe.directory",
+                    dest_in_docker,
+                ],
+                "Add /workspace to safe directories",
+            )
+            dest = Path(dest_in_docker)
+            repo = Repo(dest)
+        else:
+            dest = default_dest
+            repo_dest = f"{golden_dir}/{repo}"
+            repo_url = f"{repo_prefix}/{repo}"
+            print(f"Cloning repository {repo_url}")
+            repo = Repo.clone_from(repo_url, dest)
         repo.git.checkout(committish)
+        return str(dest)
 
     @classmethod
     def __get_library_names_from_config(cls, config_path: str) -> List[str]: