Merge branch 'main' into renovate/markupsafe-2.x

Author: alicejli

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -15,26 +15,68 @@
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
-# install tools
+ARG SYNTHTOOL_COMMITTISH=63cc541da2c45fcfca2136c43e638da1fbae174d
+ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
+ENV HOME=/home
+
+# install OS tools
 RUN apt-get update && apt-get install -y \
 	unzip openjdk-17-jdk rsync maven jq \
 	&& apt-get clean
 
-COPY library_generation /src
-
+# use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
-RUN cd /src && python -m pip install -r requirements.txt
-RUN cd /src && python -m pip install .
 
-# set dummy git credentials for empty commit used in postprocessing
-RUN git config --global user.email "[email protected]"
-RUN git config --global user.name "Cloud Java Bot"
+# copy source code
+COPY library_generation /src
 
-WORKDIR /workspace
-RUN chmod 750 /workspace
-RUN chmod 750 /src/generate_repo.py
+# install scripts as a python package
+WORKDIR /src
+RUN python -m pip install -r requirements.txt
+RUN python -m pip install .
+
+# install synthtool
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/synthtool
+WORKDIR /tools/synthtool
+RUN git checkout "${SYNTHTOOL_COMMITTISH}"
+RUN python3 -m pip install --no-deps -e .
+RUN python3 -m pip install -r requirements.in
 
-CMD [ "/src/generate_repo.py" ]
+# Install nvm with node and npm
+ENV NODE_VERSION 20.12.0
+WORKDIR /home
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+RUN chmod o+rx /home/.nvm
+ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin
+ENV PATH=${PATH}:${NODE_PATH}
+RUN node --version
+RUN npm --version
+
+# install the owl-bot CLI
+WORKDIR /tools
+RUN git clone https://github.com/googleapis/repo-automation-bots
+WORKDIR /tools/repo-automation-bots/packages/owl-bot
+RUN git checkout "${OWLBOT_CLI_COMMITTISH}"
+RUN npm i && npm run compile && npm link
+RUN owl-bot copy-code --version
+RUN chmod -R o+rx ${NODE_PATH}
+RUN ln -sf ${NODE_PATH}/* /usr/local/bin
+
+# allow users to access the script folders
+RUN chmod -R o+rx /src
+
+# set dummy git credentials for the empty commit used in postprocessing
+# we use system so all users using the container will use this configuration
+RUN git config --system user.email "[email protected]"
+RUN git config --system user.name "Cloud Java Bot"
+
+# allow read-write for /home and execution for binaries in /home/.nvm
+RUN chmod -R a+rw /home
+RUN chmod -R a+rx /home/.nvm
+
+WORKDIR /workspace
+ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]

.github/workflows/anaylze_dependency.yaml renamed to .github/workflows/analyze_dependency.yaml

@@ -25,7 +25,7 @@ jobs:
     - uses: actions/setup-java@v4
       with:
         distribution: temurin
-        java-version: 11
+        java-version: 17
         cache: maven
     - name: Set up Maven
       uses: stCarolas/[email protected]

.github/workflows/verify_library_generation.yaml

@@ -3,14 +3,38 @@ on:
     branches:
     - main
   pull_request:
-    paths:
-    - library_generation/**
 
   workflow_dispatch:
 name: verify_library_generation
 jobs:
-  integration_tests:
+  should-run-library-generation-tests:
     runs-on: ubuntu-22.04
+    outputs:
+      should_run: ${{ steps.get_changed_directories.outputs.should_run }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: get changed directories in the pull request
+      id: get_changed_directories
+      shell: bash
+      run: |
+        set -ex
+        git checkout "${base_ref}"
+        git checkout "${head_ref}"
+        changed_directories="$(git diff --name-only ${base_ref} ${head_ref})"
+        if [[ ${changed_directories} =~ "library_generation/" ]]; then
+          echo "should_run=true" >> $GITHUB_OUTPUT
+        else
+          echo "should_run=false" >> $GITHUB_OUTPUT
+        fi
+      env:
+        base_ref: ${{ github.event.pull_request.base.ref }}
+        head_ref: ${{ github.event.pull_request.head.ref }}
+  library-generation-integration-tests:
+    runs-on: ubuntu-22.04
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
@@ -41,12 +65,14 @@ jobs:
       run: |
         set -x
         python -m unittest library_generation/test/integration_tests.py
-  unit_tests:
+  library-generation-unit-tests:
+    runs-on: ${{ matrix.os }}
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     strategy:
       matrix:
         java: [ 8 ]
         os: [ ubuntu-22.04, macos-12 ]
-    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
     - name: install utils (macos)
@@ -88,8 +114,10 @@ jobs:
       run: |
         set -x
         python -m unittest discover -s library_generation/test/ -p "*unit_tests.py"
-  lint-shell:
+  library-generation-lint-shell:
     runs-on: ubuntu-22.04
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
     - name: Run ShellCheck
@@ -98,8 +126,10 @@ jobs:
         scandir: 'library_generation'
         format: tty
         severity: error
-  lint-python:
+  library-generation-lint-python:
     runs-on: ubuntu-22.04
+    needs: should-run-library-generation-tests
+    if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
     - name: install python dependencies

gapic-generator-java/pom.xml

@@ -351,7 +351,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.5.2</version>
+        <version>3.5.3</version>
         <executions>
           <execution>
             <phase>package</phase>

gax-java/dependencies.properties

@@ -83,5 +83,5 @@ maven.org_mockito_mockito_core=org.mockito:mockito-core:2.28.2
 maven.org_hamcrest_hamcrest_core=org.hamcrest:hamcrest-core:1.3
 maven.com_google_truth_truth=com.google.truth:truth:1.4.2
 maven.com_googlecode_java_diff_utils_diffutils=com.googlecode.java-diff-utils:diffutils:1.3.0
-maven.net_bytebuddy_byte_buddy=net.bytebuddy:byte-buddy:1.14.13
+maven.net_bytebuddy_byte_buddy=net.bytebuddy:byte-buddy:1.14.14
 maven.org_objenesis_objenesis=org.objenesis:objenesis:2.6

gax-java/gax/src/main/java/com/google/api/gax/core/InstantiatingExecutorProvider.java

@@ -34,13 +34,18 @@
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * InstantiatingChannelProvider is an ExecutorProvider which constructs a new
  * ScheduledExecutorService every time getExecutor() is called.
  */
 @AutoValue
 public abstract class InstantiatingExecutorProvider implements ExecutorProvider {
+  private static final Logger LOGGER =
+      Logger.getLogger(InstantiatingExecutorProvider.class.getName());
+
   // Thread factory to use to create our worker threads
   private static final ThreadFactory DEFAULT_THREAD_FACTORY =
       new ThreadFactory() {
@@ -95,6 +100,8 @@ public static Builder newBuilder() {
   public static Builder newIOBuilder() {
     int numCpus = Runtime.getRuntime().availableProcessors();
     int numThreads = IO_THREAD_MULTIPLIER * Math.max(MIN_THREAD_AMOUNT, numCpus);
+    LOGGER.log(
+        Level.CONFIG, String.format("Thread Pool for requests has Core Pool Size: %d", numThreads));
 
     return new AutoValue_InstantiatingExecutorProvider.Builder()
         .setExecutorThreadCount(numThreads)

library_generation/DEVELOPMENT.md

@@ -0,0 +1,171 @@
+> [!IMPORTANT]
+> All examples assume you are inside the `library_generation` folder
+
+
+# Linting
+
+When contributing, ensure your changes to python code have a valid
+format.
+
+```
+python -m pip install black
+black .
+```
+
+# Running the integration tests
+
+The integration tests build the docker image declared in
+`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC
+repositories, generate the libraries and compares the results with the source
+code declared in a "golden branch" of the repo.
+
+It requires docker and python 3.x to be installed.
+
+```
+python -m pip install .
+python -m pip install -r requirements.txt
+python -m unittest test/integration_tests.py
+```
+
+# Running the unit tests
+
+The unit tests of the hermetic build scripts are contained in several scripts,
+corresponding to a specific component. Every unit test script ends with
+`unit_tests.py`. To avoid them specifying them
+individually, we can use the following command:
+
+```bash
+python -m unittest discover -s test/ -p "*unit_tests.py"
+```
+
+> [!NOTE]
+> The output of this command may look erratic during the first 30 seconds.
+> This is normal. After the tests are done, an "OK" message should be shown.
+
+# Running the scripts in your local environment
+
+Although the scripts are designed to be run in a Docker container, you can also
+run them directly. This section explains how to run the entrypoint script
+(`library_generation/cli/entry_point.py`).
+
+## Installing prerequisites
+
+In order to run the generation scripts directly, there are a few tools we
+need to install beforehand.
+
+### Install synthtool
+
+It requires python 3.x to be installed.
+You will need to specify a committish of the synthtool repo in order to have
+your generation results matching exactly what the docker image would produce.
+You can achieve this by inspecting `SYNTHTOOL_COMMITISH` in
+`.cloudbuild/library_generation/library_generation.Dockerfile`.
+
+```bash
+# obtained from .cloudbuild/library_generation/library_generation.Dockerfile
+export SYNTHTOOL_COMMITTISH=6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
+```
+
+```bash
+git clone https://github.com/googleapis/synthtool
+cd synthtool
+git checkout "${SYNTHTOOL_COMMITTISH}"
+python -m pip install --require-hashes -r requirements.txt
+python -m pip install --no-deps -e .
+python -m synthtool --version
+```
+
+### Install the owl-bot CLI
+
+Requires node.js to be installed.
+Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script)
+for NVM, Node.js's version manager.
+
+After you install it, you can install the owl-bot CLI with the following
+commands:
+```bash
+git clone https://github.com/googleapis/repo-automation-bots
+cd repo-automation-bots/packages/owl-bot
+npm i && npm run compile && npm link
+owl-bot copy-code --version
+```
+
+The key step is `npm link`, which will make the command available in you current
+shell session.
+
+## Running the script
+The entrypoint script (`library_generation/cli/entry_point.py`) allows you to
+update the target repository with the latest changes starting from the
+googleapis committish declared in `generation_config.yaml`.
+
+### Download the repo
+For example, google-cloud-java
+```
+git clone https://github.com/googleapis/google-cloud-java
+export path_to_repo="$(pwd)/google-cloud-java"
+```
+
+### Install the scripts
+```
+python -m pip install .
+```
+
+### Run the script
+```
+python cli/entry_point.py generate --repository-path="${path_to_repo}"
+```
+
+
+# Running the scripts using the docker container image
+This is convenient in order to avoid installing the dependencies manually. 
+
+> [!IMPORTANT]
+> From now, the examples assume you are in the root of your sdk-platform-java
+> folder
+
+## Build the docker image
+```bash
+docker build --file .cloudbuild/library_generation/library_generation.Dockerfile --iidfile image-id .
+```
+
+This will create an `image-id` file at the root of the repo with the hash ID of
+the image.
+
+## Run the docker image
+The docker image will perform changes on its internal `/workspace` folder, to which you
+need to map a folder on your host machine (i.e. map your downloaded repo to this
+folder).
+
+To run the docker container on the google-cloud-java repo, you must run:
+```bash
+docker run -u "$(id -u)":"$(id -g)"  -v/path/to/google-cloud-java:/workspace $(cat image-id)
+```
+
+ * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating
+   yourself. This avoids folder ownership changes since it runs as root by
+   default.
+ * `-v/path/to/google-cloud-java:/workspace` maps the host machine's
+   google-cloud-java folder to the /workspace folder. The image is configured to
+   perform changes in this directory
+ * `$(cat image-id)` obtains the image ID created in the build step
+
+## Debug the created containers
+If you are working on changing the way the containers are created, you may want
+to inspect the containers to check the setup. It would be convenient in such
+case to have a text editor/viewer available. You can achieve this by modifying
+the Dockerfile as follows:
+
+```docker
+# install OS tools
+RUN apt-get update && apt-get install -y \
+	unzip openjdk-17-jdk rsync maven jq less vim \
+	&& apt-get clean
+```
+
+We add `less` and `vim` as text tools for further inspection.
+
+You can also run a shell in a new container by running:
+
+```bash
+docker run --rm -it -u=$(id -u):$(id -g)  -v/path/to/google-cloud-java:/workspace --entrypoint="bash" $(cat image-id)
+```