Merge branch 'main' into update_retrysettings_javadoc

Author: lqiu96

.github/release-please.yml

@@ -2,4 +2,5 @@ releaseType: java-yoshi
 bumpMinorPreMajor: true
 handleGHRelease: true
 primaryBranch: main
+manifest: true
 extraFiles: ["WORKSPACE"]

.github/workflows/ci.yaml

@@ -4,8 +4,6 @@ on:
       - main
   pull_request:
 name: ci
-env:
-  SHOWCASE_VERSION: 0.28.2
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -190,12 +188,14 @@ jobs:
             -P enable-golden-tests \
             --batch-mode \
             --no-transfer-progress
-
+      - name: Parse showcase version
+        working-directory: showcase/gapic-showcase
+        run: echo "SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)" >> "$GITHUB_ENV"
       - name: Install showcase server
         run: |
           sudo mkdir -p /usr/src/showcase
           sudo chown -R ${USER} /usr/src/
-          curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
+          curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${{env.SHOWCASE_VERSION}}/gapic-showcase-${{env.SHOWCASE_VERSION}}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${{env.SHOWCASE_VERSION}}-linux-amd64.tar.gz
           cd /usr/src/showcase/
           tar -xf showcase-*
           ./gapic-showcase run &
@@ -222,11 +222,14 @@ jobs:
       - run: native-image --version
       - name: Install sdk-platform-java
         run: mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
+      - name: Parse showcase version
+        working-directory: showcase/gapic-showcase
+        run: echo "SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)" >> "$GITHUB_ENV"
       - name: Install showcase server
         run: |
           sudo mkdir -p /usr/src/showcase
           sudo chown -R ${USER} /usr/src/
-          curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
+          curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${{env.SHOWCASE_VERSION}}/gapic-showcase-${{env.SHOWCASE_VERSION}}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${{env.SHOWCASE_VERSION}}-linux-amd64.tar.gz
           cd /usr/src/showcase/
           tar -xf showcase-*
           ./gapic-showcase run &
@@ -267,7 +270,7 @@ jobs:
           mvn clirr:check -B -ntp -Dclirr.skip=false -DcomparisonVersion=$SHOWCASE_CLIENT_VERSION
 
   gapic-generator-java-bom:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-java@v3
@@ -279,6 +282,7 @@ jobs:
       run: |
         mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
     - name: Validate gapic-generator-java-bom
-      uses: googleapis/java-cloud-bom/tests/validate-bom@v26.13.0
+      uses: googleapis/java-cloud-bom/tests/validate-bom@41d86dba28c2e724126568b4414ceae6a48e2a49
       with:
         bom-path: gapic-generator-java-bom/pom.xml
+

.github/workflows/downstream.yaml

@@ -0,0 +1,49 @@
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    # Changes to these directories do not directly affect the downstream libraries
+    paths-ignore:
+      - gapic-generator-java/**
+      - library_generation/**
+      - showcase/**
+      - test/**
+
+name: Downstream Check
+jobs:
+  downstream-compatibility:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        repo:
+          - google-cloud-java
+          - java-bigtable
+          - java-bigquery
+          - java-bigquerystorage
+          - java-datastore
+          - java-firestore
+          - java-logging
+          - java-logging-logback
+          - java-pubsub
+          - java-pubsublite
+          - java-spanner-jdbc
+          - java-spanner
+          - java-storage
+          - java-storage-nio
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: temurin
+      - run: mvn -version
+      - name: Install xmllint
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install libxml2-utils
+      - name: Test helper scripts
+        run: ./.kokoro/presubmit/common_test.sh
+      - name: Perform downstream compatibility testing
+        run: REPOS_UNDER_TEST="${{ matrix.repo }}" ./.kokoro/presubmit/downstream-compatibility.sh

.github/workflows/sonar.yaml

@@ -1,6 +1,4 @@
 name: SonarCloud Build
-env:
-  SHOWCASE_VERSION: 0.28.2
 on:
   push:
     branches:
@@ -15,10 +13,10 @@ jobs:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
-          java-version: 11
+          java-version: 17
           distribution: temurin
       - name: Cache SonarCloud packages
         uses: actions/cache@v3
@@ -32,11 +30,19 @@ jobs:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
+      - name: Install modules to local maven
+        run: |
+          mvn install -T 1C -DskipTests -ntp -B
+          cd showcase
+          mvn install -T 1C -DskipTests -ntp -B
+      - name: Parse showcase version
+        working-directory: showcase/gapic-showcase
+        run: echo "SHOWCASE_VERSION=$(mvn help:evaluate -Dexpression=gapic-showcase.version -q -DforceStdout)" >> "$GITHUB_ENV"
       - name: Install showcase server
         run: |
             sudo mkdir -p /usr/src/showcase
             sudo chown -R ${USER} /usr/src/
-            curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
+            curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${{env.SHOWCASE_VERSION}}/gapic-showcase-${{env.SHOWCASE_VERSION}}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${{env.SHOWCASE_VERSION}}-linux-amd64.tar.gz
             cd /usr/src/showcase/
             tar -xf showcase-*
             ./gapic-showcase run &
@@ -68,15 +74,3 @@ jobs:
               -Dsonar.organization=googleapis \
               -Dsonar.host.url=https://sonarcloud.io \
               -Dsonar.projectName=java_showcase_integration_tests
-      - name: Build and analyze Showcase Unit Tests Coverage
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        run: |
-          mvn -B clean test -Dcheckstyle.skip \
-              -DenableShowcaseTestCoverage \
-              org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \
-              -Dsonar.projectKey=googleapis_gapic-generator-java_unit_tests \
-              -Dsonar.organization=googleapis \
-              -Dsonar.host.url=https://sonarcloud.io \
-              -Dsonar.projectName=java_showcase_unit_tests

.github/workflows/verify_library_generation.yaml

@@ -0,0 +1,51 @@
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    paths:
+    - library_generation/**
+
+  workflow_dispatch:
+name: verify_library_generation
+jobs:
+  integration_tests:
+    strategy:
+      matrix:
+        java: [ 8 ]
+        os: [ ubuntu-22.04, macos-12 ]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-java@v3
+      with:
+        java-version: ${{ matrix.java }}
+        distribution: temurin
+        cache: maven
+    - name: Run integration tests
+      run: |
+        set -x
+        library_generation/test/generate_library_integration_test.sh \
+        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git
+  unit_tests:
+    strategy:
+      matrix:
+        java: [ 8 ]
+        os: [ ubuntu-22.04, macos-12 ]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Run unit tests
+      run: |
+        set -x
+        library_generation/test/generate_library_unit_tests.sh
+  lint:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Run ShellCheck
+      uses: ludeeus/[email protected]
+      with:
+        scandir: 'library_generation'
+        format: tty
+        severity: error

.gitignore

@@ -15,4 +15,9 @@ target/
 # Vscode Settings
 .vscode/settings.json
 
-*.iml
+*.iml
+
+# library generation
+output/
+library_generation/output/
+showcase/scripts/output/

.kokoro/presubmit/common.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# In the given directory ($1),
+#   update the pom.xml's dependency on the given artifact ($2) to the given version ($3)
+# ex: update_dependency google-cloud-java/google-cloud-jar-parent google-cloud-shared-dependencies 1.2.3
+function update_pom_dependency {
+  pushd "$1" || exit 1
+  xmllint --shell pom.xml &>/dev/null <<EOF
+setns x=http://maven.apache.org/POM/4.0.0
+cd .//x:artifactId[text()="$2"]
+cd ../x:version
+set $3
+save pom.xml
+EOF
+  popd || exit 1
+}
+
+# Find all pom.xml files that declare a specific version for the given artifact ($1)
+function find_all_poms_with_versioned_dependency {
+  poms=($(find . -name pom.xml))
+  for pom in "${poms[@]}"; do
+    if xmllint --xpath "//*[local-name()='artifactId' and text()='$1']/following-sibling::*[local-name()='version']" "$pom" &>/dev/null; then
+      found+=("$pom")
+    fi
+  done
+  POMS=(${found[@]})
+  unset found
+  export POMS
+}
+
+# In the given directory ($1),
+#   find and update all pom.xmls' dependencies on the given artifact ($2) to the given version ($3)
+# ex: update_all_poms_dependency google-cloud-java google-cloud-shared-dependencies 1.2.3
+function update_all_poms_dependency {
+  pushd "$1" || exit 1
+  find_all_poms_with_versioned_dependency "$2"
+  for pom in $POMS; do
+    update_pom_dependency "$(dirname "$pom")" "$2" "$3"
+  done
+  git diff
+  popd || exit 1
+}
+
+# Parse the version of the pom.xml file in the given directory ($1)
+# ex: VERSION=$(parse_pom_version java-shared-dependencies)
+function parse_pom_version {
+  # Namespace (xmlns) prevents xmllint from specifying tag names in XPath
+  result=$(sed -e 's/xmlns=".*"//' "$1/pom.xml" | xmllint --xpath '/project/version/text()' -)
+
+  if [ -z "${result}" ]; then
+    echo "Version is not found in $1"
+    exit 1
+  fi
+  echo "$result"
+}
+
+# ex: find_last_release_version java-bigtable
+# ex: find_last_release_version java-storage 2.22.x
+function find_last_release_version {
+  branch=${2-"main"} # Default to using main branch
+  curl -s -o "versions_$1.txt" "https://raw.githubusercontent.com/googleapis/$1/$branch/versions.txt"
+
+  # First check to see if there's an entry for the overall repo. Used for google-cloud-java.
+  primary_artifact=$(grep -E "^$1" "versions_$1.txt" | head -n 1)
+  if [ -z "$primary_artifact" ]; then
+    # Otherwise, use the first google-cloud-* artifact's version.
+    primary_artifact=$(grep -E "^google-cloud-" "versions_$1.txt" | head -n 1)
+  fi
+  if [ -z "$primary_artifact" ]; then
+    echo "Unable to identify primary artifact for $1"
+    exit 1
+  fi
+
+  parts=($(echo "$primary_artifact" | tr ":" "\n"))
+  echo "${parts[1]}"
+}