From a645805b18e228cf562f7db6923b1ad85daf1d8b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 13:52:12 -0400 Subject: [PATCH 01/11] chore: make generator version an optional param --- generation_config.yaml | 1 - library_generation/model/generation_config.py | 8 +++----- .../test/model/generation_config_unit_tests.py | 10 +--------- .../test-config/config_without_generator.yaml | 7 ------- .../config_without_libraries_bom_version.yaml | 9 --------- 5 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 library_generation/test/resources/test-config/config_without_generator.yaml delete mode 100644 library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml diff --git a/generation_config.yaml b/generation_config.yaml index d9439fb7db..e894663ac0 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,4 +1,3 @@ -gapic_generator_version: 2.42.1-SNAPSHOT # {x-version-update:gapic-generator-java:current} googleapis_commitish: 7160b0c31e9b99fe896d1fa29b6fe6cfbf42588f # the libraries are ordered with respect to library name, which is # java-{library.library_name} or java-{library.api-shortname} when diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 8a2b0829c4..c2be7a89ba 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -32,18 +32,18 @@ class GenerationConfig: def __init__( self, - gapic_generator_version: str, googleapis_commitish: str, libraries: list[LibraryConfig], + gapic_generator_version: Optional[str] = None, libraries_bom_version: Optional[str] = None, grpc_version: Optional[str] = None, protoc_version: Optional[str] = None, ): - self.gapic_generator_version = gapic_generator_version self.googleapis_commitish = googleapis_commitish self.libraries_bom_version = ( libraries_bom_version if libraries_bom_version else "" ) + self.gapic_generator_version = gapic_generator_version self.libraries = libraries self.grpc_version = grpc_version self.protoc_version = protoc_version @@ -147,12 +147,10 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig: parsed_libraries.append(new_library) parsed_config = GenerationConfig( - gapic_generator_version=__required( - config, GAPIC_GENERATOR_VERSION, REPO_LEVEL_PARAMETER - ), googleapis_commitish=__required( config, "googleapis_commitish", REPO_LEVEL_PARAMETER ), + gapic_generator_version=__optional(config, GAPIC_GENERATOR_VERSION, None), grpc_version=__optional(config, "grpc_version", None), protoc_version=__optional(config, "protoc_version", None), libraries_bom_version=__optional(config, LIBRARIES_BOM_VERSION, None), diff --git a/library_generation/test/model/generation_config_unit_tests.py b/library_generation/test/model/generation_config_unit_tests.py index 258b0db415..67ceba3062 100644 --- a/library_generation/test/model/generation_config_unit_tests.py +++ b/library_generation/test/model/generation_config_unit_tests.py @@ -47,11 +47,11 @@ class GenerationConfigTest(unittest.TestCase): def test_generation_config_default_value(self): config = GenerationConfig( - gapic_generator_version="", googleapis_commitish="", libraries=[], ) self.assertEqual("", config.libraries_bom_version) + self.assertIsNone(config.gapic_generator_version) def test_from_yaml_succeeds(self): config = from_yaml(f"{test_config_dir}/generation_config.yaml") @@ -160,14 +160,6 @@ def test_validate_with_duplicate_library_name_raise_exception(self): ], ) - def test_from_yaml_without_gapic_generator_version_raise_exception(self): - self.assertRaisesRegex( - ValueError, - "Repo level parameter, gapic_generator_version", - from_yaml, - f"{test_config_dir}/config_without_generator.yaml", - ) - def test_from_yaml_without_googleapis_commitish_raise_exception(self): self.assertRaisesRegex( ValueError, diff --git a/library_generation/test/resources/test-config/config_without_generator.yaml b/library_generation/test/resources/test-config/config_without_generator.yaml deleted file mode 100644 index c78b8b9700..0000000000 --- a/library_generation/test/resources/test-config/config_without_generator.yaml +++ /dev/null @@ -1,7 +0,0 @@ -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 diff --git a/library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml b/library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml deleted file mode 100644 index 7921f68bd2..0000000000 --- a/library_generation/test/resources/test-config/config_without_libraries_bom_version.yaml +++ /dev/null @@ -1,9 +0,0 @@ -gapic_generator_version: 2.34.0 -googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - api_description: "allows the Apigee hybrid management" - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 From c3a3ada0f9ae3a682760862a4bf84cd0f0b7054a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 15:55:22 -0400 Subject: [PATCH 02/11] get generator version from env --- .../scripts/hermetic_library_generation.sh | 4 ++++ library_generation/model/generation_config.py | 22 ++++++++++++++++++- .../model/generation_config_unit_tests.py | 20 ++++++++++++++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index cf29c87b3f..599ac42456 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -90,6 +90,8 @@ fi git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) +export generator_version=(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java) + # install generator locally since we're using a SNAPSHOT version. mvn -V -B -ntp clean install -DskipTests @@ -108,6 +110,8 @@ docker run \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" +unset generator_version + # commit the change to the pull request. rm -rdf output googleapis "${baseline_generation_config}" git add --all -- ':!pr_description.txt' diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index c2be7a89ba..04698d9517 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + import yaml from typing import Optional from library_generation.model.library_config import LibraryConfig @@ -23,6 +25,7 @@ COMMON_PROTOS_LIBRARY_NAME = "common-protos" GAPIC_GENERATOR_VERSION = "gapic_generator_version" LIBRARIES_BOM_VERSION = "libraries_bom_version" +GENERATOR_VERSION_ENV_KEY = "generator_version" class GenerationConfig: @@ -43,7 +46,9 @@ def __init__( self.libraries_bom_version = ( libraries_bom_version if libraries_bom_version else "" ) - self.gapic_generator_version = gapic_generator_version + self.gapic_generator_version = GenerationConfig.__set_generator_version( + gapic_generator_version + ) self.libraries = libraries self.grpc_version = grpc_version self.protoc_version = protoc_version @@ -76,6 +81,21 @@ def contains_common_protos(self) -> bool: break return self.__contains_common_protos + @staticmethod + def __set_generator_version(gapic_generator_version: Optional[str]) -> str: + if gapic_generator_version is not None: + return gapic_generator_version + # if the generator version is not set through generation config, + # get it from environment variable. + gapic_generator_version = os.getenv(GENERATOR_VERSION_ENV_KEY) + if not gapic_generator_version: + raise ValueError( + f"Environment variable {GENERATOR_VERSION_ENV_KEY}" + f" is not set when the generator version is not" + f" specified in the generation config." + ) + return gapic_generator_version + def __validate(self) -> None: seen_library_names = dict() for library in self.libraries: diff --git a/library_generation/test/model/generation_config_unit_tests.py b/library_generation/test/model/generation_config_unit_tests.py index 67ceba3062..e40b96815c 100644 --- a/library_generation/test/model/generation_config_unit_tests.py +++ b/library_generation/test/model/generation_config_unit_tests.py @@ -47,11 +47,29 @@ class GenerationConfigTest(unittest.TestCase): def test_generation_config_default_value(self): config = GenerationConfig( + gapic_generator_version="", googleapis_commitish="", libraries=[], ) self.assertEqual("", config.libraries_bom_version) - self.assertIsNone(config.gapic_generator_version) + + def test_generation_config_with_generator_version_env_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "Environment variable generator_version is not set", + GenerationConfig, + googleapis_commitish="", + libraries=[], + ) + + def test_generation_config_set_generator_version_from_env(self): + os.environ["generator_version"] = "1.2.3" + config = GenerationConfig( + googleapis_commitish="", + libraries=[], + ) + self.assertEqual("1.2.3", config.gapic_generator_version) + os.environ.pop("generator_version") def test_from_yaml_succeeds(self): config = from_yaml(f"{test_config_dir}/generation_config.yaml") From 485e023c80756fb4ad6a784c8a4fec942b0a9425 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 15:59:32 -0400 Subject: [PATCH 03/11] update config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index e894663ac0..8af5369287 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -7,7 +7,7 @@ libraries: name_pretty: Common Protos product_documentation: https://github.com/googleapis/api-common-protos api_description: Protobuf classes for Google's common protos. - release_level: stable + release_level: preview client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history distribution_name: com.google.api.grpc:proto-google-common-protos excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" From ea1c86734c00346b2493bf500dc1dc773be8fb29 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 15:59:38 -0400 Subject: [PATCH 04/11] update config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 8af5369287..e894663ac0 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -7,7 +7,7 @@ libraries: name_pretty: Common Protos product_documentation: https://github.com/googleapis/api-common-protos api_description: Protobuf classes for Google's common protos. - release_level: preview + release_level: stable client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history distribution_name: com.google.api.grpc:proto-google-common-protos excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" From d3d751d1b753be206220ba8d9d961ee33cde6b7a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 17:03:31 -0400 Subject: [PATCH 05/11] set env in docker run --- .github/scripts/hermetic_library_generation.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 599ac42456..62aa2bccbe 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -90,7 +90,7 @@ fi git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) -export generator_version=(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java) +generator_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java) # install generator locally since we're using a SNAPSHOT version. mvn -V -B -ntp clean install -DskipTests @@ -106,12 +106,11 @@ docker run \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ -v "$HOME"/.m2:/home/.m2 \ + -e generator_version="${generator_version}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" -unset generator_version - # commit the change to the pull request. rm -rdf output googleapis "${baseline_generation_config}" git add --all -- ':!pr_description.txt' From bb925e64fb7c7632cc80e7dce880e7a51e6fb201 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 17:03:54 -0400 Subject: [PATCH 06/11] update config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index e894663ac0..8af5369287 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -7,7 +7,7 @@ libraries: name_pretty: Common Protos product_documentation: https://github.com/googleapis/api-common-protos api_description: Protobuf classes for Google's common protos. - release_level: stable + release_level: preview client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history distribution_name: com.google.api.grpc:proto-google-common-protos excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" From ab229b4bc8248b7f91cab3da56674960ce364708 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 17:03:59 -0400 Subject: [PATCH 07/11] update config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 8af5369287..e894663ac0 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -7,7 +7,7 @@ libraries: name_pretty: Common Protos product_documentation: https://github.com/googleapis/api-common-protos api_description: Protobuf classes for Google's common protos. - release_level: preview + release_level: stable client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history distribution_name: com.google.api.grpc:proto-google-common-protos excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" From b08184f32d19dd46c3099b2d06a35be6e2926438 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 17:13:45 -0400 Subject: [PATCH 08/11] update config --- .github/scripts/hermetic_library_generation.sh | 1 + generation_config.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 62aa2bccbe..82b9674325 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -91,6 +91,7 @@ git show "${target_branch}":"${generation_config}" > "${baseline_generation_conf config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) generator_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java) +echo "Local generator version: ${generator_version}" # install generator locally since we're using a SNAPSHOT version. mvn -V -B -ntp clean install -DskipTests diff --git a/generation_config.yaml b/generation_config.yaml index e894663ac0..8af5369287 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -7,7 +7,7 @@ libraries: name_pretty: Common Protos product_documentation: https://github.com/googleapis/api-common-protos api_description: Protobuf classes for Google's common protos. - release_level: stable + release_level: preview client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history distribution_name: com.google.api.grpc:proto-google-common-protos excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" From 91cda154303d97c075bd0652a64c4ec4025219d5 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 17:13:50 -0400 Subject: [PATCH 09/11] update config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 8af5369287..e894663ac0 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -7,7 +7,7 @@ libraries: name_pretty: Common Protos product_documentation: https://github.com/googleapis/api-common-protos api_description: Protobuf classes for Google's common protos. - release_level: preview + release_level: stable client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history distribution_name: com.google.api.grpc:proto-google-common-protos excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" From dd989be9f576375d69d42e76d83297826ca0b712 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 10 Jul 2024 17:23:31 -0400 Subject: [PATCH 10/11] change readme --- library_generation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/README.md b/library_generation/README.md index 7ffe6a4a57..d3d78a8e1d 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -93,7 +93,7 @@ They are shared by library level parameters. | Name | Required | Notes | |:------------------------|:--------:|:---------------------------------------------| -| gapic_generator_version | Yes | | +| gapic_generator_version | No | set through env variable if not specified | | protoc_version | No | inferred from the generator if not specified | | grpc_version | No | inferred from the generator if not specified | | googleapis-commitish | Yes | | From fc4d48eef0e63f8914a0e6ce992598d590cb497f Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 25 Jul 2024 15:42:43 +0000 Subject: [PATCH 11/11] convert env var to uppercase --- .github/scripts/hermetic_library_generation.sh | 2 +- library_generation/model/generation_config.py | 2 +- .../test/model/generation_config_unit_tests.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 82b9674325..e69e8aa223 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -107,7 +107,7 @@ docker run \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ -v "$HOME"/.m2:/home/.m2 \ - -e generator_version="${generator_version}" \ + -e GENERATOR_VERSION="${generator_version}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index b40ef4fd81..bc3ff6d440 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -25,7 +25,7 @@ COMMON_PROTOS_LIBRARY_NAME = "common-protos" GAPIC_GENERATOR_VERSION = "gapic_generator_version" LIBRARIES_BOM_VERSION = "libraries_bom_version" -GENERATOR_VERSION_ENV_KEY = "generator_version" +GENERATOR_VERSION_ENV_KEY = "GENERATOR_VERSION" class GenerationConfig: diff --git a/library_generation/test/model/generation_config_unit_tests.py b/library_generation/test/model/generation_config_unit_tests.py index e40b96815c..f6a2f2a2d8 100644 --- a/library_generation/test/model/generation_config_unit_tests.py +++ b/library_generation/test/model/generation_config_unit_tests.py @@ -56,20 +56,20 @@ def test_generation_config_default_value(self): def test_generation_config_with_generator_version_env_raise_exception(self): self.assertRaisesRegex( ValueError, - "Environment variable generator_version is not set", + "Environment variable GENERATOR_VERSION is not set", GenerationConfig, googleapis_commitish="", libraries=[], ) def test_generation_config_set_generator_version_from_env(self): - os.environ["generator_version"] = "1.2.3" + os.environ["GENERATOR_VERSION"] = "1.2.3" config = GenerationConfig( googleapis_commitish="", libraries=[], ) self.assertEqual("1.2.3", config.gapic_generator_version) - os.environ.pop("generator_version") + os.environ.pop("GENERATOR_VERSION") def test_from_yaml_succeeds(self): config = from_yaml(f"{test_config_dir}/generation_config.yaml")