Skip to content

Commit 5c95472

Browse files
diegomarquezpJoeWang1127
diegomarquezp
and
JoeWang1127
authored
feat: move synthtool templates to library_generation/owlbot (#2443)
This PR transfers the java-specific templates from synthtool to `library_geneation/owlbot` There is a new branch in synthtool (googleapis/synthtool#1923) that has the removed templates. The intention is to keep that synthtool branch as parallel until we fully roll out the hermetic build workflows to both HW libraries and the monorepo. ### Approach We add a list of template exclusions to the configuration yaml and call `java.common_templates` with a custom template path (pointing to our OwlBot Java Postprocessor implementation here in `library_generation` plus the template exclusions found in the yaml. The list of exclusions were obtained from owlbot.py files. Since google-cloud-java has the same exclusions for all libraries, we use a repo-level configuration entry. An example of template excludes is: https://github.com/googleapis/sdk-platform-java/blob/b0a57b70d589e2bdc6e5fcb8cf64d08c3496bc57/java-iam/owlbot.py#L26-L37 ### Different approach possible? We could modify all owlbot.py files to use something other than `java.common_templates`. For example ``` from custom_owlbot import common_templates ... common_templates(excludes=[/*preserved excludes*/]) ``` With such approach, we would not have to parse owlbot.py files and take advantage of the fact it's an executable script. ## Follow up? We probably don't want to call `common_templates` twice, so it may be better to modify owlbot.py files to reference our own implementation instead of synthtool. (This is similar to "Different approach possible?" but it is more of a follow up once the scripts are live). --------- Co-authored-by: Joe Wang <[email protected]>
1 parent dba4c10 commit 5c95472

26 files changed

+1273
-75
lines changed

‎.github/snippet-bot.yml

+1
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ ignoreFiles:
44
- src/test/**
55
- test/**
66
- showcase/**
7+
- library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml

‎.github/workflows/verify_library_generation.yaml

+17-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ jobs:
4242
set -ex
4343
pushd library_generation
4444
pip install -r requirements.in
45+
pip install .
4546
popd
4647
- name: Run integration tests
4748
shell: bash
@@ -74,6 +75,21 @@ jobs:
7475
pushd library_generation
7576
pip install -r requirements.in
7677
popd
78+
- name: install synthtool
79+
shell: bash
80+
run: |
81+
set -ex
82+
mkdir -p /tmp/synthtool
83+
pushd /tmp/synthtool
84+
if [ ! -d "synthtool" ]; then
85+
git clone https://github.com/googleapis/synthtool.git
86+
fi
87+
pushd "synthtool"
88+
89+
git reset --hard origin/no-java-templates
90+
91+
python3 -m pip install -e .
92+
python3 -m pip install -r requirements.in
7793
- name: Run shell unit tests
7894
run: |
7995
set -x
@@ -108,4 +124,4 @@ jobs:
108124
run: |
109125
# exclude generated golden files
110126
# exclude owlbot until further refaction
111-
black --check library_generation --exclude "(library_generation/owlbot)|(library_generation/test/resources/goldens)"
127+
black --check library_generation --exclude "(library_generation/test/resources/goldens)"

‎library_generation/generate_composed_library.py

+1
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ def generate_composed_library(
113113
config.owlbot_cli_image,
114114
config.synthtool_commitish,
115115
str(is_monorepo).lower(),
116+
config.path_to_yaml,
116117
],
117118
"Library postprocessing",
118119
)

‎library_generation/model/generation_config.py

+6
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ def __init__(
3131
googleapis_commitish: str,
3232
owlbot_cli_image: str,
3333
synthtool_commitish: str,
34+
template_excludes: str,
35+
path_to_yaml: str,
3436
libraries: List[LibraryConfig],
3537
grpc_version: Optional[str] = None,
3638
protobuf_version: Optional[str] = None,
@@ -39,6 +41,8 @@ def __init__(
3941
self.googleapis_commitish = googleapis_commitish
4042
self.owlbot_cli_image = owlbot_cli_image
4143
self.synthtool_commitish = synthtool_commitish
44+
self.template_excludes = template_excludes
45+
self.path_to_yaml = path_to_yaml
4246
self.libraries = libraries
4347
self.grpc_version = grpc_version
4448
self.protobuf_version = protobuf_version
@@ -94,6 +98,8 @@ def from_yaml(path_to_yaml: str):
9498
googleapis_commitish=__required(config, "googleapis_commitish"),
9599
owlbot_cli_image=__required(config, "owlbot_cli_image"),
96100
synthtool_commitish=__required(config, "synthtool_commitish"),
101+
template_excludes=__required(config, "template_excludes"),
102+
path_to_yaml=path_to_yaml,
97103
libraries=parsed_libraries,
98104
)
99105

‎library_generation/owlbot/bin/entrypoint.sh

+47-39
Original file line numberDiff line numberDiff line change
@@ -26,50 +26,14 @@
2626
set -ex
2727
scripts_root=$1
2828
versions_file=$2
29-
30-
# Runs template and etc in current working directory
31-
function processModule() {
32-
# templates as well as retrieving files from owl-bot-staging
33-
echo "Generating templates and retrieving files from owl-bot-staging directory..."
34-
if [ -f "owlbot.py" ]
35-
then
36-
# defaults to run owlbot.py
37-
python3 owlbot.py
38-
fi
39-
echo "...done"
40-
41-
# write or restore pom.xml files
42-
echo "Generating missing pom.xml..."
43-
python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
44-
echo "...done"
45-
46-
# write or restore clirr-ignored-differences.xml
47-
echo "Generating clirr-ignored-differences.xml..."
48-
${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
49-
echo "...done"
50-
51-
# fix license headers
52-
echo "Fixing missing license headers..."
53-
python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
54-
echo "...done"
55-
56-
# TODO: re-enable this once we resolve thrashing
57-
# restore license headers years
58-
# echo "Restoring copyright years..."
59-
# /owlbot/bin/restore_license_headers.sh
60-
# echo "...done"
61-
62-
# ensure formatting on all .java files in the repository
63-
echo "Reformatting source..."
64-
mvn fmt:format -q
65-
echo "...done"
66-
}
29+
configuration_yaml=$3
6730

6831
# This script can be used to process HW libraries and monorepo
6932
# (google-cloud-java) libraries, which require a slightly different treatment
7033
# monorepo folders have an .OwlBot.yaml file in the module folder (e.g.
7134
# java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in
7235
# `.github/.OwlBot.yaml`
36+
monorepo="false"
7337
if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then
7438
monorepo="true"
7539
fi
@@ -80,4 +44,48 @@ if [[ "${monorepo}" == "true" ]]; then
8044
mv temp owl-bot-staging
8145
fi
8246

83-
processModule
47+
48+
# Runs template and etc in current working directory
49+
monorepo=$1
50+
51+
# apply repo templates
52+
echo "Rendering templates"
53+
python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}"
54+
55+
# templates as well as retrieving files from owl-bot-staging
56+
echo "Retrieving files from owl-bot-staging directory..."
57+
if [ -f "owlbot.py" ]
58+
then
59+
# defaults to run owlbot.py
60+
python3 owlbot.py
61+
fi
62+
echo "...done"
63+
64+
# write or restore pom.xml files
65+
echo "Generating missing pom.xml..."
66+
python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
67+
echo "...done"
68+
69+
# write or restore clirr-ignored-differences.xml
70+
echo "Generating clirr-ignored-differences.xml..."
71+
${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
72+
echo "...done"
73+
74+
# fix license headers
75+
echo "Fixing missing license headers..."
76+
python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
77+
echo "...done"
78+
79+
# TODO: re-enable this once we resolve thrashing
80+
# restore license headers years
81+
# echo "Restoring copyright years..."
82+
# /owlbot/bin/restore_license_headers.sh
83+
# echo "...done"
84+
85+
# ensure formatting on all .java files in the repository
86+
echo "Reformatting source..."
87+
mvn fmt:format
88+
echo "...done"
89+
90+
91+

‎library_generation/owlbot/src/apply_repo_templates.py

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""
2+
This script parses an owlbot.py file, specifically the call to `java.common_templates` in
3+
order to extract the excluded files so it can be called with a custom template path
4+
pointing to the templates hosted in `sdk-platform-java/library_generation/owlbot/templates`.
5+
Briefly, this wraps the call to synthtool's common templates using a custom template folder.
6+
"""
7+
8+
import os
9+
import sys
10+
from collections.abc import Sequence
11+
from synthtool.languages.java import common_templates
12+
from pathlib import Path
13+
from library_generation.model.generation_config import from_yaml
14+
15+
script_dir = os.path.dirname(os.path.realpath(__file__))
16+
repo_templates_path = os.path.join(script_dir, "..", "templates", "java_library")
17+
18+
19+
def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None:
20+
config = from_yaml(configuration_yaml_path)
21+
print(f"repo_templates_path: {repo_templates_path}")
22+
print(f"excludes: {config.template_excludes}")
23+
common_templates(
24+
excludes=config.template_excludes,
25+
template_path=Path(repo_templates_path),
26+
monorepo=monorepo,
27+
)
28+
29+
30+
def main(argv: Sequence[str]) -> None:
31+
if len(argv) != 3:
32+
raise ValueError(
33+
"Usage: python apply-repo-templates.py configuration_yaml_path monorepo"
34+
)
35+
36+
configuration_yaml_path = argv[1]
37+
monorepo = argv[2]
38+
apply_repo_templates(configuration_yaml_path, monorepo.lower() == "true")
39+
40+
41+
if __name__ == "__main__":
42+
main(sys.argv)

‎library_generation/owlbot/src/fix-poms.py

+44-28
Original file line numberDiff line numberDiff line change
@@ -285,9 +285,11 @@ def update_bom_pom(filename: str, modules: List[module.Module]):
285285
# https://github.com/googleapis/google-cloud-java/issues/9304
286286
def __proto_group_id(main_artifact_group_id: str) -> str:
287287
prefix = "com.google"
288-
list_of_group_id = ["com.google.cloud",
289-
"com.google.area120",
290-
"com.google.analytics"]
288+
list_of_group_id = [
289+
"com.google.cloud",
290+
"com.google.area120",
291+
"com.google.analytics",
292+
]
291293
if main_artifact_group_id not in list_of_group_id:
292294
prefix = main_artifact_group_id
293295
return f"{prefix}.api.grpc"
@@ -343,11 +345,17 @@ def main(versions_file, monorepo):
343345
main_module = existing_modules[artifact_id]
344346

345347
# Artifact ID is part of distribution name field in .repo-metadata.json
346-
if artifact_id in ["grafeas", "google-cloud-dns",
347-
"google-cloud-notification", "google-iam-policy"]:
348+
if artifact_id in [
349+
"grafeas",
350+
"google-cloud-dns",
351+
"google-cloud-notification",
352+
"google-iam-policy",
353+
]:
348354
# There are special libraries that are not automatically generated
349-
print(f"Skipping a special case library {artifact_id} that do not have "
350-
" the standard module structure.")
355+
print(
356+
f"Skipping a special case library {artifact_id} that do not have "
357+
" the standard module structure."
358+
)
351359
return
352360

353361
parent_artifact_id = f"{artifact_id}-parent"
@@ -383,8 +391,10 @@ def main(versions_file, monorepo):
383391
version=main_module.version,
384392
release_version=main_module.release_version,
385393
)
386-
if path not in excluded_dependencies_list \
387-
and path not in main_module.artifact_id:
394+
if (
395+
path not in excluded_dependencies_list
396+
and path not in main_module.artifact_id
397+
):
388398
required_dependencies[path] = module.Module(
389399
group_id=__proto_group_id(group_id),
390400
artifact_id=path,
@@ -401,8 +411,10 @@ def main(versions_file, monorepo):
401411
main_module=main_module,
402412
monorepo=monorepo,
403413
)
404-
if path not in excluded_dependencies_list \
405-
and path not in main_module.artifact_id:
414+
if (
415+
path not in excluded_dependencies_list
416+
and path not in main_module.artifact_id
417+
):
406418
required_dependencies[path] = module.Module(
407419
group_id=__proto_group_id(group_id),
408420
artifact_id=path,
@@ -418,8 +430,10 @@ def main(versions_file, monorepo):
418430
version=main_module.version,
419431
release_version=main_module.release_version,
420432
)
421-
if path not in excluded_dependencies_list \
422-
and path not in main_module.artifact_id:
433+
if (
434+
path not in excluded_dependencies_list
435+
and path not in main_module.artifact_id
436+
):
423437
required_dependencies[path] = module.Module(
424438
group_id=__proto_group_id(group_id),
425439
artifact_id=path,
@@ -439,8 +453,10 @@ def main(versions_file, monorepo):
439453
proto_module=existing_modules[proto_artifact_id],
440454
monorepo=monorepo,
441455
)
442-
if path not in excluded_dependencies_list \
443-
and path not in main_module.artifact_id:
456+
if (
457+
path not in excluded_dependencies_list
458+
and path not in main_module.artifact_id
459+
):
444460
required_dependencies[path] = module.Module(
445461
group_id=__proto_group_id(group_id),
446462
artifact_id=path,
@@ -451,13 +467,13 @@ def main(versions_file, monorepo):
451467
module
452468
for module in required_dependencies.values()
453469
if module.artifact_id.startswith("proto-")
454-
and module.artifact_id not in parent_artifact_id
470+
and module.artifact_id not in parent_artifact_id
455471
]
456472
grpc_modules = [
457473
module
458474
for module in required_dependencies.values()
459-
if module.artifact_id.startswith("grpc-") \
460-
and module.artifact_id not in parent_artifact_id
475+
if module.artifact_id.startswith("grpc-")
476+
and module.artifact_id not in parent_artifact_id
461477
]
462478
if main_module in grpc_modules or main_module in proto_modules:
463479
modules = grpc_modules + proto_modules
@@ -489,12 +505,11 @@ def main(versions_file, monorepo):
489505

490506
if os.path.isfile(f"{artifact_id}-bom/pom.xml"):
491507
print("updating modules in bom pom.xml")
492-
if artifact_id+"-bom" not in excluded_poms_list:
508+
if artifact_id + "-bom" not in excluded_poms_list:
493509
update_bom_pom(f"{artifact_id}-bom/pom.xml", modules)
494-
elif artifact_id+"-bom" not in excluded_poms_list:
510+
elif artifact_id + "-bom" not in excluded_poms_list:
495511
print("creating missing bom pom.xml")
496-
monorepo_version = __get_monorepo_version(versions_file) \
497-
if monorepo else ""
512+
monorepo_version = __get_monorepo_version(versions_file) if monorepo else ""
498513
templates.render(
499514
template_name="bom_pom.xml.j2",
500515
output_name=f"{artifact_id}-bom/pom.xml",
@@ -503,16 +518,15 @@ def main(versions_file, monorepo):
503518
modules=modules,
504519
main_module=main_module,
505520
monorepo=monorepo,
506-
monorepo_version=monorepo_version
521+
monorepo_version=monorepo_version,
507522
)
508523

509524
if os.path.isfile("pom.xml"):
510525
print("updating modules in parent pom.xml")
511526
update_parent_pom("pom.xml", modules)
512527
else:
513528
print("creating missing parent pom.xml")
514-
monorepo_version = __get_monorepo_version(versions_file) \
515-
if monorepo else ""
529+
monorepo_version = __get_monorepo_version(versions_file) if monorepo else ""
516530
templates.render(
517531
template_name="parent_pom.xml.j2",
518532
output_name="./pom.xml",
@@ -521,7 +535,7 @@ def main(versions_file, monorepo):
521535
main_module=main_module,
522536
name=name,
523537
monorepo=monorepo,
524-
monorepo_version=monorepo_version
538+
monorepo_version=monorepo_version,
525539
)
526540

527541
print(f"updating modules in {versions_file}")
@@ -537,13 +551,15 @@ def main(versions_file, monorepo):
537551
release_version=main_module.release_version,
538552
)
539553
templates.render(
540-
template_name="versions.txt.j2", output_name=versions_file, modules=existing_modules.values(),
554+
template_name="versions.txt.j2",
555+
output_name=versions_file,
556+
modules=existing_modules.values(),
541557
)
542558

543559

544560
if __name__ == "__main__":
545561
versions_file = sys.argv[1]
546562
monorepo = sys.argv[2]
547-
if monorepo == 'true':
563+
if monorepo == "true":
548564
monorepo = True
549565
main(versions_file, monorepo)

0 commit comments

Comments
 (0)