Skip to content

REF: Spin off tree -> code compilation into a separate module #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 9, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions src/betterproto/plugin/compiler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import os.path

try:
# betterproto[compiler] specific dependencies
import black
import jinja2
except ImportError as err:
missing_import = err.args[0][17:-1]
print(
"\033[31m"
f"Unable to import `{missing_import}` from betterproto plugin! "
"Please ensure that you've installed betterproto as "
'`pip install "betterproto[compiler]"` so that compiler dependencies '
"are included."
"\033[0m"
)
raise SystemExit(1)

from betterproto.plugin.models import OutputTemplate


def outputfile_compiler(output_file: OutputTemplate) -> str:

templates_folder = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "templates")
)

env = jinja2.Environment(
trim_blocks=True,
lstrip_blocks=True,
loader=jinja2.FileSystemLoader(templates_folder),
)
template = env.get_template("template.py.j2")

res = black.format_str(
template.render(output_file=output_file),
mode=black.FileMode(target_versions={black.TargetVersion.PY37}),
)

return res
22 changes: 4 additions & 18 deletions src/betterproto/plugin/parser.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
import itertools
import os.path
import pathlib
import sys
from typing import List, Iterator

try:
# betterproto[compiler] specific dependencies
import black
from google.protobuf.compiler import plugin_pb2 as plugin
from google.protobuf.descriptor_pb2 import (
DescriptorProto,
EnumDescriptorProto,
FieldDescriptorProto,
ServiceDescriptorProto,
)
import jinja2
except ImportError as err:
missing_import = err.args[0][17:-1]
print(
Expand All @@ -41,6 +38,8 @@
is_oneof,
)

from betterproto.plugin.compiler import outputfile_compiler


def traverse(proto_file: FieldDescriptorProto) -> Iterator:
# Todo: Keep information about nested hierarchy
Expand Down Expand Up @@ -70,16 +69,6 @@ def generate_code(
) -> None:
plugin_options = request.parameter.split(",") if request.parameter else []
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does anyone know how to encode an alternative template/compiler (i.e. to swap out betterproto.plugin.compiler.py) as part of the command line options?

Copy link
Contributor Author

@adriangb adriangb Aug 6, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The goal here is that at the very least someone could copy compiler.py to my_compiler.py and edit the template (or the rendering method altogether) and pass an option to protoc that would tell plugin.py to import my_compiler.outputfile_compiler and use that instead of the default one. It's not a super clean API or anything but it should be powerful and a good starting place for other changes. This options parsing doesn't need to be part of this PR, just asking for future reference.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess you have in mind being able to do something like:

protoc --python_betterproto_out=lib --custom_opt=TYPE_VALIDATION,SILLY_WALKS,WHATEVER -I ...

I like the idea of stuff being modular and potentially configurable in this way. But I wouldn't race to provide an API to support fully support this without a real use case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this change stands on it's own, and I agree with you that more discussion is needed before providing an API to do the above, but I do want to make sure that this change is at least compatible with that approach.


templates_folder = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "templates")
)

env = jinja2.Environment(
trim_blocks=True,
lstrip_blocks=True,
loader=jinja2.FileSystemLoader(templates_folder),
)
template = env.get_template("template.py.j2")
request_data = PluginRequestCompiler(plugin_request_obj=request)
# Gather output packages
for proto_file in request.proto_file:
Expand Down Expand Up @@ -116,7 +105,7 @@ def generate_code(

# Generate output files
output_paths: pathlib.Path = set()
for output_package_name, template_data in request_data.output_packages.items():
for output_package_name, output_package in request_data.output_packages.items():

# Add files to the response object
output_path = pathlib.Path(*output_package_name.split("."), "__init__.py")
Expand All @@ -126,10 +115,7 @@ def generate_code(
f.name: str = str(output_path)

# Render and then format the output file
f.content: str = black.format_str(
template.render(description=template_data),
mode=black.FileMode(target_versions={black.TargetVersion.PY37}),
)
f.content: str = outputfile_compiler(output_file=output_package)

# Make each output directory a package with __init__ file
init_files = (
Expand Down
22 changes: 11 additions & 11 deletions src/betterproto/templates/template.py.j2
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# sources: {{ ', '.join(description.input_filenames) }}
# sources: {{ ', '.join(output_file.input_filenames) }}
# plugin: python-betterproto
{% for i in description.python_module_imports|sort %}
{% for i in output_file.python_module_imports|sort %}
import {{ i }}
{% endfor %}
from dataclasses import dataclass
{% if description.datetime_imports %}
from datetime import {% for i in description.datetime_imports|sort %}{{ i }}{% if not loop.last %}, {% endif %}{% endfor %}
{% if output_file.datetime_imports %}
from datetime import {% for i in output_file.datetime_imports|sort %}{{ i }}{% if not loop.last %}, {% endif %}{% endfor %}

{% endif%}
{% if description.typing_imports %}
from typing import {% for i in description.typing_imports|sort %}{{ i }}{% if not loop.last %}, {% endif %}{% endfor %}
{% if output_file.typing_imports %}
from typing import {% for i in output_file.typing_imports|sort %}{{ i }}{% if not loop.last %}, {% endif %}{% endfor %}

{% endif %}

import betterproto
{% if description.services %}
{% if output_file.services %}
import grpclib
{% endif %}


{% if description.enums %}{% for enum in description.enums %}
{% if output_file.enums %}{% for enum in output_file.enums %}
class {{ enum.py_name }}(betterproto.Enum):
{% if enum.comment %}
{{ enum.comment }}
Expand All @@ -36,7 +36,7 @@ class {{ enum.py_name }}(betterproto.Enum):

{% endfor %}
{% endif %}
{% for message in description.messages %}
{% for message in output_file.messages %}
@dataclass
class {{ message.py_name }}(betterproto.Message):
{% if message.comment %}
Expand Down Expand Up @@ -67,7 +67,7 @@ class {{ message.py_name }}(betterproto.Message):


{% endfor %}
{% for service in description.services %}
{% for service in output_file.services %}
class {{ service.py_name }}Stub(betterproto.ServiceStub):
{% if service.comment %}
{{ service.comment }}
Expand Down Expand Up @@ -154,6 +154,6 @@ class {{ service.py_name }}Stub(betterproto.ServiceStub):
{% endfor %}
{% endfor %}

{% for i in description.imports|sort %}
{% for i in output_file.imports|sort %}
{{ i }}
{% endfor %}