Implement proto3 field presence

Author: roblabla

src/betterproto/__init__.py

@@ -159,10 +159,11 @@ def dataclass_field(
     map_types: Optional[Tuple[str, str]] = None,
     group: Optional[str] = None,
     wraps: Optional[str] = None,
+    optional: bool = False,
 ) -> dataclasses.Field:
     """Creates a dataclass field with attached protobuf metadata."""
     return dataclasses.field(
-        default=PLACEHOLDER,
+        default=None if optional else PLACEHOLDER,
         metadata={
             "betterproto": FieldMetadata(number, proto_type, map_types, group, wraps)
         },
@@ -174,74 +175,74 @@ def dataclass_field(
 # out at runtime. The generated dataclass variables are still typed correctly.
 
 
-def enum_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_ENUM, group=group)
+def enum_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_ENUM, group=group, optional=optional)
 
 
-def bool_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_BOOL, group=group)
+def bool_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_BOOL, group=group, optional=optional)
 
 
-def int32_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_INT32, group=group)
+def int32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_INT32, group=group, optional=optional)
 
 
-def int64_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_INT64, group=group)
+def int64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_INT64, group=group, optional=optional)
 
 
-def uint32_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_UINT32, group=group)
+def uint32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_UINT32, group=group, optional=optional)
 
 
-def uint64_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_UINT64, group=group)
+def uint64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_UINT64, group=group, optional=optional)
 
 
-def sint32_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_SINT32, group=group)
+def sint32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_SINT32, group=group, optional=optional)
 
 
-def sint64_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_SINT64, group=group)
+def sint64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_SINT64, group=group, optional=optional)
 
 
-def float_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_FLOAT, group=group)
+def float_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_FLOAT, group=group, optional=optional)
 
 
-def double_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_DOUBLE, group=group)
+def double_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_DOUBLE, group=group, optional=optional)
 
 
-def fixed32_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_FIXED32, group=group)
+def fixed32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_FIXED32, group=group, optional=optional)
 
 
-def fixed64_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_FIXED64, group=group)
+def fixed64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_FIXED64, group=group, optional=optional)
 
 
-def sfixed32_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_SFIXED32, group=group)
+def sfixed32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_SFIXED32, group=group, optional=optional)
 
 
-def sfixed64_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_SFIXED64, group=group)
+def sfixed64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_SFIXED64, group=group, optional=optional)
 
 
-def string_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_STRING, group=group)
+def string_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_STRING, group=group, optional=optional)
 
 
-def bytes_field(number: int, group: Optional[str] = None) -> Any:
-    return dataclass_field(number, TYPE_BYTES, group=group)
+def bytes_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
+    return dataclass_field(number, TYPE_BYTES, group=group, optional=optional)
 
 
 def message_field(
-    number: int, group: Optional[str] = None, wraps: Optional[str] = None
+    number: int, group: Optional[str] = None, wraps: Optional[str] = None, optional: bool = False
 ) -> Any:
-    return dataclass_field(number, TYPE_MESSAGE, group=group, wraps=wraps)
+    return dataclass_field(number, TYPE_MESSAGE, group=group, wraps=wraps, optional=optional)
 
 
 def map_field(
@@ -701,12 +702,16 @@ def __bytes__(self) -> bytes:
 
             if value is None:
                 # Optional items should be skipped. This is used for the Google
-                # wrapper types.
+                # wrapper types and proto3 field presence/optional fields.
                 continue
 
             # Being selected in a a group means this field is the one that is
             # currently set in a `oneof` group, so it must be serialized even
             # if the value is the default zero value.
+            #
+            # Note that proto3 field presence/optional fields are put in a 
+            # synthetic single-item oneof by protoc, which helps us ensure we
+            # send the value even if the value is the default zero value.
             selected_in_group = (
                 meta.group and self._group_current[meta.group] == field_name
             )

src/betterproto/plugin/main.py

@@ -28,11 +28,8 @@ def main() -> None:
     if dump_file:
         dump_request(dump_file, request)
 
-    # Create response
-    response = CodeGeneratorResponse()
-
     # Generate code
-    generate_code(request, response)
+    response = generate_code(request)
 
     # Serialise response message
     output = response.SerializeToString()

src/betterproto/plugin/models.py

@@ -383,6 +383,8 @@ def betterproto_field_args(self) -> List[str]:
         args = []
         if self.field_wraps:
             args.append(f"wraps={self.field_wraps}")
+        if self.optional:
+            args.append(f"optional=True")
         return args
 
     @property
@@ -431,6 +433,12 @@ def repeated(self) -> bool:
             and not is_map(self.proto_obj, self.parent)
         )
 
+    @property
+    def optional(self) -> bool:
+        # TODO: Should proto2 optional fields with kind=Message also be
+        # considered Optional.
+        return self.proto_obj.proto3_optional
+
     @property
     def mutable(self) -> bool:
         """True if the field is a mutable type, otherwise False."""
@@ -450,6 +458,8 @@ def default_value_string(self) -> Union[Text, None, float, int]:
         """Python representation of the default proto value."""
         if self.repeated:
             return "[]"
+        if self.optional:
+            return "None"
         if self.py_type == "int":
             return "0"
         if self.py_type == "float":
@@ -506,6 +516,8 @@ def py_type(self) -> str:
     def annotation(self) -> str:
         if self.repeated:
             return f"List[{self.py_type}]"
+        if self.optional:
+            return f"Optional[{self.py_type}]"
         return self.py_type
 
 

src/betterproto/plugin/parser.py

@@ -8,6 +8,7 @@
 from betterproto.lib.google.protobuf.compiler import (
     CodeGeneratorRequest,
     CodeGeneratorResponse,
+    CodeGeneratorResponseFeature,
     CodeGeneratorResponseFile,
 )
 import itertools
@@ -60,10 +61,11 @@ def _traverse(
     )
 
 
-def generate_code(
-    request: CodeGeneratorRequest, response: CodeGeneratorResponse
-) -> None:
+def generate_code(request: CodeGeneratorRequest) -> CodeGeneratorResponse:
+    response = CodeGeneratorResponse()
+
     plugin_options = request.parameter.split(",") if request.parameter else []
+    response.supported_features = CodeGeneratorResponseFeature.FEATURE_PROTO3_OPTIONAL
 
     request_data = PluginRequestCompiler(plugin_request_obj=request)
     # Gather output packages
@@ -133,6 +135,7 @@ def generate_code(
     for output_package_name in sorted(output_paths.union(init_files)):
         print(f"Writing {output_package_name}", file=sys.stderr)
 
+    return response
 
 def read_protobuf_type(
     item: DescriptorProto,

tests/inputs/proto3_field_presence/proto3_field_presence.json

@@ -0,0 +1,7 @@
+{
+  "test1": null,
+  "test2": null,
+  "test3": null,
+  "test4": null,
+  "test5": null
+}

tests/inputs/proto3_field_presence/proto3_field_presence.proto

@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+message InnerTest {
+    string test = 1;
+}
+
+message Test {
+    optional uint32 test1 = 1;
+    optional bool test2 = 2;
+    optional string test3 = 3;
+    optional bytes test4 = 4;
+    optional InnerTest test5 = 5;
+}