Allow unbundled linking of rust libraries when using buck2

Summary: This diff ports the `native_unbundle_deps` feature from buck1 to buck2. The effect of this change is to make all `cxx_library` -> `rust_library` edges in the dep graph result in linking `rlib` artifacts rather than `staticlib` artifacts. The benefit being that we can then avoid issues with duplicate deps across libraries. rust-lang/rust#73632 has much more detail about this approach.

Reviewed By: zertosh

Differential Revision: D50523279

fbshipit-source-id: 16ee22db9140dba1e882d86aa376a03e010ef352

Author: capickett

prelude/rust/build.bzl

@@ -213,6 +213,7 @@ def generate_rustdoc_test(
     exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows"
 
     toolchain_info = compile_ctx.toolchain_info
+    native_unbundle_deps = toolchain_info.native_unbundle_deps
 
     resources = create_resource_db(
         ctx = ctx,
@@ -230,7 +231,7 @@ def generate_rustdoc_test(
     if link_style == LinkStyle("shared"):
         shlib_info = merge_shared_libraries(
             ctx.actions,
-            deps = inherited_shared_libs(ctx, include_doc_deps = True),
+            deps = inherited_shared_libs(ctx, native_unbundle_deps, include_doc_deps = True),
         )
         for soname, shared_lib in traverse_shared_library_info(shlib_info).items():
             shared_libs[soname] = shared_lib.lib
@@ -258,7 +259,7 @@ def generate_rustdoc_test(
             LinkArgs(flags = extra_link_args),
             get_link_args_for_strategy(
                 ctx,
-                inherited_merged_link_infos(ctx, include_doc_deps = True),
+                inherited_merged_link_infos(ctx, native_unbundle_deps, include_doc_deps = True),
                 # TODO(cjhopman): It's unclear how rust is using link_style. I'm not sure if it's intended to be a LibOutputStyle or a LinkStrategy.
                 to_link_strategy(link_style),
             ),
@@ -374,6 +375,7 @@ def rust_compile(
     exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows"
 
     toolchain_info = compile_ctx.toolchain_info
+    native_unbundle_deps = toolchain_info.native_unbundle_deps
 
     lints, clippy_lints = _lint_flags(compile_ctx)
 
@@ -444,6 +446,7 @@ def rust_compile(
                 ctx,
                 inherited_merged_link_infos(
                     ctx,
+                    native_unbundle_deps,
                     include_doc_deps = False,
                 ),
                 # TODO(cjhopman): It's unclear how rust is using link_style. I'm not sure if it's intended to be a LibOutputStyle or a LinkStrategy.

prelude/rust/build_params.bzl

@@ -124,8 +124,37 @@ def output_filename(cratename: str, emit: Emit, buildparams: BuildParams, extra:
 # Rule type - 'binary' also covers 'test'
 RuleType = enum("binary", "library")
 
-# What language we're generating artifacts to be linked with
-LinkageLang = enum("rust", "c++")
+# Controls how we build our rust libraries, largely dependent on whether rustc
+# or buck is driving the final linking and whether we are linking the artifact
+# into other rust targets.
+#
+# Rust: In this mode, we build rust libraries as rlibs. This is the primary
+# approach for building rust targets when the final link step is driven by
+# rustc (e.g. rust_binary, rust_unittest, etc).
+#
+# Native: In this mode, we build rust libraries as staticlibs, where rustc
+# will bundle all of this target's rust dependencies into a single library
+# artifact. This approach is the most standardized way to build rust libraries
+# for linkage in non-rust code.
+#
+# NOTE: This approach does not scale well. It's possible to end up with
+# non-rust target A depending on two rust targets B and C, which can cause
+# duplicate symbols if B and C share common rust dependencies.
+#
+# Native Unbundled: In this mode, we revert back to building as rlibs. This
+# approach mitigates the duplicate symbol downside of the "Native" approach.
+# However, this option is not formally supported by rustc, and depends on an
+# implementation detail of rlibs (they're effectively .a archives and can be
+# linked with other native code using the CXX linker).
+#
+# See https://github.com/rust-lang/rust/issues/73632 for more details on
+# stabilizing this approach.
+
+LinkageLang = enum(
+    "rust",
+    "native",
+    "native-unbundled",
+)
 
 _BINARY_SHARED = 0
 _BINARY_PIE = 1
@@ -231,10 +260,15 @@ _INPUTS = {
     ("binary", False, "static", "shared", "rust"): _BINARY_NON_PIE,
     ("binary", False, "static", "static", "rust"): _BINARY_NON_PIE,
     # Native linkable shared object
-    ("library", False, "shared", "any", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
-    ("library", False, "shared", "shared", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
-    ("library", False, "static", "shared", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
-    ("library", False, "static_pic", "shared", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "shared", "any", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "shared", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "static", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "static_pic", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    # Native unbundled linkable shared object
+    ("library", False, "shared", "any", "native-unbundled"): _RUST_DYLIB_SHARED,
+    ("library", False, "shared", "shared", "native-unbundled"): _RUST_DYLIB_SHARED,
+    ("library", False, "static", "shared", "native-unbundled"): _RUST_DYLIB_SHARED,
+    ("library", False, "static_pic", "shared", "native-unbundled"): _RUST_DYLIB_SHARED,
     # Rust dylib shared object
     ("library", False, "shared", "any", "rust"): _RUST_DYLIB_SHARED,
     ("library", False, "shared", "shared", "rust"): _RUST_DYLIB_SHARED,
@@ -258,12 +292,19 @@ _INPUTS = {
     ("library", False, "static", "any", "rust"): _RUST_STATIC_NON_PIC_LIBRARY,
     ("library", False, "static", "static", "rust"): _RUST_STATIC_NON_PIC_LIBRARY,
     # Native linkable static_pic
-    ("library", False, "shared", "static", "c++"): _NATIVE_LINKABLE_STATIC_PIC,
-    ("library", False, "static_pic", "any", "c++"): _NATIVE_LINKABLE_STATIC_PIC,
-    ("library", False, "static_pic", "static", "c++"): _NATIVE_LINKABLE_STATIC_PIC,
+    ("library", False, "shared", "static", "native"): _NATIVE_LINKABLE_STATIC_PIC,
+    ("library", False, "static_pic", "any", "native"): _NATIVE_LINKABLE_STATIC_PIC,
+    ("library", False, "static_pic", "static", "native"): _NATIVE_LINKABLE_STATIC_PIC,
     # Native linkable static non-pic
-    ("library", False, "static", "any", "c++"): _NATIVE_LINKABLE_STATIC_NON_PIC,
-    ("library", False, "static", "static", "c++"): _NATIVE_LINKABLE_STATIC_NON_PIC,
+    ("library", False, "static", "any", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC,
+    ("library", False, "static", "static", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC,
+    # Native Unbundled static_pic library
+    ("library", False, "shared", "static", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY,
+    ("library", False, "static_pic", "any", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY,
+    ("library", False, "static_pic", "static", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY,
+    # Native Unbundled static (non-pic) library
+    ("library", False, "static", "any", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY,
+    ("library", False, "static", "static", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY,
 }
 
 # Check types of _INPUTS, writing these out as types is too verbose, but let's make sure we don't have any typos.

prelude/rust/link_info.bzl

@@ -172,7 +172,7 @@ RustCxxLinkGroupInfo = record(
     link_group_libs = field(dict[str, [LinkGroupLib, None]]),
     # mapping from target labels to the corresponding link group link_info
     labels_to_links_map = field(dict[Label, LinkGroupLinkInfo]),
-    # prefrred linkage mode for link group libraries
+    # preferred linkage mode for link group libraries
     link_group_preferred_linkage = field(dict[Label, Linkage]),
 )
 
@@ -265,8 +265,9 @@ def _create_linkable_graph(
     return linkable_graph
 
 # Returns native link dependencies.
-def _non_rust_link_deps(
+def _native_link_dependencies(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[Dependency]:
     """
     Return all first-order native linkable dependencies of all transitive Rust
@@ -276,30 +277,31 @@ def _non_rust_link_deps(
     looking for non-Rust native link infos (and terminating the search there).
     """
     first_order_deps = [dep.dep for dep in resolve_deps(ctx, include_doc_deps)]
-    return [
-        d
-        for d in first_order_deps
-        if RustLinkInfo not in d and MergedLinkInfo in d
-    ]
+
+    if native_unbundle_deps:
+        return [d for d in first_order_deps if MergedLinkInfo in d]
+    else:
+        return [
+            d
+            for d in first_order_deps
+            if RustLinkInfo not in d and MergedLinkInfo in d
+        ]
 
 # Returns native link dependencies.
-def _non_rust_link_infos(
+def _native_link_infos(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[MergedLinkInfo]:
     """
     Return all first-order native link infos of all transitive Rust libraries.
-
-    This emulates v1's graph walk, where it traverses through Rust libraries
-    looking for non-Rust native link infos (and terminating the search there).
-    MergedLinkInfo is a mapping from link style to all the transitive deps
-    rolled up in a tset.
     """
-    link_deps = _non_rust_link_deps(ctx, include_doc_deps)
+    link_deps = _native_link_dependencies(ctx, native_unbundle_deps, include_doc_deps)
     return [d[MergedLinkInfo] for d in link_deps]
 
 # Returns native link dependencies.
-def _non_rust_shared_lib_infos(
+def _native_shared_lib_infos(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[SharedLibraryInfo]:
     """
     Return all transitive shared libraries for non-Rust native linkabes.
@@ -308,11 +310,15 @@ def _non_rust_shared_lib_infos(
     Rust libraries to collect all transitive shared libraries.
     """
     first_order_deps = [dep.dep for dep in resolve_deps(ctx, include_doc_deps)]
-    return [
-        d[SharedLibraryInfo]
-        for d in first_order_deps
-        if RustLinkInfo not in d and SharedLibraryInfo in d
-    ]
+
+    if native_unbundle_deps:
+        return [d[SharedLibraryInfo] for d in first_order_deps if SharedLibraryInfo in d]
+    else:
+        return [
+            d[SharedLibraryInfo]
+            for d in first_order_deps
+            if RustLinkInfo not in d and SharedLibraryInfo in d
+        ]
 
 # Returns native link dependencies.
 def _rust_link_infos(
@@ -323,21 +329,21 @@ def _rust_link_infos(
 def normalize_crate(label: str) -> str:
     return label.replace("-", "_")
 
-# TODO(pickett): Currently this assumes the library target is being built as a
-# staticlib or cdylib.
-def inherited_exported_link_deps(ctx: AnalysisContext) -> list[Dependency]:
+def inherited_exported_link_deps(ctx: AnalysisContext, native_unbundle_deps: bool) -> list[Dependency]:
     deps = {}
-    for dep in _non_rust_link_deps(ctx):
+    for dep in _native_link_dependencies(ctx, native_unbundle_deps):
         deps[dep.label] = dep
-    for info in _rust_link_infos(ctx):
-        for dep in info.exported_link_deps:
-            deps[dep.label] = dep
+    if not native_unbundle_deps:
+        for info in _rust_link_infos(ctx):
+            for dep in info.exported_link_deps:
+                deps[dep.label] = dep
     return deps.values()
 
 def inherited_rust_cxx_link_group_info(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         link_style: [LinkStyle, None] = None) -> RustCxxLinkGroupInfo:
-    link_deps = inherited_exported_link_deps(ctx)
+    link_deps = inherited_exported_link_deps(ctx, native_unbundle_deps)
 
     # Assume a rust executable wants to use link groups if a link group map
     # is present
@@ -415,24 +421,24 @@ def inherited_rust_cxx_link_group_info(
         link_group_preferred_linkage = link_group_preferred_linkage,
     )
 
-# TODO(pickett): Currently this assumes the library target is being built as a
-# staticlib or cdylib.
 def inherited_merged_link_infos(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[MergedLinkInfo]:
     infos = []
-    infos.extend(_non_rust_link_infos(ctx, include_doc_deps))
-    infos.extend([d.merged_link_info for d in _rust_link_infos(ctx, include_doc_deps) if d.merged_link_info])
+    infos.extend(_native_link_infos(ctx, native_unbundle_deps, include_doc_deps))
+    if not native_unbundle_deps:
+        infos.extend([d.merged_link_info for d in _rust_link_infos(ctx, include_doc_deps) if d.merged_link_info])
     return infos
 
-# TODO(pickett): Currently this assumes the library target is being built as a
-# staticlib or cdylib.
 def inherited_shared_libs(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[SharedLibraryInfo]:
     infos = []
-    infos.extend(_non_rust_shared_lib_infos(ctx, include_doc_deps))
-    infos.extend([d.shared_libs for d in _rust_link_infos(ctx, include_doc_deps)])
+    infos.extend(_native_shared_lib_infos(ctx, native_unbundle_deps, include_doc_deps))
+    if not native_unbundle_deps:
+        infos.extend([d.shared_libs for d in _rust_link_infos(ctx, include_doc_deps)])
     return infos
 
 def inherited_external_debug_info(

prelude/rust/rust_binary.bzl

@@ -97,6 +97,7 @@ def _rust_binary_common(
 
     target_os_type = ctx.attrs._target_os_type[OsLookup]
     linker_type = compile_ctx.cxx_toolchain_info.linker_info.type
+    native_unbundle_deps = compile_ctx.toolchain_info.native_unbundle_deps
 
     resources = flatten_dict(gather_resources(
         label = ctx.label,
@@ -133,6 +134,7 @@ def _rust_binary_common(
         if enable_link_groups(ctx, link_style, specified_link_style, is_binary = True):
             rust_cxx_link_group_info = inherited_rust_cxx_link_group_info(
                 ctx,
+                native_unbundle_deps,
                 link_style = link_style,
             )
             link_group_mappings = rust_cxx_link_group_info.link_group_info.mappings
@@ -147,7 +149,7 @@ def _rust_binary_common(
         if link_style == LinkStyle("shared") or rust_cxx_link_group_info != None:
             shlib_info = merge_shared_libraries(
                 ctx.actions,
-                deps = inherited_shared_libs(ctx),
+                deps = inherited_shared_libs(ctx, native_unbundle_deps),
             )
 
             link_group_ctx = LinkGroupContext(