[SYCL] Embed bfloat16 devicelib into executable if necessary

clang/lib/Driver/ToolChains/Clang.cpp

@@ -11020,6 +11020,7 @@ static void getNonTripleBasedSYCLPostLinkOpts(const ToolChain &TC,
                                               const JobAction &JA,
                                               const llvm::opt::ArgList &TCArgs,
                                               ArgStringList &PostLinkArgs) {
+
   // See if device code splitting is requested
   if (Arg *A = TCArgs.getLastArg(options::OPT_fsycl_device_code_split_EQ)) {
     auto CodeSplitValue = StringRef(A->getValue());
@@ -11045,6 +11046,25 @@ static void getNonTripleBasedSYCLPostLinkOpts(const ToolChain &TC,
 
   if (allowDeviceImageDependencies(TCArgs))
     addArgs(PostLinkArgs, TCArgs, {"-allow-device-image-dependencies"});
+
+  // For bfloat16 conversions LLVM IR devicelib, we only need to embed it
+  // when non-AOT compilation is used.
+  if (TC.getTriple().isSPIROrSPIRV() && !TC.getTriple().isSPIRAOT()) {
+    SYCLInstallationDetector SYCLInstall(TC.getDriver());
+    SmallVector<SmallString<128>, 4> DeviceLibLocCandidates;
+    SmallString<128> NativeBfloat16Name("libsycl-native-bfloat16.bc");
+    SYCLInstall.getSYCLDeviceLibPath(DeviceLibLocCandidates);
+    for (const auto &DeviceLibLoc : DeviceLibLocCandidates) {
+      SmallString<128> FullLibName(DeviceLibLoc);
+      llvm::sys::path::append(FullLibName, NativeBfloat16Name);
+      if (llvm::sys::fs::exists(FullLibName)) {
+        SmallString<128> SYCLDeviceLibDir("--device-lib-dir=");
+        SYCLDeviceLibDir += DeviceLibLoc.str();
+        addArgs(PostLinkArgs, TCArgs, {SYCLDeviceLibDir.str()});
+        break;
+      }
+    }
+  }
 }
 
 // On Intel targets we don't need non-kernel functions as entry points,

clang/test/Driver/linker-wrapper-sycl-win.cpp

@@ -116,3 +116,17 @@
 // Error handling when --linker-path is not provided for clang-linker-wrapper
 // RUN: not clang-linker-wrapper 2>&1 | FileCheck --check-prefix=LINKER-PATH-NOT-PROVIDED %s
 // LINKER-PATH-NOT-PROVIDED: linker path missing, must pass 'linker-path'
+
+/// Check --device-lib-dir for sycl-post-link tool
+// ------
+// Generate .o file as linker wrapper input.
+
+// RUN: %clang %s -fsycl -fsycl-targets=spir64-unknown-unknown -c --offload-new-driver -o %t5.o
+//
+// Run clang-linker-wrapper test
+//
+// RUN: clang-linker-wrapper -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-pc-windows-msvc" "--linker-path=/usr/bin/ld" "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t5.o -sycl-device-libraries=libsycl-crt.new.o -sycl-device-library-location=%S/Inputs/SYCL/lib --dry-run 2>&1 | FileCheck -check-prefix=CHK-CMDS-DEVICE-LIB-DIR %s
+// CHK-CMDS-DEVICE-LIB-DIR: "{{.*}}spirv-to-ir-wrapper.exe" {{.*}} --llvm-spirv-opts --spirv-preserve-auxdata --spirv-target-env=SPV-IR --spirv-builtin-format=global
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link.exe" {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link.exe" -only-needed {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}sycl-post-link.exe"{{.*}} --device-lib-dir={{.*}}/Inputs/SYCL/lib {{.*}} SYCL_POST_LINK_OPTIONS {{.*}}

clang/test/Driver/linker-wrapper-sycl.cpp

@@ -194,3 +194,15 @@
 // Error handling when --linker-path is not provided for clang-linker-wrapper
 // RUN: not clang-linker-wrapper 2>&1 | FileCheck --check-prefix=LINKER-PATH-NOT-PROVIDED %s
 // LINKER-PATH-NOT-PROVIDED: linker path missing, must pass 'linker-path'
+
+/// check for --device-lib-dir options for sycl-post-link.
+// -------
+// Generate .o file as linker wrapper input.
+//
+// RUN: %clang %s -fsycl -fsycl-targets=spir64-unknown-unknown -c --offload-new-driver -o %t5.o
+//
+// RUN: clang-linker-wrapper -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--linker-path=/usr/bin/ld" "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t5.o -sycl-device-libraries=libsycl-crt.new.o -sycl-device-library-location=%S/Inputs/SYCL/lib --dry-run 2>&1 | FileCheck -check-prefix=CHK-CMDS-DEVICE-LIB-DIR %s
+// CHK-CMDS-DEVICE-LIB-DIR: "{{.*}}spirv-to-ir-wrapper" {{.*}} --llvm-spirv-opts --spirv-preserve-auxdata --spirv-target-env=SPV-IR --spirv-builtin-format=global
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link" {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link" -only-needed {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}sycl-post-link"{{.*}} --device-lib-dir={{.*}}/Inputs/SYCL/lib {{.*}} SYCL_POST_LINK_OPTIONS {{.*}}

clang/test/Driver/sycl-post-link-options-win.cpp

@@ -1,9 +1,9 @@
 // REQUIRES: system-windows
 /// Verify same set of sycl-post-link options generated for old and new offloading model
 // RUN: %clangxx -### --target=x86_64-pc-windows-msvc -fsycl \
-// RUN:          -Xdevice-post-link -O0 %s 2>&1 \
+// RUN:          -Xdevice-post-link -O0 %s --sysroot=%S/Inputs/SYCL 2>&1 \
 // RUN:   | FileCheck -check-prefix OPTIONS_POSTLINK_JIT_OLD %s
-// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
+// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "--device-lib-dir={{.*}}" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
 // -------
 // Generate .o file as linker wrapper input.
 //

clang/test/Driver/sycl-post-link-options.cpp

@@ -1,9 +1,9 @@
 // REQUIRES: system-linux
 /// Verify same set of sycl-post-link options generated for old and new offloading model
 // RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl -### \
-// RUN:          --no-offload-new-driver -Xdevice-post-link -O0 %s 2>&1 \
+// RUN:          --no-offload-new-driver -Xdevice-post-link -O0 %s --sysroot=%S/Inputs/SYCL 2>&1 \
 // RUN:   | FileCheck -check-prefix OPTIONS_POSTLINK_JIT_OLD %s
-// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
+// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "--device-lib-dir={{.*}}" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
 //
 // Generate .o file as linker wrapper input.
 //

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

@@ -726,6 +726,16 @@ runSYCLPostLinkTool(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   SmallVector<StringRef, 8> CmdArgs;
   CmdArgs.push_back(*SYCLPostLinkPath);
   const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
+  Arg *SYCLDeviceLibLoc = Args.getLastArg(OPT_sycl_device_library_location_EQ);
+  if (SYCLDeviceLibLoc && !Triple.isSPIRAOT()) {
+    std::string SYCLDeviceLibLocParam = SYCLDeviceLibLoc->getValue();
+    std::string BF16DeviceLibLoc =
+        SYCLDeviceLibLocParam + "/libsycl-native-bfloat16.bc";
+    if (llvm::sys::fs::exists(BF16DeviceLibLoc)) {
+      SYCLDeviceLibLocParam = "--device-lib-dir=" + SYCLDeviceLibLocParam;
+      CmdArgs.push_back(Args.MakeArgString(StringRef(SYCLDeviceLibLocParam)));
+    }
+  }
   getTripleBasedSYCLPostLinkOpts(Args, CmdArgs, Triple);
   StringRef SYCLPostLinkOptions;
   if (Arg *A = Args.getLastArg(OPT_sycl_post_link_options_EQ))

llvm/include/llvm/SYCLLowerIR/ComputeModuleRuntimeInfo.h

@@ -12,6 +12,7 @@
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/SYCLLowerIR/ModuleSplitter.h"
+#include "llvm/SYCLLowerIR/SYCLDeviceLibReqMask.h"
 #include "llvm/Support/PropertySetIO.h"
 #include <string>
 namespace llvm {
@@ -34,6 +35,9 @@ bool isModuleUsingTsan(const Module &M);
 using PropSetRegTy = llvm::util::PropertySetRegistry;
 using EntryPointSet = SetVector<Function *>;
 
+PropSetRegTy computeDeviceLibProperties(const Module &M,
+                                        const std::string &SYCLDeviceLibName);
+
 PropSetRegTy computeModuleProperties(const Module &M,
                                      const EntryPointSet &EntryPoints,
                                      const GlobalBinImageProps &GlobProps);

llvm/include/llvm/SYCLLowerIR/ModuleSplitter.h

@@ -21,6 +21,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/PropertySetIO.h"
 
+#include <array>
 #include <memory>
 #include <optional>
 #include <string>
@@ -38,6 +39,8 @@ class OptionCategory;
 namespace module_split {
 
 constexpr char SYCL_ESIMD_SPLIT_MD_NAME[] = "sycl-esimd-split-status";
+constexpr std::array<const char *, 2> SYCLDeviceLibs = {
+    "libsycl-fallback-bfloat16.bc", "libsycl-native-bfloat16.bc"};
 
 extern cl::OptionCategory &getModuleSplitCategory();
 
@@ -129,6 +132,7 @@ class ModuleDesc {
   std::unique_ptr<Module> M;
   EntryPointGroup EntryPoints;
   bool IsTopLevel = false;
+  bool IsSYCLDeviceLib = false;
   mutable std::optional<SYCLDeviceRequirements> Reqs;
 
 public:
@@ -140,7 +144,16 @@ class ModuleDesc {
   Properties Props;
 
   ModuleDesc(std::unique_ptr<Module> &&M, StringRef Name = "TOP-LEVEL")
-      : M(std::move(M)), IsTopLevel(true), Name(Name) {}
+      : M(std::move(M)), IsTopLevel(true), Name(Name) {
+    // DeviceLib module doesn't include any entry point,it can be constructed
+    // using ctor without any entry point related parameter.
+    for (auto Fn : SYCLDeviceLibs) {
+      if (StringRef(Fn) == Name) {
+        IsSYCLDeviceLib = true;
+        break;
+      }
+    }
+  }
 
   ModuleDesc(std::unique_ptr<Module> &&M, EntryPointGroup &&EntryPoints,
              const Properties &Props)
@@ -166,6 +179,7 @@ class ModuleDesc {
 
   bool isESIMD() const { return EntryPoints.isEsimd(); }
   bool isSYCL() const { return EntryPoints.isSycl(); }
+  bool isSYCLDeviceLib() const { return IsSYCLDeviceLib; }
 
   const EntryPointSet &entries() const { return EntryPoints.Functions; }
   const EntryPointGroup &getEntryPointGroup() const { return EntryPoints; }

llvm/include/llvm/SYCLLowerIR/SYCLDeviceLibReqMask.h

@@ -20,11 +20,14 @@
 
 namespace llvm {
 
+class Function;
 class Module;
 
 // DeviceLibExt is shared between sycl-post-link tool and sycl runtime.
 // If any change is made here, need to sync with DeviceLibExt definition
 // in sycl/source/detail/program_manager/program_manager.hpp
+// TODO: clear all these DeviceLibExt defs when begin to remove sycl
+// devicelib online link path.
 enum class DeviceLibExt : std::uint32_t {
   cl_intel_devicelib_assert,
   cl_intel_devicelib_math,
@@ -39,5 +42,6 @@ enum class DeviceLibExt : std::uint32_t {
 };
 
 uint32_t getSYCLDeviceLibReqMask(const Module &M);
-
+bool isSYCLDeviceLibBF16Used(const Module &M);
+bool isBF16DeviceLibFuncDecl(const Function &F);
 } // namespace llvm

llvm/include/llvm/Support/PropertySetIO.h

@@ -199,7 +199,10 @@ class PropertySetRegistry {
       "SYCL/specialization constants";
   static constexpr char SYCL_SPEC_CONSTANTS_DEFAULT_VALUES[] =
       "SYCL/specialization constants default values";
+  // TODO: remove SYCL_DEVICELIB_REQ_MASK when devicelib online linking path
+  // is totally removed.
   static constexpr char SYCL_DEVICELIB_REQ_MASK[] = "SYCL/devicelib req mask";
+  static constexpr char SYCL_DEVICELIB_METADATA[] = "SYCL/devicelib metadata";
   static constexpr char SYCL_KERNEL_PARAM_OPT_INFO[] = "SYCL/kernel param opt";
   static constexpr char SYCL_PROGRAM_METADATA[] = "SYCL/program metadata";
   static constexpr char SYCL_MISC_PROP[] = "SYCL/misc properties";

llvm/lib/SYCLLowerIR/ComputeModuleRuntimeInfo.cpp

@@ -159,6 +159,35 @@ std::optional<T> getKernelSingleEltMetadata(const Function &Func,
   return std::nullopt;
 }
 
+PropSetRegTy computeDeviceLibProperties(const Module &M,
+                                        const std::string &DeviceLibName) {
+  PropSetRegTy PropSet;
+
+  {
+    for (const auto &F : M.functions()) {
+      if (F.isDeclaration() || !F.getName().starts_with("__devicelib_"))
+        continue;
+      if (F.getCallingConv() == CallingConv::SPIR_FUNC) {
+        PropSet.add(PropSetRegTy::SYCL_EXPORTED_SYMBOLS, F.getName(),
+                    /*PropVal=*/true);
+      }
+    }
+  }
+
+  {
+    // Currently, only bfloat16 conversion devicelib is supported, the
+    // metadata value '0' means fallback version and '1' means native version.
+    uint32_t IsNativeBF16DeviceLib = 0;
+    if (DeviceLibName.find("native") != std::string::npos)
+      IsNativeBF16DeviceLib = 1;
+    std::map<StringRef, uint32_t> BF16DeviceLibMeta = {
+        {"bfloat16", IsNativeBF16DeviceLib}};
+    PropSet.add(PropSetRegTy::SYCL_DEVICELIB_METADATA, BF16DeviceLibMeta);
+  }
+
+  return PropSet;
+}
+
 PropSetRegTy computeModuleProperties(const Module &M,
                                      const EntryPointSet &EntryPoints,
                                      const GlobalBinImageProps &GlobProps) {
@@ -473,7 +502,7 @@ PropSetRegTy computeModuleProperties(const Module &M,
       }
 
       PropSet.add(PropSetRegTy::SYCL_VIRTUAL_FUNCTIONS,
-                   "uses-virtual-functions-set", AllSets);
+                  "uses-virtual-functions-set", AllSets);
     }
   }
 

llvm/lib/SYCLLowerIR/ModuleSplitter.cpp

@@ -25,6 +25,7 @@
 #include "llvm/IRPrinter/IRPrintingPasses.h"
 #include "llvm/SYCLLowerIR/DeviceGlobals.h"
 #include "llvm/SYCLLowerIR/LowerInvokeSimd.h"
+#include "llvm/SYCLLowerIR/SYCLDeviceLibReqMask.h"
 #include "llvm/SYCLLowerIR/SYCLUtils.h"
 #include "llvm/SYCLLowerIR/SpecConstants.h"
 #include "llvm/Support/CommandLine.h"
@@ -1428,6 +1429,12 @@ splitSYCLModule(std::unique_ptr<Module> M, ModuleSplitterSettings Settings) {
 }
 
 bool canBeImportedFunction(const Function &F) {
+
+  // We use sycl dynamic library mechanism to involve bf16 devicelib when
+  // necessary, all __devicelib_* functions from native or fallback bf16
+  // devicelib will be treated as imported function to user's device image.
+  if (llvm::isBF16DeviceLibFuncDecl(F))
+    return true;
   // It may be theoretically possible to determine what is importable
   // based solely on function F, but the "SYCL/imported symbols"
   // property list MUST NOT have any imported symbols that are not supplied

llvm/lib/SYCLLowerIR/SYCLDeviceLibReqMask.cpp

@@ -15,6 +15,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SYCLLowerIR/SYCLDeviceLibReqMask.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -24,7 +26,10 @@
 static constexpr char DEVICELIB_FUNC_PREFIX[] = "__devicelib_";
 
 using namespace llvm;
-
+// We will gradually remove devicelib spv online linking path but keep
+// bfloat16 devicelib spv as an exception for a short-term solution.
+// For bfloat16 devicelib spv link, we won't rely on ReqMask but to embed
+// the bits into executable if necessary
 namespace {
 
 using SYCLDeviceLibFuncMap = std::unordered_map<std::string, DeviceLibExt>;
@@ -702,34 +707,6 @@ SYCLDeviceLibFuncMap SDLMap = {
     {"__devicelib_imf_floorbf16", DeviceLibExt::cl_intel_devicelib_imf_bf16},
     {"__devicelib_imf_ceilbf16", DeviceLibExt::cl_intel_devicelib_imf_bf16},
     {"__devicelib_imf_truncbf16", DeviceLibExt::cl_intel_devicelib_imf_bf16},
-    {"__devicelib_ConvertFToBF16INTEL",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTEL",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertFToBF16INTELVec1",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTELVec1",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertFToBF16INTELVec2",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTELVec2",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertFToBF16INTELVec3",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTELVec3",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertFToBF16INTELVec4",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTELVec4",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertFToBF16INTELVec8",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTELVec8",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertFToBF16INTELVec16",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
-    {"__devicelib_ConvertBF16ToFINTELVec16",
-     DeviceLibExt::cl_intel_devicelib_bfloat16},
 };
 
 // Each fallback device library corresponds to one bit in "require mask" which
@@ -775,3 +752,44 @@ uint32_t llvm::getSYCLDeviceLibReqMask(const Module &M) {
   }
   return ReqMask;
 }
+
+static llvm::SmallVector<StringRef, 14> BF16DeviceLibFuncs = {
+    "__devicelib_ConvertFToBF16INTEL",
+    "__devicelib_ConvertBF16ToFINTEL",
+    "__devicelib_ConvertFToBF16INTELVec1",
+    "__devicelib_ConvertBF16ToFINTELVec1",
+    "__devicelib_ConvertFToBF16INTELVec2",
+    "__devicelib_ConvertBF16ToFINTELVec2",
+    "__devicelib_ConvertFToBF16INTELVec3",
+    "__devicelib_ConvertBF16ToFINTELVec3",
+    "__devicelib_ConvertFToBF16INTELVec4",
+    "__devicelib_ConvertBF16ToFINTELVec4",
+    "__devicelib_ConvertFToBF16INTELVec8",
+    "__devicelib_ConvertBF16ToFINTELVec8",
+    "__devicelib_ConvertFToBF16INTELVec16",
+    "__devicelib_ConvertBF16ToFINTELVec16",
+};
+
+bool llvm::isSYCLDeviceLibBF16Used(const Module &M) {
+  if (!Triple(M.getTargetTriple()).isSPIROrSPIRV())
+    return false;
+
+  for (auto Fn : BF16DeviceLibFuncs) {
+    Function *BF16Func = M.getFunction(Fn);
+    if (BF16Func && BF16Func->isDeclaration())
+      return true;
+  }
+
+  return false;
+}
+
+bool llvm::isBF16DeviceLibFuncDecl(const Function &F) {
+  if (!F.isDeclaration() || !F.getName().starts_with(DEVICELIB_FUNC_PREFIX))
+    return false;
+  for (auto BFunc : BF16DeviceLibFuncs) {
+    if (!F.getName().compare(BFunc))
+      return true;
+  }
+
+  return false;
+}