[SYCL] Enable program cache if build options are set (#3277)

 - Enable in-memory program cache for programs when build options are
provided. Three sources of build options are supported:
   - SYCL API (e.g. build options passed to build_with_kernel_type()
method);
   - build options specifed in the program image;
   - SYCL_PROGRAM_COMPILE_OPTIONS, SYCL_PROGRAM_LINK_OPTIONS environment
variable (overrides options from first two points.
 - optimize cache map keys:
   - add build options to program cache map keys;
   - remove device ID from kernel cache map keys.

Author: vladimirlaz

sycl/source/detail/kernel_program_cache.hpp

@@ -67,17 +67,16 @@ class KernelProgramCache {
   using PiProgramT = std::remove_pointer<RT::PiProgram>::type;
   using PiProgramPtrT = std::atomic<PiProgramT *>;
   using ProgramWithBuildStateT = BuildResult<PiProgramT>;
-  using ProgramCacheKeyT =
-      std::pair<std::pair<SerializedObj, KernelSetId>, RT::PiDevice>;
+  using ProgramCacheKeyT = std::pair<std::pair<SerializedObj, KernelSetId>,
+                                     std::pair<RT::PiDevice, std::string>>;
   using ProgramCacheT = std::map<ProgramCacheKeyT, ProgramWithBuildStateT>;
   using ContextPtr = context_impl *;
 
   using PiKernelT = std::remove_pointer<RT::PiKernel>::type;
 
   using PiKernelPtrT = std::atomic<PiKernelT *>;
   using KernelWithBuildStateT = BuildResult<PiKernelT>;
-  using KernelByNameT =
-      std::map<std::pair<string_class, RT::PiDevice>, KernelWithBuildStateT>;
+  using KernelByNameT = std::map<string_class, KernelWithBuildStateT>;
   using KernelCacheT = std::map<RT::PiProgram, KernelByNameT>;
 
   ~KernelProgramCache();

sycl/source/detail/program_impl.cpp

@@ -253,20 +253,13 @@ void program_impl::build_with_kernel_name(string_class KernelName,
   throw_if_state_is_not(program_state::none);
   MProgramModuleHandle = Module;
   if (!is_host()) {
-    // If there are no build options, program can be safely cached
-    if (is_cacheable_with_options(BuildOptions)) {
-      MProgramAndKernelCachingAllowed = true;
-      MProgram = ProgramManager::getInstance().getBuiltPIProgram(
-          Module, get_context(), get_devices()[0], KernelName, this,
-          /*JITCompilationIsRequired=*/(!BuildOptions.empty()));
-      const detail::plugin &Plugin = getPlugin();
-      Plugin.call<PiApiKind::piProgramRetain>(MProgram);
-    } else {
-      create_pi_program_with_kernel_name(
-          Module, KernelName,
-          /*JITCompilationIsRequired=*/(!BuildOptions.empty()));
-      build(BuildOptions);
-    }
+    MProgramAndKernelCachingAllowed = true;
+    MBuildOptions = BuildOptions;
+    MProgram = ProgramManager::getInstance().getBuiltPIProgram(
+        Module, get_context(), get_devices()[0], KernelName, this,
+        /*JITCompilationIsRequired=*/(!BuildOptions.empty()));
+    const detail::plugin &Plugin = getPlugin();
+    Plugin.call<PiApiKind::piProgramRetain>(MProgram);
   }
   MState = program_state::linked;
 }

sycl/source/detail/program_manager/program_manager.cpp

@@ -367,11 +367,36 @@ RT::PiProgram ProgramManager::getBuiltPIProgram(OSModuleHandle M,
   auto GetF = [](const Locked<ProgramCacheT> &LockedCache) -> ProgramCacheT & {
     return LockedCache.get();
   };
-  auto BuildF = [this, &M, &KSId, &Context, &Device, Prg,
-                 &JITCompilationIsRequired] {
-    const RTDeviceBinaryImage &Img =
-        getDeviceImage(M, KSId, Context, Device, JITCompilationIsRequired);
+  std::string BuildOptions;
+  if (Prg)
+    BuildOptions = Prg->get_build_options();
+  const RTDeviceBinaryImage &Img =
+      getDeviceImage(M, KSId, Context, Device, JITCompilationIsRequired);
+  std::string CompileOpts = Img.getCompileOptions();
+  std::string LinkOpts = Img.getLinkOptions();
+  pi_device_binary_property isEsimdImage = Img.getProperty("isEsimdImage");
+  if (!BuildOptions.empty()) {
+    CompileOpts += " ";
+    CompileOpts += BuildOptions;
+  }
+  if (isEsimdImage && pi::DeviceBinaryProperty(isEsimdImage).asUint32()) {
+    if (!CompileOpts.empty())
+      CompileOpts += " ";
+    CompileOpts += "-vc-codegen";
+  }
 
+  // Build options are overridden if environment variables are present
+  const char *CompileOptsEnv = SYCLConfig<SYCL_PROGRAM_COMPILE_OPTIONS>::get();
+  if (CompileOptsEnv) {
+    CompileOpts = CompileOptsEnv;
+  }
+  const char *LinkOptsEnv = SYCLConfig<SYCL_PROGRAM_LINK_OPTIONS>::get();
+  if (LinkOptsEnv) {
+    LinkOpts = LinkOptsEnv;
+  }
+
+  auto BuildF = [this, &M, &KSId, &Context, &Device, Prg, &Img,
+                 &JITCompilationIsRequired, &CompileOpts, &LinkOpts] {
     ContextImplPtr ContextImpl = getSyclObjImpl(Context);
     const detail::plugin &Plugin = ContextImpl->getPlugin();
     RT::PiProgram NativePrg = createPIProgram(Img, Context, Device);
@@ -390,17 +415,9 @@ RT::PiProgram ProgramManager::getBuiltPIProgram(OSModuleHandle M,
         !SYCLConfig<SYCL_DEVICELIB_NO_FALLBACK>::get())
       DeviceLibReqMask = getDeviceLibReqMask(Img);
 
-    std::string CompileOpts = Img.getCompileOptions();
-    pi_device_binary_property isEsimdImage = Img.getProperty("isEsimdImage");
-    if (isEsimdImage && pi::DeviceBinaryProperty(isEsimdImage).asUint32()) {
-      if (!CompileOpts.empty())
-        CompileOpts += " ";
-      CompileOpts += "-vc-codegen";
-    }
-
     ProgramPtr BuiltProgram =
-        build(std::move(ProgramManaged), ContextImpl, CompileOpts,
-              Img.getLinkOptions(), getRawSyclObjImpl(Device)->getHandleRef(),
+        build(std::move(ProgramManaged), ContextImpl, CompileOpts, LinkOpts,
+              getRawSyclObjImpl(Device)->getHandleRef(),
               ContextImpl->getCachedLibPrograms(), DeviceLibReqMask);
 
     {
@@ -417,7 +434,8 @@ RT::PiProgram ProgramManager::getBuiltPIProgram(OSModuleHandle M,
   const RT::PiDevice PiDevice = getRawSyclObjImpl(Device)->getHandleRef();
   auto BuildResult = getOrBuild<PiProgramT, compile_program_error>(
       Cache,
-      std::make_pair(std::make_pair(std::move(SpecConsts), KSId), PiDevice),
+      std::make_pair(std::make_pair(std::move(SpecConsts), KSId),
+                     std::make_pair(PiDevice, CompileOpts + LinkOpts)),
       AcquireF, GetF, BuildF);
   return BuildResult->Ptr.load();
 }
@@ -465,9 +483,8 @@ std::pair<RT::PiKernel, std::mutex *> ProgramManager::getOrCreateKernel(
     return Result;
   };
 
-  const RT::PiDevice PiDevice = getRawSyclObjImpl(Device)->getHandleRef();
   auto BuildResult = getOrBuild<PiKernelT, invalid_object_error>(
-      Cache, std::make_pair(KernelName, PiDevice), AcquireF, GetF, BuildF);
+      Cache, KernelName, AcquireF, GetF, BuildF);
   return std::make_pair(BuildResult->Ptr.load(),
                         &(BuildResult->MBuildResultMutex));
 }

sycl/unittests/kernel-and-program/Cache.cpp

@@ -201,36 +201,51 @@ TEST_F(KernelAndProgramCacheTest, ProgramSourceNegativeCompileAndLinkWithOpts) {
   EXPECT_EQ(Cache.size(), 0) << "Expect empty cache for source programs";
 }
 
-// Check that probrams built without options are cached.
+// Check that programs built without options are cached.
 TEST_F(KernelAndProgramCacheTest, ProgramBuildPositive) {
   if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
     return;
   }
 
   context Ctx{Plt};
-  program Prg{Ctx};
+  program Prg1{Ctx};
+  program Prg2{Ctx};
 
-  Prg.build_with_kernel_type<TestKernel>();
+  Prg1.build_with_kernel_type<TestKernel>();
+  Prg2.build_with_kernel_type<TestKernel>();
   auto CtxImpl = detail::getSyclObjImpl(Ctx);
   detail::KernelProgramCache::ProgramCacheT &Cache =
       CtxImpl->getKernelProgramCache().acquireCachedPrograms().get();
   EXPECT_EQ(Cache.size(), 1) << "Expect non-empty cache for programs";
 }
 
-// Check that probrams built with options are not cached.
-TEST_F(KernelAndProgramCacheTest, ProgramBuildNegativeBuildOpts) {
+// Check that programs built with options are cached.
+TEST_F(KernelAndProgramCacheTest, ProgramBuildPositiveBuildOpts) {
   if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
     return;
   }
 
   context Ctx{Plt};
-  program Prg{Ctx};
+  program Prg1{Ctx};
+  program Prg2{Ctx};
+  program Prg3{Ctx};
+  program Prg4{Ctx};
+  program Prg5{Ctx};
+
+  /* Build 5 instances of the same program. It is expected that there will be 3
+   * instances of the program in the cache because Build of Prg1 is equal to
+   * build of Prg5 and build of Prg2 is equal to build of Prg3.
+   * */
+  Prg1.build_with_kernel_type<TestKernel>("-a");
+  Prg2.build_with_kernel_type<TestKernel>("-b");
+  Prg3.build_with_kernel_type<TestKernel>("-b");
+  Prg4.build_with_kernel_type<TestKernel>();
+  Prg5.build_with_kernel_type<TestKernel2>("-a");
 
-  Prg.build_with_kernel_type<TestKernel>("-g");
   auto CtxImpl = detail::getSyclObjImpl(Ctx);
   detail::KernelProgramCache::ProgramCacheT &Cache =
       CtxImpl->getKernelProgramCache().acquireCachedPrograms().get();
-  EXPECT_EQ(Cache.size(), 0) << "Expect empty cache for programs";
+  EXPECT_EQ(Cache.size(), 3) << "Expect non-empty cache for programs";
 }
 
 // Check that programs built with compile options are not cached.
@@ -287,8 +302,8 @@ TEST_F(KernelAndProgramCacheTest, KernelPositive) {
   EXPECT_EQ(Cache.size(), 1) << "Expect non-empty cache for kernels";
 }
 
-// Check that kernels built with options are not cached.
-TEST_F(KernelAndProgramCacheTest, KernelNegativeBuildOpts) {
+// Check that kernels built with options are cached.
+TEST_F(KernelAndProgramCacheTest, KernelPositiveBuildOpts) {
   if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
     return;
   }
@@ -301,10 +316,11 @@ TEST_F(KernelAndProgramCacheTest, KernelNegativeBuildOpts) {
   program Prg{Ctx};
 
   Prg.build_with_kernel_type<TestKernel>("-g");
+
   kernel Ker = Prg.get_kernel<TestKernel>();
   detail::KernelProgramCache::KernelCacheT &Cache =
       CtxImpl->getKernelProgramCache().acquireKernelsPerProgramCache().get();
-  EXPECT_EQ(Cache.size(), 0) << "Expect empty cache for kernels";
+  EXPECT_EQ(Cache.size(), 1) << "Expect non-empty cache for kernels";
 }
 
 // Check that kernels built with compile options are not cached.