[L0] Interrupt-based event implementation

winstonzhang-intel · winstonzhang-intel · commit 085631310ec6 · 2024-11-25T16:40:59.000-08:00
Signed-off-by: Zhang, Winston &lt;winston.zhang@intel.com&gt;
diff --git a/source/adapters/level_zero/context.cpp b/source/adapters/level_zero/context.cpp
@@ -488,9 +488,9 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
   if (Device) {
     ZeDevice = Device->ZeDevice;
   }
-  std::list<ze_event_pool_handle_t> *ZePoolCache =
-      getZeEventPoolCache(HostVisible, ProfilingEnabled,
-                          CounterBasedEventEnabled, UsingImmCmdList, ZeDevice);
+  std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
+      HostVisible, ProfilingEnabled, CounterBasedEventEnabled, UsingImmCmdList,
+      InterruptBasedEventEnabled, ZeDevice);
 
   if (!ZePoolCache->empty()) {
     if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -572,7 +572,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
 
 ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
     bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
-    bool CounterBasedEventEnabled) {
+    bool CounterBasedEventEnabled, bool InterruptBasedEventEnabled) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
   auto Cache = getEventCache(HostVisible, WithProfiling, Device);
   if (Cache->empty())
@@ -583,6 +583,9 @@ ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
   if (Event->CounterBasedEventsEnabled != CounterBasedEventEnabled) {
     return nullptr;
   }
+  if (Event->InterruptBasedEventsEnabled != InterruptBasedEventEnabled) {
+    return nullptr;
+  }
   Cache->erase(It);
   // We have to reset event before using it.
   Event->reset();
@@ -623,7 +626,8 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
 
   std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
       Event->isHostVisible(), Event->isProfilingEnabled(),
-      Event->CounterBasedEventsEnabled, UsingImmediateCommandlists, ZeDevice);
+      Event->CounterBasedEventsEnabled, UsingImmediateCommandlists,
+      Event->InterruptBasedEventsEnabled, ZeDevice);
 
   // Put the empty pool to the cache of the pools.
   if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)
diff --git a/source/adapters/level_zero/context.hpp b/source/adapters/level_zero/context.hpp
@@ -168,9 +168,9 @@ struct ur_context_handle_t_ : _ur_object {
   // head.
   //
   // Cache of event pools to which host-visible events are added to.
-  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{12};
+  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{30};
   std::vector<std::unordered_map<ze_device_handle_t, size_t>>
-      ZeEventPoolCacheDeviceMap{12};
+      ZeEventPoolCacheDeviceMap{30};
 
   // This map will be used to determine if a pool is full or not
   // by storing number of empty slots available in the pool.
@@ -233,7 +233,8 @@ struct ur_context_handle_t_ : _ur_object {
   ur_event_handle_t getEventFromContextCache(bool HostVisible,
                                              bool WithProfiling,
                                              ur_device_handle_t Device,
-                                             bool CounterBasedEventEnabled);
+                                             bool CounterBasedEventEnabled,
+                                             bool InterruptBasedEventEnabled);
 
   // Add ur_event_handle_t to cache.
   void addEventToContextCache(ur_event_handle_t);
@@ -244,17 +245,29 @@ struct ur_context_handle_t_ : _ur_object {
     HostVisibleCounterBasedRegularCacheType,
     HostInvisibleCounterBasedRegularCacheType,
     HostVisibleCounterBasedImmediateCacheType,
-    HostInvisibleCounterBasedImmediateCacheType
+    HostInvisibleCounterBasedImmediateCacheType,
+
+    HostVisibleInterruptBasedRegularCacheType,
+    HostInvisibleInterruptBasedRegularCacheType,
+    HostVisibleInterruptBasedImmediateCacheType,
+    HostInvisibleInterruptBasedImmediateCacheType,
+
+    HostVisibleInterruptAndCounterBasedRegularCacheType,
+    HostInvisibleInterruptAndCounterBasedRegularCacheType,
+    HostVisibleInterruptAndCounterBasedImmediateCacheType,
+    HostInvisibleInterruptAndCounterBasedImmediateCacheType
   };
 
   std::list<ze_event_pool_handle_t> *
   getZeEventPoolCache(bool HostVisible, bool WithProfiling,
                       bool CounterBasedEventEnabled, bool UsingImmediateCmdList,
+                      bool InterruptBasedEventEnabled,
                       ze_device_handle_t ZeDevice) {
     EventPoolCacheType CacheType;
 
     calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
-                        UsingImmediateCmdList, CacheType);
+                        InterruptBasedEventEnabled, UsingImmediateCmdList,
+                        CacheType);
     if (ZeDevice) {
       auto ZeEventPoolCacheMap =
           WithProfiling ? &ZeEventPoolCacheDeviceMap[CacheType * 2]
@@ -274,23 +287,57 @@ struct ur_context_handle_t_ : _ur_object {
   ur_result_t calculateCacheIndex(bool HostVisible,
                                   bool CounterBasedEventEnabled,
                                   bool UsingImmediateCmdList,
+                                  bool InterruptBasedEventEnabled,
                                   EventPoolCacheType &CacheType) {
-    if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
-      CacheType = HostVisibleCounterBasedRegularCacheType;
-    } else if (CounterBasedEventEnabled && !HostVisible &&
-               !UsingImmediateCmdList) {
-      CacheType = HostInvisibleCounterBasedRegularCacheType;
-    } else if (CounterBasedEventEnabled && HostVisible &&
-               UsingImmediateCmdList) {
-      CacheType = HostVisibleCounterBasedImmediateCacheType;
-    } else if (CounterBasedEventEnabled && !HostVisible &&
-               UsingImmediateCmdList) {
-      CacheType = HostInvisibleCounterBasedImmediateCacheType;
-    } else if (!CounterBasedEventEnabled && HostVisible) {
-      CacheType = HostVisibleCacheType;
+    if (InterruptBasedEventEnabled) {
+      if (CounterBasedEventEnabled) {
+        if (HostVisible) {
+          if (UsingImmediateCmdList) {
+            CacheType = HostVisibleInterruptAndCounterBasedImmediateCacheType;
+          } else {
+            CacheType = HostVisibleInterruptAndCounterBasedRegularCacheType;
+          }
+        } else {
+          if (UsingImmediateCmdList) {
+            CacheType = HostInvisibleInterruptAndCounterBasedImmediateCacheType;
+          } else {
+            CacheType = HostInvisibleInterruptAndCounterBasedRegularCacheType;
+          }
+        }
+      } else {
+        if (HostVisible) {
+          if (UsingImmediateCmdList) {
+            CacheType = HostVisibleInterruptBasedImmediateCacheType;
+          } else {
+            CacheType = HostVisibleInterruptBasedRegularCacheType;
+          }
+        } else {
+          if (UsingImmediateCmdList) {
+            CacheType = HostInvisibleInterruptBasedImmediateCacheType;
+          } else {
+            CacheType = HostInvisibleInterruptBasedRegularCacheType;
+          }
+        }
+      }
     } else {
-      CacheType = HostInvisibleCacheType;
+      if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
+        CacheType = HostVisibleCounterBasedRegularCacheType;
+      } else if (CounterBasedEventEnabled && !HostVisible &&
+                 !UsingImmediateCmdList) {
+        CacheType = HostInvisibleCounterBasedRegularCacheType;
+      } else if (CounterBasedEventEnabled && HostVisible &&
+                 UsingImmediateCmdList) {
+        CacheType = HostVisibleCounterBasedImmediateCacheType;
+      } else if (CounterBasedEventEnabled && !HostVisible &&
+                 UsingImmediateCmdList) {
+        CacheType = HostInvisibleCounterBasedImmediateCacheType;
+      } else if (!CounterBasedEventEnabled && HostVisible) {
+        CacheType = HostVisibleCacheType;
+      } else {
+        CacheType = HostInvisibleCacheType;
+      }
     }
+
     return UR_RESULT_SUCCESS;
   }
 
diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp
@@ -486,7 +486,7 @@ ur_result_t urDeviceGetInfo(
     // TODO: To find out correct value
     return ReturnValue("");
   case UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP:
-    return ReturnValue(UR_DEVICE_INFO_LOW_POWER_EVENTS_EXP);
+    return ReturnValue(static_cast<ur_bool_t>(true));
   case UR_DEVICE_INFO_QUEUE_PROPERTIES:
     return ReturnValue(
         ur_queue_flag_t(UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE |
diff --git a/source/adapters/level_zero/event.cpp b/source/adapters/level_zero/event.cpp
@@ -438,7 +438,7 @@ ur_result_t urEnqueueEventsWaitWithBarrier(
       ur_queue_handle_t, uint32_t, const ur_event_handle_t *,
       ur_event_handle_t *, bool)>(EnqueueEventsWaitWithBarrier)(
       Queue, NumEventsInWaitList, EventWaitList, OutEvent,
-      Queue->interruptBasedEventsEnabled());
+      Queue == nullptr ? false : Queue->InterruptBasedEventsEnabled);
 }
 
 ur_result_t urEnqueueEventsWaitWithBarrierExt(
@@ -470,7 +470,7 @@ ur_result_t urEnqueueEventsWaitWithBarrierExt(
         ur_queue_handle_t, uint32_t, const ur_event_handle_t *,
         ur_event_handle_t *, bool)>(EnqueueEventsWaitWithBarrier)(
         Queue, NumEventsInWaitList, EventWaitList, OutEvent,
-        Queue->interruptBasedEventsEnabled());
+        Queue ? Queue->InterruptBasedEventsEnabled : false);
   }
 }
 
@@ -1342,7 +1342,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   }
 
   if (auto CachedEvent = Context->getEventFromContextCache(
-          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled)) {
+          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled,
+          InterruptBasedEventEnabled)) {
     *RetEvent = CachedEvent;
     return UR_RESULT_SUCCESS;
   }
@@ -1355,7 +1356,7 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   if (auto Res = Context->getFreeSlotInExistingOrNewPool(
           ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
           CounterBasedEventEnabled, UsingImmediateCommandlists,
-          Queue->interruptBasedEventsEnabled()))
+          InterruptBasedEventEnabled))
     return Res;
 
   ZeStruct<ze_event_desc_t> ZeEventDesc;
diff --git a/source/adapters/level_zero/queue.cpp b/source/adapters/level_zero/queue.cpp
@@ -1192,10 +1192,20 @@ ur_queue_handle_t_::ur_queue_handle_t_(
     }
     return std::atoi(UrRet) != 0;
   }();
+  static const bool useInterruptBasedEvents = [] {
+    const char *UrRet = std::getenv("UR_L0_USE_INTERRUPT_BASED_EVENTS");
+    if (!UrRet) {
+      return true;
+    }
+    return std::atoi(UrRet) != 0;
+  }();
   this->CounterBasedEventsEnabled =
       UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() &&
       useDriverCounterBasedEvents &&
       Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound;
+  this->InterruptBasedEventsEnabled = useInterruptBasedEvents &&
+                                      isLowPowerEvents() && isInOrderQueue() &&
+                                      Device->useDriverInOrderLists();
 }
 
 void ur_queue_handle_t_::adjustBatchSizeForFullBatch(bool IsCopy) {
@@ -1494,11 +1504,6 @@ bool ur_queue_handle_t_::doReuseDiscardedEvents() {
   return ReuseDiscardedEvents && isInOrderQueue() && isDiscardEvents();
 }
 
-bool ur_queue_handle_t_::interruptBasedEventsEnabled() {
-  return isInOrderQueue() && Device->useDriverInOrderLists() &&
-         isLowPowerEvents();
-}
-
 ur_result_t
 ur_queue_handle_t_::resetDiscardedEvent(ur_command_list_ptr_t CommandList) {
   if (LastCommandEvent && LastCommandEvent->IsDiscarded) {
@@ -1877,10 +1882,12 @@ ur_result_t setSignalEvent(ur_queue_handle_t Queue, bool UseCopyEngine,
 //        visible pool.
 // \param HostVisible tells if the event must be created in the
 //        host-visible pool. If not set then this function will decide.
-ur_result_t createEventAndAssociateQueue(
-    ur_queue_handle_t Queue, ur_event_handle_t *Event, ur_command_t CommandType,
-    ur_command_list_ptr_t CommandList, bool IsInternal, bool IsMultiDevice,
-    std::optional<bool> HostVisible, std::optional<bool> InterruptBasedEvents) {
+ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue,
+                                         ur_event_handle_t *Event,
+                                         ur_command_t CommandType,
+                                         ur_command_list_ptr_t CommandList,
+                                         bool IsInternal, bool IsMultiDevice,
+                                         std::optional<bool> HostVisible) {
 
   if (!HostVisible.has_value()) {
     // Internal/discarded events do not need host-scope visibility.
@@ -1896,9 +1903,7 @@ ur_result_t createEventAndAssociateQueue(
     UR_CALL(EventCreate(
         Queue->Context, Queue, IsMultiDevice, HostVisible.value(), Event,
         Queue->CounterBasedEventsEnabled, false /*ForceDisableProfiling*/,
-        InterruptBasedEvents.has_value()
-            ? InterruptBasedEvents.value()
-            : Queue->interruptBasedEventsEnabled()));
+        Queue->InterruptBasedEventsEnabled));
 
   (*Event)->UrQueue = Queue;
   (*Event)->CommandType = CommandType;
diff --git a/source/adapters/level_zero/queue.hpp b/source/adapters/level_zero/queue.hpp
@@ -375,6 +375,8 @@ struct ur_queue_handle_t_ : _ur_object {
   // Keeps track of whether we are using Counter-based Events
   bool CounterBasedEventsEnabled = false;
 
+  bool InterruptBasedEventsEnabled = false;
+
   // Map of all command lists used in this queue.
   ur_command_list_map_t CommandListMap;
 
@@ -533,8 +535,6 @@ struct ur_queue_handle_t_ : _ur_object {
   // queue.
   bool doReuseDiscardedEvents();
 
-  bool interruptBasedEventsEnabled();
-
   // Append command to provided command list to wait and reset the last event if
   // it is discarded and create new ur_event_handle_t wrapper using the same
   // native event and put it to the cache. We call this method after each
@@ -713,7 +713,7 @@ struct ur_queue_handle_t_ : _ur_object {
 ur_result_t createEventAndAssociateQueue(
     ur_queue_handle_t Queue, ur_event_handle_t *Event, ur_command_t CommandType,
     ur_command_list_ptr_t CommandList, bool IsInternal, bool IsMultiDevice,
-    std::optional<bool> HostVisible = std::nullopt, std::optional<bool> InterruptBasedEvents = std::nullopt);
+    std::optional<bool> HostVisible = std::nullopt);
 
 // This helper function checks to see if an event for a command can be included
 // at the end of a command list batch. This will only be true if the event does
