[L0] Support for counter-based events using L0 driver

winstonzhang-intel · winstonzhang-intel · commit 0c9b051274a2 · 2024-03-13T17:49:20.000Z
Signed-off-by: Zhang, Winston &lt;winston.zhang@intel.com&gt;
diff --git a/source/adapters/level_zero/context.cpp b/source/adapters/level_zero/context.cpp
@@ -468,7 +468,8 @@ static const uint32_t MaxNumEventsPerPool = [] {
 
 ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
     ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
-    bool ProfilingEnabled, ur_device_handle_t Device) {
+    bool ProfilingEnabled, ur_device_handle_t Device,
+    bool CounterBasedEventEnabled, bool UsingImmCmdList) {
   // Lock while updating event pool machinery.
   std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
 
@@ -477,8 +478,8 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
   if (Device) {
     ZeDevice = Device->ZeDevice;
   }
-  std::list<ze_event_pool_handle_t> *ZePoolCache =
-      getZeEventPoolCache(HostVisible, ProfilingEnabled, ZeDevice);
+  std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
+      HostVisible, ProfilingEnabled, CounterBasedEventEnabled, ZeDevice);
 
   if (!ZePoolCache->empty()) {
     if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -510,6 +511,18 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
       ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
     if (ProfilingEnabled)
       ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_KERNEL_TIMESTAMP;
+    if (CounterBasedEventEnabled) {
+      ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
+      ze_event_pool_counter_based_exp_desc_t counterBasedExt = {
+          ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC};
+      if (UsingImmCmdList) {
+        counterBasedExt.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE;
+      } else {
+        counterBasedExt.flags =
+            ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_NON_IMMEDIATE;
+      }
+      ZeEventPoolDesc.pNext = &counterBasedExt;
+    }
     urPrint("ze_event_pool_desc_t flags set to: %d\n", ZeEventPoolDesc.flags);
 
     std::vector<ze_device_handle_t> ZeDevices;
@@ -580,8 +593,9 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
     ZeDevice = Event->UrQueue->Device->ZeDevice;
   }
 
-  std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
-      Event->isHostVisible(), Event->isProfilingEnabled(), ZeDevice);
+  std::list<ze_event_pool_handle_t> *ZePoolCache =
+      getZeEventPoolCache(Event->isHostVisible(), Event->isProfilingEnabled(),
+                          Event->usingCounterBasedEvents(), ZeDevice);
 
   // Put the empty pool to the cache of the pools.
   if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)
diff --git a/source/adapters/level_zero/context.hpp b/source/adapters/level_zero/context.hpp
@@ -141,9 +141,9 @@ struct ur_context_handle_t_ : _ur_object {
   // head.
   //
   // Cache of event pools to which host-visible events are added to.
-  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{4};
+  std::vector<std::list<ze_event_pool_handle_t>> ZeEventPoolCache{8};
   std::vector<std::unordered_map<ze_device_handle_t, size_t>>
-      ZeEventPoolCacheDeviceMap{4};
+      ZeEventPoolCacheDeviceMap{8};
 
   // This map will be used to determine if a pool is full or not
   // by storing number of empty slots available in the pool.
@@ -194,7 +194,9 @@ struct ur_context_handle_t_ : _ur_object {
   ur_result_t getFreeSlotInExistingOrNewPool(ze_event_pool_handle_t &, size_t &,
                                              bool HostVisible,
                                              bool ProfilingEnabled,
-                                             ur_device_handle_t Device);
+                                             ur_device_handle_t Device,
+                                             bool CounterBasedEventEnabled,
+                                             bool UsingImmCmdList);
 
   // Get ur_event_handle_t from cache.
   ur_event_handle_t getEventFromContextCache(bool HostVisible,
@@ -206,36 +208,43 @@ struct ur_context_handle_t_ : _ur_object {
 
   std::list<ze_event_pool_handle_t> *
   getZeEventPoolCache(bool HostVisible, bool WithProfiling,
+                      bool CounterBasedEventEnabled,
                       ze_device_handle_t ZeDevice) {
-    if (HostVisible) {
-      if (ZeDevice) {
-        auto ZeEventPoolCacheMap = WithProfiling
-                                       ? &ZeEventPoolCacheDeviceMap[0]
-                                       : &ZeEventPoolCacheDeviceMap[1];
-        if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
-          ZeEventPoolCache.emplace_back();
-          ZeEventPoolCacheMap->insert(
-              std::make_pair(ZeDevice, ZeEventPoolCache.size() - 1));
-        }
-        return &ZeEventPoolCache[(*ZeEventPoolCacheMap)[ZeDevice]];
-      } else {
-        return WithProfiling ? &ZeEventPoolCache[0] : &ZeEventPoolCache[1];
+    int profiling_index_a, profiling_index_b;
+    calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
+                        &profiling_index_a, &profiling_index_b);
+    if (ZeDevice) {
+      auto ZeEventPoolCacheMap =
+          WithProfiling ? &ZeEventPoolCacheDeviceMap[profiling_index_a]
+                        : &ZeEventPoolCacheDeviceMap[profiling_index_b];
+      if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
+        ZeEventPoolCache.emplace_back();
+        ZeEventPoolCacheMap->insert(
+            std::make_pair(ZeDevice, ZeEventPoolCache.size() - 1));
       }
+      return &ZeEventPoolCache[(*ZeEventPoolCacheMap)[ZeDevice]];
     } else {
-      if (ZeDevice) {
-        auto ZeEventPoolCacheMap = WithProfiling
-                                       ? &ZeEventPoolCacheDeviceMap[2]
-                                       : &ZeEventPoolCacheDeviceMap[3];
-        if (ZeEventPoolCacheMap->find(ZeDevice) == ZeEventPoolCacheMap->end()) {
-          ZeEventPoolCache.emplace_back();
-          ZeEventPoolCacheMap->insert(
-              std::make_pair(ZeDevice, ZeEventPoolCache.size() - 1));
-        }
-        return &ZeEventPoolCache[(*ZeEventPoolCacheMap)[ZeDevice]];
-      } else {
-        return WithProfiling ? &ZeEventPoolCache[2] : &ZeEventPoolCache[3];
-      }
+      return WithProfiling ? &ZeEventPoolCache[profiling_index_a]
+                           : &ZeEventPoolCache[profiling_index_b];
+    }
+  }
+
+  ur_result_t calculateCacheIndex(bool HostVisible,
+                                  bool CounterBasedEventEnabled,
+                                  int *profiling_index_a,
+                                  int *profiling_index_b) {
+    if (HostVisible) {
+      *profiling_index_a = 0;
+      *profiling_index_b = 1;
+    } else {
+      *profiling_index_a = 2;
+      *profiling_index_b = 3;
+    }
+    if (CounterBasedEventEnabled) {
+      *profiling_index_a += 4;
+      *profiling_index_b += 4;
     }
+    return UR_RESULT_SUCCESS;
   }
 
   // Decrement number of events living in the pool upon event destroy
diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp
@@ -1054,6 +1054,19 @@ bool ur_device_handle_t_::useRelaxedAllocationLimits() {
   return EnableRelaxedAllocationLimits;
 }
 
+bool ur_device_handle_t_::useDriverInOrderLists() {
+  // Use in-order lists implementation from L0 driver instead
+  // of adapter's implementation.
+  static const bool UseDriverInOrderLists = [] {
+    const char *UrRet = std::getenv("UR_L0_USE_DRIVER_INORDER_LISTS");
+    if (!UrRet)
+      return false;
+    return std::atoi(UrRet) != 0;
+  }();
+
+  return UseDriverInOrderLists;
+}
+
 ur_result_t ur_device_handle_t_::initialize(int SubSubDeviceOrdinal,
                                             int SubSubDeviceIndex) {
   // Maintain various device properties cache.
diff --git a/source/adapters/level_zero/device.hpp b/source/adapters/level_zero/device.hpp
@@ -143,6 +143,9 @@ struct ur_device_handle_t_ : _ur_object {
   // Read env settings to select immediate commandlist mode.
   ImmCmdlistMode useImmediateCommandLists();
 
+  // Whether Adapter uses driver's implementation of in-order lists or not
+  bool useDriverInOrderLists();
+
   // Returns whether immediate command lists are used on this device.
   ImmCmdlistMode ImmCommandListUsed{};
 
diff --git a/source/adapters/level_zero/event.cpp b/source/adapters/level_zero/event.cpp
@@ -43,6 +43,20 @@ static const bool UseMultipleCmdlistBarriers = [] {
   return std::atoi(UseMultipleCmdlistBarriersFlag) > 0;
 }();
 
+bool WaitListEmptyOrAllEventsFromSameQueue(
+    ur_queue_handle_t Queue, uint32_t NumEventsInWaitList,
+    const ur_event_handle_t *EventWaitList) {
+  if (!NumEventsInWaitList)
+    return true;
+
+  for (uint32_t i = 0; i < NumEventsInWaitList; ++i) {
+    if (Queue != EventWaitList[i]->UrQueue)
+      return false;
+  }
+
+  return true;
+}
+
 UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWait(
     ur_queue_handle_t Queue,      ///< [in] handle of the queue object
     uint32_t NumEventsInWaitList, ///< [in] size of the event wait list
@@ -206,21 +220,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWaitWithBarrier(
   bool IsInternal = OutEvent == nullptr;
   ur_event_handle_t *Event = OutEvent ? OutEvent : &InternalEvent;
 
-  auto WaitListEmptyOrAllEventsFromSameQueue = [Queue, NumEventsInWaitList,
-                                                EventWaitList]() {
-    if (!NumEventsInWaitList)
-      return true;
-
-    for (uint32_t I = 0; I < NumEventsInWaitList; ++I)
-      if (Queue != EventWaitList[I]->UrQueue)
-        return false;
-
-    return true;
-  };
-
   // For in-order queue and wait-list which is empty or has events from
   // the same queue just use the last command event as the barrier event.
-  if (Queue->isInOrderQueue() && WaitListEmptyOrAllEventsFromSameQueue() &&
+  if (Queue->isInOrderQueue() &&
+      WaitListEmptyOrAllEventsFromSameQueue(Queue, NumEventsInWaitList,
+                                            EventWaitList) &&
       Queue->LastCommandEvent && !Queue->LastCommandEvent->IsDiscarded) {
     UR_CALL(urEventRetain(Queue->LastCommandEvent));
     *Event = Queue->LastCommandEvent;
@@ -1053,7 +1057,8 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
 //
 ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
                         bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent) {
+                        ur_event_handle_t *RetEvent,
+                        bool CounterBasedEventEnabled) {
 
   bool ProfilingEnabled = !Queue || Queue->isProfilingEnabled();
 
@@ -1075,14 +1080,15 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   size_t Index = 0;
 
   if (auto Res = Context->getFreeSlotInExistingOrNewPool(
-          ZeEventPool, Index, HostVisible, ProfilingEnabled, Device))
+          ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
+          CounterBasedEventEnabled, Queue->UsingImmCmdLists))
     return Res;
 
   ZeStruct<ze_event_desc_t> ZeEventDesc;
   ZeEventDesc.index = Index;
   ZeEventDesc.wait = 0;
 
-  if (HostVisible) {
+  if (HostVisible || CounterBasedEventEnabled) {
     ZeEventDesc.signal = ZE_EVENT_SCOPE_FLAG_HOST;
   } else {
     //
@@ -1107,7 +1113,7 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   } catch (...) {
     return UR_RESULT_ERROR_UNKNOWN;
   }
-
+  (*RetEvent)->CounterBasedEventsEnabled = CounterBasedEventEnabled;
   if (HostVisible)
     (*RetEvent)->HostVisibleEvent =
         reinterpret_cast<ur_event_handle_t>(*RetEvent);
@@ -1128,8 +1134,8 @@ ur_result_t ur_event_handle_t_::reset() {
 
   if (!isHostVisible())
     HostVisibleEvent = nullptr;
-
-  ZE2UR_CALL(zeEventHostReset, (ZeEvent));
+  if (!usingCounterBasedEvents())
+    ZE2UR_CALL(zeEventHostReset, (ZeEvent));
   return UR_RESULT_SUCCESS;
 }
 
@@ -1189,6 +1195,23 @@ ur_result_t _ur_ze_event_list_t::createAndRetainUrZeEventList(
       CurQueue->LastCommandEvent && CurQueue->LastCommandEvent->IsDiscarded)
     IncludeLastCommandEvent = false;
 
+  // If we are using L0 native implementation for handling in-order queues,
+  // then we don't need to add the last enqueued event into the waitlist, as
+  // the native driver implementation will already ensure in-order semantics.
+  // The only exception is when a different immediate command was last used on
+  // the same UR Queue.
+  if (CurQueue->Device->useDriverInOrderLists() && CurQueue->isInOrderQueue() &&
+      CurQueue->UsingImmCmdLists) {
+    auto QueueGroup = CurQueue->getQueueGroup(UseCopyEngine);
+    uint32_t QueueGroupOrdinal, QueueIndex;
+    auto NextIndex = QueueGroup.getQueueIndex(&QueueGroupOrdinal, &QueueIndex,
+                                              /*QueryOnly */ true);
+    auto NextImmCmdList = QueueGroup.ImmCmdLists[NextIndex];
+    IncludeLastCommandEvent &=
+        CurQueue->LastUsedCommandList != CurQueue->CommandListMap.end() &&
+        NextImmCmdList != CurQueue->LastUsedCommandList;
+  }
+
   try {
     uint32_t TmpListLength = 0;
 
@@ -1205,6 +1228,16 @@ ur_result_t _ur_ze_event_list_t::createAndRetainUrZeEventList(
       this->UrEventList = new ur_event_handle_t[EventListLength];
     }
 
+    // For in-order queue and wait-list which is empty or has events only from
+    // the same queue then we don't need to wait on any other additional events
+    if (CurQueue->Device->useDriverInOrderLists() &&
+        CurQueue->isInOrderQueue() &&
+        WaitListEmptyOrAllEventsFromSameQueue(CurQueue, EventListLength,
+                                              EventList)) {
+      this->Length = TmpListLength;
+      return UR_RESULT_SUCCESS;
+    }
+
     if (EventListLength > 0) {
       for (uint32_t I = 0; I < EventListLength; I++) {
         {
diff --git a/source/adapters/level_zero/event.hpp b/source/adapters/level_zero/event.hpp
@@ -31,7 +31,8 @@ extern "C" {
 ur_result_t urEventReleaseInternal(ur_event_handle_t Event);
 ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
                         bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent);
+                        ur_event_handle_t *RetEvent,
+                        bool CounterBasedEventEnabled = false);
 } // extern "C"
 
 // This is an experimental option that allows to disable caching of events in
@@ -222,6 +223,11 @@ struct ur_event_handle_t_ : _ur_object {
 
   // Get the host-visible event or create one and enqueue its signal.
   ur_result_t getOrCreateHostVisibleEvent(ze_event_handle_t &HostVisibleEvent);
+
+  // Keeps track of whether we are using Counter-based Events.
+  bool CounterBasedEventsEnabled = false;
+
+  bool usingCounterBasedEvents() const { return CounterBasedEventsEnabled; }
 };
 
 // Helper function to implement zeHostSynchronize.
diff --git a/source/adapters/level_zero/kernel.cpp b/source/adapters/level_zero/kernel.cpp
@@ -214,8 +214,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunch(
   // the code can do a urKernelRelease on this kernel.
   (*Event)->CommandData = (void *)Kernel;
 
-  // Increment the reference count of the Kernel and indicate that the Kernel is
-  // in use. Once the event has been signalled, the code in
+  // Increment the reference count of the Kernel and indicate that the Kernel
+  // is in use. Once the event has been signalled, the code in
   // CleanupCompletedEvent(Event) will do a urKernelRelease to update the
   // reference count on the kernel, using the kernel saved in CommandData.
   UR_CALL(urKernelRetain(Kernel));
diff --git a/source/adapters/level_zero/queue.cpp b/source/adapters/level_zero/queue.cpp
diff --git a/source/adapters/level_zero/queue.hpp b/source/adapters/level_zero/queue.hpp
