[UR] Support for counter-based events

Counter-based events implementation. As of now, only the creation of event, cmdlists, cmdqueue/pools are implemented.

Signed-off-by: Zhang, Winston <[email protected]>

Author: winstonzhang-intel

source/adapters/level_zero/context.cpp

@@ -468,7 +468,8 @@ static const uint32_t MaxNumEventsPerPool = [] {
 
 ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
     ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
-    bool ProfilingEnabled, ur_device_handle_t Device) {
+    bool ProfilingEnabled, ur_device_handle_t Device,
+    bool CounterBasedEventEnabled) {
   // Lock while updating event pool machinery.
   std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
 
@@ -510,6 +511,12 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
       ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
     if (ProfilingEnabled)
       ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_KERNEL_TIMESTAMP;
+    if (CounterBasedEventEnabled) {
+      ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
+      ze_event_pool_counter_based_exp_desc_t counterBasedExt = {
+          ZE_STRUCTURE_TYPE_COUNTER_BASED_EVENT_POOL_EXP_DESC};
+      ZeEventPoolDesc.pNext = &counterBasedExt;
+    }
     urPrint("ze_event_pool_desc_t flags set to: %d\n", ZeEventPoolDesc.flags);
 
     std::vector<ze_device_handle_t> ZeDevices;

source/adapters/level_zero/context.hpp

@@ -195,7 +195,8 @@ struct ur_context_handle_t_ : _ur_object {
   ur_result_t getFreeSlotInExistingOrNewPool(ze_event_pool_handle_t &, size_t &,
                                              bool HostVisible,
                                              bool ProfilingEnabled,
-                                             ur_device_handle_t Device);
+                                             ur_device_handle_t Device,
+                                             bool CounterBasedEventEnabled);
 
   // Get ur_event_handle_t from cache.
   ur_event_handle_t getEventFromContextCache(bool HostVisible,

source/adapters/level_zero/event.cpp

@@ -1049,7 +1049,8 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
 //
 ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
                         bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent) {
+                        ur_event_handle_t *RetEvent,
+                        std::optional<bool> CounterBasedEventEnabled) {
 
   bool ProfilingEnabled = !Queue || Queue->isProfilingEnabled();
 
@@ -1071,14 +1072,18 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   size_t Index = 0;
 
   if (auto Res = Context->getFreeSlotInExistingOrNewPool(
-          ZeEventPool, Index, HostVisible, ProfilingEnabled, Device))
+          ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
+          CounterBasedEventEnabled.has_value()
+              ? CounterBasedEventEnabled.value()
+              : false))
     return Res;
 
   ZeStruct<ze_event_desc_t> ZeEventDesc;
   ZeEventDesc.index = Index;
   ZeEventDesc.wait = 0;
 
-  if (HostVisible) {
+  if (HostVisible || (CounterBasedEventEnabled.has_value() &&
+                      CounterBasedEventEnabled.value())) {
     ZeEventDesc.signal = ZE_EVENT_SCOPE_FLAG_HOST;
   } else {
     //
@@ -1103,7 +1108,9 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   } catch (...) {
     return UR_RESULT_ERROR_UNKNOWN;
   }
-
+  (*RetEvent)->CounterBasedEventsEnabled =
+      CounterBasedEventEnabled.has_value() ? CounterBasedEventEnabled.value()
+                                           : false;
   if (HostVisible)
     (*RetEvent)->HostVisibleEvent =
         reinterpret_cast<ur_event_handle_t>(*RetEvent);
@@ -1124,8 +1131,8 @@ ur_result_t ur_event_handle_t_::reset() {
 
   if (!isHostVisible())
     HostVisibleEvent = nullptr;
-
-  ZE2UR_CALL(zeEventHostReset, (ZeEvent));
+  if (!usingCounterBasedEvents())
+    ZE2UR_CALL(zeEventHostReset, (ZeEvent));
   return UR_RESULT_SUCCESS;
 }
 

source/adapters/level_zero/event.hpp

@@ -29,9 +29,10 @@
 
 extern "C" {
 ur_result_t urEventReleaseInternal(ur_event_handle_t Event);
-ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
-                        bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent);
+ur_result_t
+EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
+            bool IsMultiDevice, bool HostVisible, ur_event_handle_t *RetEvent,
+            std::optional<bool> CounterBasedEventEnabled = std::nullopt);
 } // extern "C"
 
 // This is an experimental option that allows to disable caching of events in
@@ -222,6 +223,11 @@ struct ur_event_handle_t_ : _ur_object {
 
   // Get the host-visible event or create one and enqueue its signal.
   ur_result_t getOrCreateHostVisibleEvent(ze_event_handle_t &HostVisibleEvent);
+
+  // Keeps track of whether we are using Counter-based Events.
+  bool CounterBasedEventsEnabled = false;
+
+  bool usingCounterBasedEvents() const { return CounterBasedEventsEnabled; }
 };
 
 // Helper function to implement zeHostSynchronize.

source/adapters/level_zero/queue.cpp

@@ -955,6 +955,15 @@ ur_queue_handle_t_::ur_queue_handle_t_(
   ComputeCommandBatch.QueueBatchSize =
       ZeCommandListBatchComputeConfig.startSize();
   CopyCommandBatch.QueueBatchSize = ZeCommandListBatchCopyConfig.startSize();
+
+  static const bool useDriverCounterBasedEvents = [] {
+    const char *UrRet = std::getenv("UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS");
+    if (!UrRet)
+      return false;
+    return std::atoi(UrRet) != 0;
+  }();
+  this->counterBasedEventsEnabled =
+      isInOrderQueue() && useDriverCounterBasedEvents;
 }
 
 void ur_queue_handle_t_::adjustBatchSizeForFullBatch(bool IsCopy) {
@@ -1236,7 +1245,8 @@ bool ur_queue_handle_t_::doReuseDiscardedEvents() {
 
 ur_result_t
 ur_queue_handle_t_::resetDiscardedEvent(ur_command_list_ptr_t CommandList) {
-  if (LastCommandEvent && LastCommandEvent->IsDiscarded) {
+  if (!usingCounterBasedEvents() && LastCommandEvent &&
+      LastCommandEvent->IsDiscarded) {
     ZE2UR_CALL(zeCommandListAppendBarrier,
                (CommandList->first, nullptr, 1, &(LastCommandEvent->ZeEvent)));
     ZE2UR_CALL(zeCommandListAppendEventReset,
@@ -1364,6 +1374,10 @@ bool ur_queue_handle_t_::isInOrderQueue() const {
           0);
 }
 
+bool ur_queue_handle_t_::usingCounterBasedEvents() const {
+  return this->counterBasedEventsEnabled;
+}
+
 // Helper function to perform the necessary cleanup of the events from reset cmd
 // list.
 ur_result_t CleanupEventListFromResetCmdList(
@@ -1517,7 +1531,8 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue,
 
   if (*Event == nullptr)
     UR_CALL(EventCreate(Queue->Context, Queue, IsMultiDevice,
-                        HostVisible.value(), Event));
+                        HostVisible.value(), Event,
+                        Queue->usingCounterBasedEvents()));
 
   (*Event)->UrQueue = Queue;
   (*Event)->CommandType = CommandType;
@@ -1818,6 +1833,9 @@ ur_queue_handle_t_::ur_queue_group_t::getZeQueue(uint32_t *QueueGroupOrdinal) {
   if (QueueIndex != 0) {
     ZeCommandQueueDesc.flags = ZE_COMMAND_QUEUE_FLAG_EXPLICIT_ONLY;
   }
+  if (Queue->usingCounterBasedEvents()) {
+    ZeCommandQueueDesc.flags = ZE_COMMAND_QUEUE_FLAG_IN_ORDER;
+  }
 
   urPrint("[getZeQueue]: create queue ordinal = %d, index = %d "
           "(round robin in [%d, %d]) priority = %s\n",
@@ -1859,15 +1877,18 @@ ur_result_t ur_queue_handle_t_::createCommandList(
   ze_command_list_handle_t ZeCommandList;
 
   uint32_t QueueGroupOrdinal;
+  ZeStruct<ze_command_list_desc_t> ZeCommandListDesc;
+  if (usingCounterBasedEvents()) {
+    ZeCommandListDesc.flags = ZE_COMMAND_LIST_FLAG_IN_ORDER;
+    ZeCommandListDesc.stype = ZE_STRUCTURE_TYPE_COMMAND_LIST_DESC;
+  }
   auto &QGroup = getQueueGroup(UseCopyEngine);
   auto &ZeCommandQueue =
       ForcedCmdQueue ? *ForcedCmdQueue : QGroup.getZeQueue(&QueueGroupOrdinal);
   if (ForcedCmdQueue)
     QueueGroupOrdinal = QGroup.getCmdQueueOrdinal(ZeCommandQueue);
 
-  ZeStruct<ze_command_list_desc_t> ZeCommandListDesc;
   ZeCommandListDesc.commandQueueGroupOrdinal = QueueGroupOrdinal;
-
   ZE2UR_CALL(zeCommandListCreate, (Context->ZeContext, Device->ZeDevice,
                                    &ZeCommandListDesc, &ZeCommandList));
 
@@ -1877,9 +1898,10 @@ ur_result_t ur_queue_handle_t_::createCommandList(
   std::tie(CommandList, std::ignore) = CommandListMap.insert(
       std::pair<ze_command_list_handle_t, ur_command_list_info_t>(
           ZeCommandList, {ZeFence, false, false, ZeCommandQueue, ZeQueueDesc}));
-
-  UR_CALL(insertStartBarrierIfDiscardEventsMode(CommandList));
-  UR_CALL(insertActiveBarriers(CommandList, UseCopyEngine));
+  if (!usingCounterBasedEvents()) {
+    UR_CALL(insertStartBarrierIfDiscardEventsMode(CommandList));
+    UR_CALL(insertActiveBarriers(CommandList, UseCopyEngine));
+  }
   return UR_RESULT_SUCCESS;
 }
 

source/adapters/level_zero/queue.hpp

@@ -234,6 +234,9 @@ struct ur_queue_handle_t_ : _ur_object {
   // Keeps the properties of this queue.
   ur_queue_flags_t Properties;
 
+  // Keeps track of whether we are using Counter-based Events
+  bool counterBasedEventsEnabled = false;
+
   // Map of all command lists used in this queue.
   ur_command_list_map_t CommandListMap;
 
@@ -399,6 +402,8 @@ struct ur_queue_handle_t_ : _ur_object {
   // Returns true if the queue is a in-order queue.
   bool isInOrderQueue() const;
 
+  bool usingCounterBasedEvents() const;
+
   // Returns true if the queue has discard events property.
   bool isDiscardEvents() const;