[UR] Fix correct usage of In Order sync list given counting events

- Fixes a race condition when the wait events are counting events when
  executing the context's synchronous immediate command list.
- If a counting event is used in a non immediate command list, then an
  error will be returned.
- Two list types are created such that the in order sync list is only
  used when needed. In the future in order may be used everywhere, but
currently the in order list will only be used where needed.

Signed-off-by: Neil R. Spruit <[email protected]>

Author: nrspruit

source/adapters/level_zero/context.cpp

@@ -322,6 +322,19 @@ ur_result_t ur_context_handle_t_::initialize() {
   ZE2UR_CALL(
       zeCommandListCreateImmediate,
       (ZeContext, Device->ZeDevice, &ZeCommandQueueDesc, &ZeCommandListInit));
+
+  if (Device->useDriverInOrderLists() &&
+      Device->useDriverCounterBasedEvents()) {
+    logger::debug(
+        "L0 Synchronous Immediate Command List needed with In Order property.");
+    ZeStruct<ze_command_queue_desc_t> ZeCommandQueueDescInOrder;
+    ZeCommandQueueDescInOrder = ZeCommandQueueDesc;
+    ZeCommandQueueDescInOrder.flags |= ZE_COMMAND_LIST_FLAG_IN_ORDER;
+    ZE2UR_CALL(zeCommandListCreateImmediate,
+               (ZeContext, Device->ZeDevice, &ZeCommandQueueDescInOrder,
+                &ZeCommandListInitInOrder));
+    this->InOrderListInitEnabled = true;
+  }
   return UR_RESULT_SUCCESS;
 }
 
@@ -434,6 +447,15 @@ ur_result_t ur_context_handle_t_::finalize() {
   if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED)
     return ze2urResult(ZeResult);
 
+  if (this->InOrderListInitEnabled) {
+    // Destroy the in order command list used for initializations
+    auto ZeResultInOrder =
+        ZE_CALL_NOCHECK(zeCommandListDestroy, (ZeCommandListInitInOrder));
+    // Gracefully handle the case that L0 was already unloaded.
+    if (ZeResultInOrder && ZeResultInOrder != ZE_RESULT_ERROR_UNINITIALIZED)
+      return ze2urResult(ZeResultInOrder);
+  }
+
   std::scoped_lock<ur_mutex> Lock(ZeCommandListCacheMutex);
   for (auto &List : ZeComputeCommandListCache) {
     for (auto &Item : List.second) {

source/adapters/level_zero/context.hpp

@@ -85,6 +85,15 @@ struct ur_context_handle_t_ : _ur_object {
   // support of the multiple devices per context will be added.
   ze_command_list_handle_t ZeCommandListInit{};
 
+  // Immediate Level Zero command list for the device that has the property of
+  // in-order execution. This command list is used for memory copy operations.
+  // Due to different event types supported, one needs to use separate command
+  // list types given Counting Events are being used.
+  ze_command_list_handle_t ZeCommandListInitInOrder{};
+
+  // Indicates if the immediate command list with in-order property is enabled.
+  bool InOrderListInitEnabled = false;
+
   // Mutex for the immediate command list. Per the Level Zero spec memory copy
   // operations submitted to an immediate command list are not allowed to be
   // called from simultaneous threads.

source/adapters/level_zero/device.cpp

@@ -1528,6 +1528,20 @@ bool ur_device_handle_t_::useDriverInOrderLists() {
   return UseDriverInOrderLists;
 }
 
+bool ur_device_handle_t_::useDriverCounterBasedEvents() {
+  // Use counter-based events implementation from L0 driver.
+
+  static const bool DriverCounterBasedEventsEnabled = [] {
+    const char *UrRet = std::getenv("UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS");
+    if (!UrRet) {
+      return true;
+    }
+    return std::atoi(UrRet) != 0;
+  }();
+
+  return DriverCounterBasedEventsEnabled;
+}
+
 ur_result_t ur_device_handle_t_::initialize(int SubSubDeviceOrdinal,
                                             int SubSubDeviceIndex) {
   // Maintain various device properties cache.

source/adapters/level_zero/device.hpp

@@ -159,6 +159,9 @@ struct ur_device_handle_t_ : _ur_object {
   // Whether Adapter uses driver's implementation of in-order lists or not
   bool useDriverInOrderLists();
 
+  // Whether Adapter uses driver's implementation of counter-based events or not
+  bool useDriverCounterBasedEvents();
+
   // Returns whether immediate command lists are used on this device.
   ImmCmdlistMode ImmCommandListUsed{};
 

source/adapters/level_zero/memory.cpp

@@ -2165,7 +2165,13 @@ ur_result_t _ur_buffer::getZeHandle(char *&ZeHandle, access_mode_t AccessMode,
       waitlist.ZeEventList = nullptr;
       waitlist.Length = 0;
       uint32_t EventListIndex = 0;
+      bool InOrderListRequired = false;
       for (unsigned i = 0; i < numWaitEvents; ++i) {
+        // If any of the wait events are of the type "counter" then we need to
+        // use an in order list otherwise the event is invalid in this list.
+        if (phWaitEvents[i]->CounterBasedEventsEnabled) {
+          InOrderListRequired = true;
+        }
         if (phWaitEvents[i]->HostVisibleEvent) {
           ZE2UR_CALL(zeEventHostSynchronize,
                      (phWaitEvents[i]->ZeEvent, UINT64_MAX));
@@ -2181,9 +2187,14 @@ ur_result_t _ur_buffer::getZeHandle(char *&ZeHandle, access_mode_t AccessMode,
         }
       }
       if (waitlist.Length > 0) {
-        ZE2UR_CALL(zeCommandListAppendWaitOnEvents,
-                   (UrContext->ZeCommandListInit, waitlist.Length,
-                    waitlist.ZeEventList));
+        ze_command_list_handle_t ZeCommandListForInit =
+            UrContext->ZeCommandListInit;
+        if (InOrderListRequired) {
+          ZeCommandListForInit = UrContext->ZeCommandListInitInOrder;
+        }
+        ZE2UR_CALL(
+            zeCommandListAppendWaitOnEvents,
+            (ZeCommandListForInit, waitlist.Length, waitlist.ZeEventList));
       }
 
       // Copy valid buffer data to this allocation.

source/adapters/level_zero/queue.cpp

@@ -1186,16 +1186,9 @@ ur_queue_handle_t_::ur_queue_handle_t_(
       ZeCommandListBatchComputeConfig.startSize();
   CopyCommandBatch.QueueBatchSize = ZeCommandListBatchCopyConfig.startSize();
 
-  static const bool useDriverCounterBasedEvents = [] {
-    const char *UrRet = std::getenv("UR_L0_USE_DRIVER_COUNTER_BASED_EVENTS");
-    if (!UrRet) {
-      return true;
-    }
-    return std::atoi(UrRet) != 0;
-  }();
   this->CounterBasedEventsEnabled =
       UsingImmCmdLists && isInOrderQueue() && Device->useDriverInOrderLists() &&
-      useDriverCounterBasedEvents &&
+      Device->useDriverCounterBasedEvents() &&
       Device->Platform->ZeDriverEventPoolCountingEventsExtensionFound;
   this->InterruptBasedEventsEnabled =
       isLowPowerEvents() && isInOrderQueue() && Device->useDriverInOrderLists();