[L0] Simplify and allow disabling of queue sync non blocking

Signed-off-by: Neil R. Spruit <[email protected]>

Author: nrspruit

source/adapters/level_zero/common.hpp

@@ -231,6 +231,16 @@ static const uint32_t UrL0Serialize = [] {
   return SerializeModeValue;
 }();
 
+static const uint32_t UrL0QueueSyncNonBlocking = [] {
+  const char *UrL0QueueSyncNonBlocking =
+      std::getenv("UR_L0_QUEUE_SYNCHRONIZE_NON_BLOCKING");
+  uint32_t L0QueueSyncLockingModeValue = 1;
+  if (UrL0QueueSyncNonBlocking) {
+    L0QueueSyncLockingModeValue = std::atoi(UrL0QueueSyncNonBlocking);
+  }
+  return L0QueueSyncLockingModeValue;
+}();
+
 // This class encapsulates actions taken along with a call to Level Zero API.
 class ZeCall {
 private:

source/adapters/level_zero/event.cpp

@@ -113,8 +113,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWait(
     //
     // TODO: find a way to do that without blocking the host.
 
-    std::unique_lock<ur_shared_mutex> QueueLock(Queue->Mutex, std::defer_lock);
-    QueueLock.lock();
+    // Lock automatically releases when this goes out of scope.
+    std::scoped_lock<ur_shared_mutex> lock(Queue->Mutex);
 
     if (OutEvent) {
       UR_CALL(createEventAndAssociateQueue(Queue, OutEvent,
@@ -123,9 +123,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWait(
                                            /* IsInternal */ false));
     }
 
-    QueueLock.unlock();
     UR_CALL(Queue->synchronize());
-    QueueLock.lock();
 
     if (OutEvent) {
       Queue->LastCommandEvent = reinterpret_cast<ur_event_handle_t>(*OutEvent);

source/adapters/level_zero/queue.cpp

@@ -395,8 +395,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueRelease(
 
   std::vector<ur_event_handle_t> EventListToCleanup;
   {
-    std::unique_lock<ur_shared_mutex> QueueLock(Queue->Mutex, std::defer_lock);
-    QueueLock.lock();
+    std::scoped_lock<ur_shared_mutex> Lock(Queue->Mutex);
 
     if ((--Queue->RefCountExternal) != 0)
       return UR_RESULT_SUCCESS;
@@ -411,11 +410,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueRelease(
     auto Res = Queue->executeAllOpenCommandLists();
 
     // Make sure all commands get executed.
-    if (Res == UR_RESULT_SUCCESS) {
-      QueueLock.unlock();
+    if (Res == UR_RESULT_SUCCESS)
       UR_CALL(Queue->synchronize());
-      QueueLock.lock();
-    }
 
     // Destroy all the fences created associated with this queue.
     for (auto it = Queue->CommandListMap.begin();
@@ -628,6 +624,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueFinish(
     ur_queue_handle_t UrQueue ///< [in] handle of the queue to be finished.
 ) {
   if (UrQueue->UsingImmCmdLists) {
+    // Lock automatically releases when this goes out of scope.
+    std::scoped_lock<ur_shared_mutex> Lock(UrQueue->Mutex);
     UR_CALL(UrQueue->synchronize());
   } else {
     std::unique_lock<ur_shared_mutex> Lock(UrQueue->Mutex);
@@ -1314,9 +1312,17 @@ ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) {
       for (auto &QueueGroup : QueueMap)
         for (auto &ZeQueue : QueueGroup.second.ZeQueues)
           if (ZeQueue) {
-            {
-              std::scoped_lock<ur_shared_mutex> Lock(
-                  UrQueue->zeQueueInUseMutex);
+            if (UrL0QueueSyncNonBlocking) {
+              {
+                std::scoped_lock<ur_shared_mutex> Lock(
+                    UrQueue->zeQueueInUseMutex);
+                auto ZeResult =
+                    ZE_CALL_NOCHECK(zeCommandQueueDestroy, (ZeQueue));
+                // Gracefully handle the case that L0 was already unloaded.
+                if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED)
+                  return ze2urResult(ZeResult);
+              }
+            } else {
               auto ZeResult = ZE_CALL_NOCHECK(zeCommandQueueDestroy, (ZeQueue));
               // Gracefully handle the case that L0 was already unloaded.
               if (ZeResult && ZeResult != ZE_RESULT_ERROR_UNINITIALIZED)
@@ -1396,24 +1402,26 @@ ur_result_t CleanupEventListFromResetCmdList(
 // runtime. Need to investigate whether relase can be done earlier, at sync
 // points such as this, to reduce total number of active Events.
 ur_result_t ur_queue_handle_t_::synchronize() {
-  this->Mutex.lock();
-  if (!Healthy) {
-    this->Mutex.unlock();
+  if (!Healthy)
     return UR_RESULT_SUCCESS;
-  }
 
   auto syncImmCmdList = [](ur_queue_handle_t_ *Queue,
                            ur_command_list_ptr_t ImmCmdList) {
     if (ImmCmdList == Queue->CommandListMap.end())
       return UR_RESULT_SUCCESS;
 
     // wait for all commands previously submitted to this immediate command list
-    Queue->Mutex.unlock();
-    {
-      std::scoped_lock<ur_shared_mutex> Lock(Queue->zeQueueInUseMutex);
+    if (UrL0QueueSyncNonBlocking) {
+      Queue->Mutex.unlock();
+      {
+        std::scoped_lock<ur_shared_mutex> Lock(Queue->zeQueueInUseMutex);
+        ZE2UR_CALL(zeCommandListHostSynchronize,
+                   (ImmCmdList->first, UINT64_MAX));
+      }
+      Queue->Mutex.lock();
+    } else {
       ZE2UR_CALL(zeCommandListHostSynchronize, (ImmCmdList->first, UINT64_MAX));
     }
-    Queue->Mutex.lock();
 
     // Cleanup all events from the synced command list.
     CleanupEventListFromResetCmdList(ImmCmdList->second.EventList, true);
@@ -1427,12 +1435,16 @@ ur_result_t ur_queue_handle_t_::synchronize() {
     // zero handle can have device scope, so we can't synchronize the last
     // event.
     if (isInOrderQueue() && !LastCommandEvent->IsDiscarded) {
-      this->Mutex.unlock();
-      {
-        std::scoped_lock<ur_shared_mutex> Lock(this->zeQueueInUseMutex);
+      if (UrL0QueueSyncNonBlocking) {
+        this->Mutex.unlock();
+        {
+          std::scoped_lock<ur_shared_mutex> Lock(this->zeQueueInUseMutex);
+          ZE2UR_CALL(zeHostSynchronize, (LastCommandEvent->ZeEvent));
+        }
+        this->Mutex.lock();
+      } else {
         ZE2UR_CALL(zeHostSynchronize, (LastCommandEvent->ZeEvent));
       }
-      this->Mutex.lock();
 
       // clean up all events known to have been completed as well,
       // so they can be reused later
@@ -1460,13 +1472,17 @@ ur_result_t ur_queue_handle_t_::synchronize() {
           } else {
             for (auto &ZeQueue : QueueGroup.second.ZeQueues)
               if (ZeQueue) {
-                this->Mutex.unlock();
-                {
-                  std::scoped_lock<ur_shared_mutex> Lock(
-                      this->zeQueueInUseMutex);
+                if (UrL0QueueSyncNonBlocking) {
+                  this->Mutex.unlock();
+                  {
+                    std::scoped_lock<ur_shared_mutex> Lock(
+                        this->zeQueueInUseMutex);
+                    ZE2UR_CALL(zeHostSynchronize, (ZeQueue));
+                  }
+                  this->Mutex.lock();
+                } else {
                   ZE2UR_CALL(zeHostSynchronize, (ZeQueue));
                 }
-                this->Mutex.lock();
               }
           }
         }
@@ -1477,12 +1493,9 @@ ur_result_t ur_queue_handle_t_::synchronize() {
 
   // With the entire queue synchronized, the active barriers must be done so we
   // can remove them.
-  if (auto Res = ActiveBarriers.clear()) {
-    this->Mutex.unlock();
+  if (auto Res = ActiveBarriers.clear())
     return Res;
-  }
 
-  this->Mutex.unlock();
   return UR_RESULT_SUCCESS;
 }