[L0] Refactor of boolean variables for Event

This now includes the interrupt changes and is rebased on top of
getEventCache changes.

Signed-off-by: Zhang, Winston <[email protected]>

Author: winstonzhang-intel

source/adapters/level_zero/common.hpp

@@ -240,6 +240,18 @@ enum {
       2, // blocking UR calls, where supported (usually in enqueue commands)
 };
 
+using ur_event_flags_t = uint32_t;
+enum ur_event_flag_t {
+  EVENT_FLAG_HOST_VISIBLE = UR_BIT(0),
+  EVENT_FLAG_WITH_PROFILING = UR_BIT(1),
+  EVENT_FLAG_COUNTER = UR_BIT(2),
+  EVENT_FLAG_INTERRUPT = UR_BIT(3),
+  EVENT_FLAG_IMM_CMDLIST = UR_BIT(4),
+  EVENT_FLAG_MULTIDEVICE = UR_BIT(6),
+  EVENT_FLAG_DEVICE = UR_BIT(7), // if set, subsequent bits are device id
+  MAX_EVENT_FLAG_BITS = 8,
+};
+
 static const uint32_t UrL0Serialize = [] {
   const char *ZeSerializeMode = std::getenv("ZE_SERIALIZE");
   const char *UrL0SerializeMode = std::getenv("UR_L0_SERIALIZE");

source/adapters/level_zero/context.cpp

@@ -491,10 +491,8 @@ static const uint32_t MaxNumEventsPerPool = [] {
 }();
 
 ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
-    ze_event_pool_handle_t &Pool, size_t &Index, bool HostVisible,
-    bool ProfilingEnabled, ur_device_handle_t Device,
-    bool CounterBasedEventEnabled, bool UsingImmCmdList,
-    bool InterruptBasedEventEnabled) {
+    ze_event_pool_handle_t &Pool, size_t &Index, ur_event_flags_t Flags,
+    ur_device_handle_t Device) {
   // Lock while updating event pool machinery.
   std::scoped_lock<ur_mutex> Lock(ZeEventPoolCacheMutex);
 
@@ -503,9 +501,8 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
   if (Device) {
     ZeDevice = Device->ZeDevice;
   }
-  std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
-      HostVisible, ProfilingEnabled, CounterBasedEventEnabled, UsingImmCmdList,
-      InterruptBasedEventEnabled, ZeDevice);
+  std::list<ze_event_pool_handle_t> *ZePoolCache =
+      getZeEventPoolCache(Flags, ZeDevice);
 
   if (!ZePoolCache->empty()) {
     if (NumEventsAvailableInEventPool[ZePoolCache->front()] == 0) {
@@ -536,14 +533,14 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
     ZeEventPoolDesc.count = MaxNumEventsPerPool;
     ZeEventPoolDesc.flags = 0;
     ZeEventPoolDesc.pNext = nullptr;
-    if (HostVisible)
+    if (Flags & EVENT_FLAG_HOST_VISIBLE)
       ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
-    if (ProfilingEnabled)
+    if (Flags & EVENT_FLAG_WITH_PROFILING)
       ZeEventPoolDesc.flags |= ZE_EVENT_POOL_FLAG_KERNEL_TIMESTAMP;
     logger::debug("ze_event_pool_desc_t flags set to: {}",
                   ZeEventPoolDesc.flags);
-    if (CounterBasedEventEnabled) {
-      if (UsingImmCmdList) {
+    if (Flags & EVENT_FLAG_COUNTER) {
+      if (Flags & EVENT_FLAG_IMM_CMDLIST) {
         counterBasedExt.flags = ZE_EVENT_POOL_COUNTER_BASED_EXP_FLAG_IMMEDIATE;
       } else {
         counterBasedExt.flags =
@@ -553,7 +550,7 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
                     counterBasedExt.flags);
       ZeEventPoolDesc.pNext = &counterBasedExt;
     }
-    if (InterruptBasedEventEnabled) {
+    if (Flags & EVENT_FLAG_INTERRUPT) {
       ze_intel_event_sync_mode_exp_desc_t eventSyncMode = {
           ZE_INTEL_STRUCTURE_TYPE_EVENT_SYNC_MODE_EXP_DESC, nullptr, 0};
       eventSyncMode.syncModeFlags =
@@ -585,18 +582,17 @@ ur_result_t ur_context_handle_t_::getFreeSlotInExistingOrNewPool(
   return UR_RESULT_SUCCESS;
 }
 
-ur_event_handle_t ur_context_handle_t_::getEventFromContextCache(
-    bool HostVisible, bool WithProfiling, ur_device_handle_t Device,
-    bool CounterBasedEventEnabled, bool InterruptBasedEventEnabled) {
+ur_event_handle_t
+ur_context_handle_t_::getEventFromContextCache(ur_event_flags_t Flags,
+                                               ur_device_handle_t Device) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
-  auto Cache =
-      getEventCache(HostVisible, WithProfiling, Device,
-                    CounterBasedEventEnabled, InterruptBasedEventEnabled);
+  auto Cache = getEventCache(Flags, Device);
   if (Cache->empty()) {
-    logger::info("Cache empty (Host Visible: {}, Profiling: {}, Counter: {}, "
-                 "Interrupt: {}, Device: {})",
-                 HostVisible, WithProfiling, CounterBasedEventEnabled,
-                 InterruptBasedEventEnabled, Device);
+    logger::info(
+        "Cache empty (Host Visible: {}, Profiling: {}, Counter: {}, "
+        "Interrupt: {}, Device: {})",
+        (Flags & EVENT_FLAG_HOST_VISIBLE), (Flags & EVENT_FLAG_WITH_PROFILING),
+        (Flags & EVENT_FLAG_COUNTER), (Flags & EVENT_FLAG_INTERRUPT), Device);
     return nullptr;
   }
 
@@ -624,9 +620,16 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) {
     Device = Event->UrQueue->Device;
   }
 
-  auto Cache = getEventCache(
-      Event->isHostVisible(), Event->isProfilingEnabled(), Device,
-      Event->CounterBasedEventsEnabled, Event->InterruptBasedEventsEnabled);
+  ur_event_flags_t Flags = 0;
+  if (Event->HostVisibleEvent)
+    Flags |= EVENT_FLAG_HOST_VISIBLE;
+  if (Event->isProfilingEnabled())
+    Flags |= EVENT_FLAG_WITH_PROFILING;
+  if (Event->CounterBasedEventsEnabled)
+    Flags |= EVENT_FLAG_COUNTER;
+  if (Event->InterruptBasedEventsEnabled)
+    Flags |= EVENT_FLAG_INTERRUPT;
+  auto Cache = getEventCache(Flags, Device);
   logger::info("Inserting {} event (Host Visible: {}, Profiling: {}, Counter: "
                "{}, Device: {}) into cache {}",
                Event, Event->HostVisibleEvent, Event->isProfilingEnabled(),
@@ -652,11 +655,11 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
   if (!Event->IsMultiDevice && Event->UrQueue) {
     ZeDevice = Event->UrQueue->Device->ZeDevice;
   }
+  if (UsingImmediateCommandlists)
+    Event->Flags |= EVENT_FLAG_IMM_CMDLIST;
 
-  std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
-      Event->isHostVisible(), Event->isProfilingEnabled(),
-      Event->CounterBasedEventsEnabled, UsingImmediateCommandlists,
-      Event->InterruptBasedEventsEnabled, ZeDevice);
+  std::list<ze_event_pool_handle_t> *ZePoolCache =
+      getZeEventPoolCache(Event->Flags, ZeDevice);
 
   // Put the empty pool to the cache of the pools.
   if (NumEventsUnreleasedInEventPool[Event->ZeEventPool] == 0)

source/adapters/level_zero/context.hpp

@@ -213,19 +213,12 @@ struct ur_context_handle_t_ : _ur_object {
   // slot for a host-visible event. The ProfilingEnabled tells is we need a
   // slot for an event with profiling capabilities.
   ur_result_t getFreeSlotInExistingOrNewPool(ze_event_pool_handle_t &, size_t &,
-                                             bool HostVisible,
-                                             bool ProfilingEnabled,
-                                             ur_device_handle_t Device,
-                                             bool CounterBasedEventEnabled,
-                                             bool UsingImmCmdList,
-                                             bool InterruptBasedEventEnabled);
+                                             ur_event_flags_t Flags,
+                                             ur_device_handle_t Device);
 
   // Get ur_event_handle_t from cache.
-  ur_event_handle_t getEventFromContextCache(bool HostVisible,
-                                             bool WithProfiling,
-                                             ur_device_handle_t Device,
-                                             bool CounterBasedEventEnabled,
-                                             bool InterruptBasedEventEnabled);
+  ur_event_handle_t getEventFromContextCache(ur_event_flags_t Flags,
+                                             ur_device_handle_t Device);
 
   // Add ur_event_handle_t to cache.
   void addEventToContextCache(ur_event_handle_t);
@@ -250,15 +243,11 @@ struct ur_context_handle_t_ : _ur_object {
   };
 
   std::list<ze_event_pool_handle_t> *
-  getZeEventPoolCache(bool HostVisible, bool WithProfiling,
-                      bool CounterBasedEventEnabled, bool UsingImmediateCmdList,
-                      bool InterruptBasedEventEnabled,
-                      ze_device_handle_t ZeDevice) {
+  getZeEventPoolCache(ur_event_flags_t Flags, ze_device_handle_t ZeDevice) {
     EventPoolCacheType CacheType;
+    bool WithProfiling = Flags & EVENT_FLAG_WITH_PROFILING;
 
-    calculateCacheIndex(HostVisible, CounterBasedEventEnabled,
-                        UsingImmediateCmdList, InterruptBasedEventEnabled,
-                        CacheType);
+    calculateCacheIndex(Flags, CacheType);
     if (ZeDevice) {
       auto ZeEventPoolCacheMap =
           WithProfiling ? &ZeEventPoolCacheDeviceMap[CacheType * 2]
@@ -275,61 +264,56 @@ struct ur_context_handle_t_ : _ur_object {
     }
   }
 
-  ur_result_t calculateCacheIndex(bool HostVisible,
-                                  bool CounterBasedEventEnabled,
-                                  bool UsingImmediateCmdList,
-                                  bool InterruptBasedEventEnabled,
+  ur_result_t calculateCacheIndex(ur_event_flags_t Flags,
                                   EventPoolCacheType &CacheType) {
+    bool InterruptBasedEventEnabled = Flags & EVENT_FLAG_INTERRUPT;
+    bool CounterBasedEventEnabled = Flags & EVENT_FLAG_COUNTER;
+    bool HostVisible = Flags & EVENT_FLAG_HOST_VISIBLE;
+    bool UsingImmediateCmdList = Flags & EVENT_FLAG_IMM_CMDLIST;
+
     if (InterruptBasedEventEnabled) {
       if (CounterBasedEventEnabled) {
         if (HostVisible) {
-          if (UsingImmediateCmdList) {
-            CacheType = HostVisibleInterruptAndCounterBasedImmediateCacheType;
-          } else {
-            CacheType = HostVisibleInterruptAndCounterBasedRegularCacheType;
-          }
+          CacheType =
+              UsingImmediateCmdList
+                  ? HostVisibleInterruptAndCounterBasedImmediateCacheType
+                  : HostVisibleInterruptAndCounterBasedRegularCacheType;
         } else {
-          if (UsingImmediateCmdList) {
-            CacheType = HostInvisibleInterruptAndCounterBasedImmediateCacheType;
-          } else {
-            CacheType = HostInvisibleInterruptAndCounterBasedRegularCacheType;
-          }
+          CacheType =
+              UsingImmediateCmdList
+                  ? HostInvisibleInterruptAndCounterBasedImmediateCacheType
+                  : HostInvisibleInterruptAndCounterBasedRegularCacheType;
         }
       } else {
         if (HostVisible) {
-          if (UsingImmediateCmdList) {
-            CacheType = HostVisibleInterruptBasedImmediateCacheType;
-          } else {
-            CacheType = HostVisibleInterruptBasedRegularCacheType;
-          }
+          CacheType = UsingImmediateCmdList
+                          ? HostVisibleInterruptBasedImmediateCacheType
+                          : HostVisibleInterruptBasedRegularCacheType;
         } else {
-          if (UsingImmediateCmdList) {
-            CacheType = HostInvisibleInterruptBasedImmediateCacheType;
-          } else {
-            CacheType = HostInvisibleInterruptBasedRegularCacheType;
-          }
+          CacheType = UsingImmediateCmdList
+                          ? HostInvisibleInterruptBasedImmediateCacheType
+                          : HostInvisibleInterruptBasedRegularCacheType;
         }
       }
     } else {
-      if (CounterBasedEventEnabled && HostVisible && !UsingImmediateCmdList) {
-        CacheType = HostVisibleCounterBasedRegularCacheType;
-      } else if (CounterBasedEventEnabled && !HostVisible &&
-                 !UsingImmediateCmdList) {
-        CacheType = HostInvisibleCounterBasedRegularCacheType;
-      } else if (CounterBasedEventEnabled && HostVisible &&
-                 UsingImmediateCmdList) {
-        CacheType = HostVisibleCounterBasedImmediateCacheType;
-      } else if (CounterBasedEventEnabled && !HostVisible &&
-                 UsingImmediateCmdList) {
-        CacheType = HostInvisibleCounterBasedImmediateCacheType;
-      } else if (!CounterBasedEventEnabled && HostVisible) {
-        CacheType = HostVisibleCacheType;
+      if (CounterBasedEventEnabled) {
+        if (HostVisible) {
+          CacheType = UsingImmediateCmdList
+                          ? HostVisibleCounterBasedImmediateCacheType
+                          : HostVisibleCounterBasedRegularCacheType;
+        } else {
+          CacheType = UsingImmediateCmdList
+                          ? HostInvisibleCounterBasedImmediateCacheType
+                          : HostInvisibleCounterBasedRegularCacheType;
+        }
       } else {
-        CacheType = HostInvisibleCacheType;
+        CacheType = HostVisible ? HostVisibleCacheType : HostInvisibleCacheType;
       }
     }
 
     return UR_RESULT_SUCCESS;
+
+    return UR_RESULT_SUCCESS;
   }
 
   // Decrement number of events living in the pool upon event destroy
@@ -370,16 +354,6 @@ struct ur_context_handle_t_ : _ur_object {
   ze_context_handle_t getZeHandle() const;
 
 private:
-  enum EventFlags {
-    EVENT_FLAG_HOST_VISIBLE = UR_BIT(0),
-    EVENT_FLAG_WITH_PROFILING = UR_BIT(1),
-    EVENT_FLAG_COUNTER = UR_BIT(2),
-    EVENT_FLAG_INTERRUPT = UR_BIT(3),
-    EVENT_FLAG_DEVICE = UR_BIT(4), // if set, subsequent bits are device id
-    MAX_EVENT_FLAG_BITS =
-        5, // this is used as an offset for embedding device id
-  };
-
   // Mutex to control operations on event caches.
   ur_mutex EventCacheMutex;
 
@@ -388,26 +362,25 @@ struct ur_context_handle_t_ : _ur_object {
   std::vector<EventCache> EventCaches;
 
   // Get the cache of events for a provided scope and profiling mode.
-  EventCache *getEventCache(bool HostVisible, bool WithProfiling,
-                            ur_device_handle_t Device, bool Counter,
-                            bool Interrupt) {
+  EventCache *getEventCache(ur_event_flags_t Flags, ur_device_handle_t Device) {
 
     size_t index = 0;
-    if (HostVisible) {
-      index |= EVENT_FLAG_HOST_VISIBLE;
-    }
-    if (WithProfiling) {
-      index |= EVENT_FLAG_WITH_PROFILING;
-    }
-    if (Counter) {
-      index |= EVENT_FLAG_COUNTER;
-    }
-    if (Interrupt) {
-      index |= EVENT_FLAG_INTERRUPT;
-    }
+    // if (HostVisible) {
+    // index |= EVENT_FLAG_HOST_VISIBLE;
+    //}
+    // if (WithProfiling) {
+    // index |= EVENT_FLAG_WITH_PROFILING;
+    //}
+    // if (Counter) {
+    // index |= EVENT_FLAG_COUNTER;
+    //}
+    // if (Interrupt) {
+    // index |= EVENT_FLAG_INTERRUPT;
+    //}
     if (Device) {
       index |= EVENT_FLAG_DEVICE | (*Device->Id << MAX_EVENT_FLAG_BITS);
     }
+    index |= Flags;
 
     if (index >= EventCaches.size()) {
       EventCaches.resize(index + 1);

source/adapters/level_zero/event.cpp

@@ -1308,16 +1308,27 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   bool ProfilingEnabled =
       ForceDisableProfiling ? false : (!Queue || Queue->isProfilingEnabled());
   bool UsingImmediateCommandlists = !Queue || Queue->UsingImmCmdLists;
+  ur_event_flags_t Flags = 0;
+  if (ProfilingEnabled)
+    Flags |= EVENT_FLAG_WITH_PROFILING;
+  if (UsingImmediateCommandlists)
+    Flags |= EVENT_FLAG_IMM_CMDLIST;
+  if (HostVisible)
+    Flags |= EVENT_FLAG_HOST_VISIBLE;
+  if (IsMultiDevice)
+    Flags |= EVENT_FLAG_MULTIDEVICE;
+  if (CounterBasedEventEnabled)
+    Flags |= EVENT_FLAG_COUNTER;
+  if (InterruptBasedEventEnabled)
+    Flags |= EVENT_FLAG_INTERRUPT;
 
   ur_device_handle_t Device = nullptr;
 
   if (!IsMultiDevice && Queue) {
     Device = Queue->Device;
   }
 
-  if (auto CachedEvent = Context->getEventFromContextCache(
-          HostVisible, ProfilingEnabled, Device, CounterBasedEventEnabled,
-          InterruptBasedEventEnabled)) {
+  if (auto CachedEvent = Context->getEventFromContextCache(Flags, Device)) {
     *RetEvent = CachedEvent;
     return UR_RESULT_SUCCESS;
   }
@@ -1327,10 +1338,8 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
 
   size_t Index = 0;
 
-  if (auto Res = Context->getFreeSlotInExistingOrNewPool(
-          ZeEventPool, Index, HostVisible, ProfilingEnabled, Device,
-          CounterBasedEventEnabled, UsingImmediateCommandlists,
-          InterruptBasedEventEnabled))
+  if (auto Res = Context->getFreeSlotInExistingOrNewPool(ZeEventPool, Index,
+                                                         Flags, Device))
     return Res;
 
   ZeStruct<ze_event_desc_t> ZeEventDesc;
@@ -1367,6 +1376,7 @@ ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
   if (HostVisible)
     (*RetEvent)->HostVisibleEvent =
         reinterpret_cast<ur_event_handle_t>(*RetEvent);
+  (*RetEvent)->Flags = Flags;
 
   return UR_RESULT_SUCCESS;
 }

source/adapters/level_zero/event.hpp

@@ -139,6 +139,8 @@ struct ur_event_handle_t_ : _ur_object {
   // Level Zero event pool handle.
   ze_event_pool_handle_t ZeEventPool;
 
+  ur_event_flags_t Flags;
+
   // In case we use device-only events this holds their host-visible
   // counterpart. If this event is itself host-visble then HostVisibleEvent
   // points to this event. If this event is not host-visible then this field can