From cbef9495e2b0ed0acb242c04b7f125ef3079b256 Mon Sep 17 00:00:00 2001 From: "aidan.belton" Date: Tue, 14 Sep 2021 11:19:12 +0100 Subject: [PATCH 1/5] ignore usm_prefetch dummy flag --- sycl/plugins/cuda/pi_cuda.cpp | 4 ---- sycl/plugins/hip/pi_hip.cpp | 4 ---- sycl/plugins/level_zero/pi_level_zero.cpp | 1 - sycl/source/detail/memory_manager.cpp | 2 +- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/sycl/plugins/cuda/pi_cuda.cpp b/sycl/plugins/cuda/pi_cuda.cpp index 528c10c708569..93f913769bc8d 100644 --- a/sycl/plugins/cuda/pi_cuda.cpp +++ b/sycl/plugins/cuda/pi_cuda.cpp @@ -4601,10 +4601,6 @@ pi_result cuda_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, pi_result result = PI_SUCCESS; std::unique_ptr<_pi_event> event_ptr{nullptr}; - // TODO implement handling the flags once the expected behaviour - // of piextUSMEnqueuePrefetch is detailed in the USM extension - assert(flags == 0u); - try { ScopedContext active(queue->get_context()); result = cuda_piEnqueueEventsWait(queue, num_events_in_waitlist, diff --git a/sycl/plugins/hip/pi_hip.cpp b/sycl/plugins/hip/pi_hip.cpp index d5019a234d30c..b6ff9602ab540 100644 --- a/sycl/plugins/hip/pi_hip.cpp +++ b/sycl/plugins/hip/pi_hip.cpp @@ -4438,10 +4438,6 @@ pi_result hip_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, pi_result result = PI_SUCCESS; std::unique_ptr<_pi_event> event_ptr{nullptr}; - // TODO implement handling the flags once the expected behaviour - // of piextUSMEnqueuePrefetch is detailed in the USM extension - assert(flags == 0u); - try { ScopedContext active(queue->get_context()); result = hip_piEnqueueEventsWait(queue, num_events_in_waitlist, diff --git a/sycl/plugins/level_zero/pi_level_zero.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp index 054e7306aec9e..e668447758721 100644 --- a/sycl/plugins/level_zero/pi_level_zero.cpp +++ b/sycl/plugins/level_zero/pi_level_zero.cpp @@ -6633,7 +6633,6 @@ pi_result piextUSMEnqueuePrefetch(pi_queue Queue, const void *Ptr, size_t Size, pi_uint32 NumEventsInWaitList, const pi_event *EventWaitList, pi_event *Event) { - PI_ASSERT(!(Flags & ~PI_USM_MIGRATION_TBD0), PI_INVALID_VALUE); PI_ASSERT(Queue, PI_INVALID_QUEUE); PI_ASSERT(Event, PI_INVALID_EVENT); diff --git a/sycl/source/detail/memory_manager.cpp b/sycl/source/detail/memory_manager.cpp index d8d4a05c4b75c..55fb5ec681b46 100644 --- a/sycl/source/detail/memory_manager.cpp +++ b/sycl/source/detail/memory_manager.cpp @@ -698,7 +698,7 @@ void MemoryManager::prefetch_usm(void *Mem, QueueImplPtr Queue, size_t Length, } else { const detail::plugin &Plugin = Queue->getPlugin(); Plugin.call( - Queue->getHandleRef(), Mem, Length, PI_USM_MIGRATION_TBD0, + Queue->getHandleRef(), Mem, Length, _pi_usm_migration_flags(0), DepEvents.size(), DepEvents.data(), &OutEvent); } } From de2a662b6b830d16382e2cf9c5c44f3b5c31af8d Mon Sep 17 00:00:00 2001 From: "aidan.belton" Date: Tue, 14 Sep 2021 15:42:00 +0100 Subject: [PATCH 2/5] Assert for unknown flag --- sycl/plugins/cuda/pi_cuda.cpp | 4 +++- sycl/plugins/hip/pi_hip.cpp | 2 ++ sycl/plugins/level_zero/pi_level_zero.cpp | 2 ++ sycl/plugins/opencl/pi_opencl.cpp | 4 +++- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sycl/plugins/cuda/pi_cuda.cpp b/sycl/plugins/cuda/pi_cuda.cpp index 93f913769bc8d..90f79f659f42e 100644 --- a/sycl/plugins/cuda/pi_cuda.cpp +++ b/sycl/plugins/cuda/pi_cuda.cpp @@ -4589,13 +4589,15 @@ pi_result cuda_piextUSMEnqueueMemcpy(pi_queue queue, pi_bool blocking, return result; } +// flags is currently ignored as it is a placeholder for future features pi_result cuda_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_usm_migration_flags flags, pi_uint32 num_events_in_waitlist, const pi_event *events_waitlist, pi_event *event) { - assert(queue != nullptr); + assert(!(flags & ~PI_USM_MIGRATION_TBD0), PI_INVALID_VALUE) + assert(queue != nullptr); assert(ptr != nullptr); CUstream cuStream = queue->get(); pi_result result = PI_SUCCESS; diff --git a/sycl/plugins/hip/pi_hip.cpp b/sycl/plugins/hip/pi_hip.cpp index b6ff9602ab540..e358e3b43e125 100644 --- a/sycl/plugins/hip/pi_hip.cpp +++ b/sycl/plugins/hip/pi_hip.cpp @@ -4426,12 +4426,14 @@ pi_result hip_piextUSMEnqueueMemcpy(pi_queue queue, pi_bool blocking, return result; } +// flags is currently ignored as it is a placeholder for future features pi_result hip_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_usm_migration_flags flags, pi_uint32 num_events_in_waitlist, const pi_event *events_waitlist, pi_event *event) { + assert(!(flags & ~PI_USM_MIGRATION_TBD0)); assert(queue != nullptr); assert(ptr != nullptr); hipStream_t hipStream = queue->get(); diff --git a/sycl/plugins/level_zero/pi_level_zero.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp index e668447758721..061345988c5fa 100644 --- a/sycl/plugins/level_zero/pi_level_zero.cpp +++ b/sycl/plugins/level_zero/pi_level_zero.cpp @@ -6628,11 +6628,13 @@ pi_result piextUSMEnqueueMemcpy(pi_queue Queue, pi_bool Blocking, void *DstPtr, /// @param NumEventsInWaitlist is the number of events to wait on /// @param EventsWaitlist is an array of events to wait on /// @param Event is the event that represents this operation +// flags is currently ignored as it is a placeholder for future features pi_result piextUSMEnqueuePrefetch(pi_queue Queue, const void *Ptr, size_t Size, pi_usm_migration_flags Flags, pi_uint32 NumEventsInWaitList, const pi_event *EventWaitList, pi_event *Event) { + PI_ASSERT(!(Flags & ~PI_USM_MIGRATION_TBD0), PI_INVALID_VALUE); PI_ASSERT(Queue, PI_INVALID_QUEUE); PI_ASSERT(Event, PI_INVALID_EVENT); diff --git a/sycl/plugins/opencl/pi_opencl.cpp b/sycl/plugins/opencl/pi_opencl.cpp index fb05ceca94b32..3072870896ff5 100644 --- a/sycl/plugins/opencl/pi_opencl.cpp +++ b/sycl/plugins/opencl/pi_opencl.cpp @@ -1005,6 +1005,7 @@ pi_result piextUSMEnqueueMemcpy(pi_queue queue, pi_bool blocking, void *dst_ptr, /// \param num_events_in_waitlist is the number of events to wait on /// \param events_waitlist is an array of events to wait on /// \param event is the event that represents this operation +// flags is currently ignored as it is a placeholder for future features pi_result piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_usm_migration_flags flags, pi_uint32 num_events_in_waitlist, @@ -1012,7 +1013,8 @@ pi_result piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_event *event) { (void)ptr; (void)size; - (void)flags; + + assert(!(flags & ~PI_USM_MIGRATION_TBD0)); return cast(clEnqueueMarkerWithWaitList( cast(queue), num_events_in_waitlist, From c33a3d05d3e1d1bb283462af1228dc2eff06fa7e Mon Sep 17 00:00:00 2001 From: "aidan.belton" Date: Tue, 14 Sep 2021 15:59:51 +0100 Subject: [PATCH 3/5] Fix check for unknown flag --- sycl/plugins/cuda/pi_cuda.cpp | 6 ++++-- sycl/plugins/hip/pi_hip.cpp | 3 ++- sycl/plugins/opencl/pi_opencl.cpp | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sycl/plugins/cuda/pi_cuda.cpp b/sycl/plugins/cuda/pi_cuda.cpp index 90f79f659f42e..559d89c065f64 100644 --- a/sycl/plugins/cuda/pi_cuda.cpp +++ b/sycl/plugins/cuda/pi_cuda.cpp @@ -4596,8 +4596,10 @@ pi_result cuda_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, pi_uint32 num_events_in_waitlist, const pi_event *events_waitlist, pi_event *event) { - assert(!(flags & ~PI_USM_MIGRATION_TBD0), PI_INVALID_VALUE) - assert(queue != nullptr); + + if (!(flags & ~PI_USM_MIGRATION_TBD0)) + return PI_INVALID_VALUE; + assert(queue != nullptr); assert(ptr != nullptr); CUstream cuStream = queue->get(); pi_result result = PI_SUCCESS; diff --git a/sycl/plugins/hip/pi_hip.cpp b/sycl/plugins/hip/pi_hip.cpp index e358e3b43e125..7c0df70bf049e 100644 --- a/sycl/plugins/hip/pi_hip.cpp +++ b/sycl/plugins/hip/pi_hip.cpp @@ -4433,7 +4433,8 @@ pi_result hip_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, const pi_event *events_waitlist, pi_event *event) { - assert(!(flags & ~PI_USM_MIGRATION_TBD0)); + if (!(flags & ~PI_USM_MIGRATION_TBD0)) + return PI_INVALID_VALUE; assert(queue != nullptr); assert(ptr != nullptr); hipStream_t hipStream = queue->get(); diff --git a/sycl/plugins/opencl/pi_opencl.cpp b/sycl/plugins/opencl/pi_opencl.cpp index 3072870896ff5..a61cc445ce7db 100644 --- a/sycl/plugins/opencl/pi_opencl.cpp +++ b/sycl/plugins/opencl/pi_opencl.cpp @@ -1014,7 +1014,8 @@ pi_result piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, (void)ptr; (void)size; - assert(!(flags & ~PI_USM_MIGRATION_TBD0)); + if (!(flags & ~PI_USM_MIGRATION_TBD0)) + return PI_INVALID_VALUE; return cast(clEnqueueMarkerWithWaitList( cast(queue), num_events_in_waitlist, From 258d15f0a441592e9c4b6fd1923e86d984af8fc2 Mon Sep 17 00:00:00 2001 From: "aidan.belton" Date: Tue, 14 Sep 2021 17:16:21 +0100 Subject: [PATCH 4/5] Update based on feedback --- sycl/plugins/cuda/pi_cuda.cpp | 4 ++-- sycl/plugins/hip/pi_hip.cpp | 4 ++-- sycl/plugins/level_zero/pi_level_zero.cpp | 5 +++-- sycl/plugins/opencl/pi_opencl.cpp | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sycl/plugins/cuda/pi_cuda.cpp b/sycl/plugins/cuda/pi_cuda.cpp index 559d89c065f64..601f7bf8ce8e5 100644 --- a/sycl/plugins/cuda/pi_cuda.cpp +++ b/sycl/plugins/cuda/pi_cuda.cpp @@ -4589,7 +4589,6 @@ pi_result cuda_piextUSMEnqueueMemcpy(pi_queue queue, pi_bool blocking, return result; } -// flags is currently ignored as it is a placeholder for future features pi_result cuda_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_usm_migration_flags flags, @@ -4597,7 +4596,8 @@ pi_result cuda_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, const pi_event *events_waitlist, pi_event *event) { - if (!(flags & ~PI_USM_MIGRATION_TBD0)) + // flags is currently unused so fail if set + if (flags != 0) return PI_INVALID_VALUE; assert(queue != nullptr); assert(ptr != nullptr); diff --git a/sycl/plugins/hip/pi_hip.cpp b/sycl/plugins/hip/pi_hip.cpp index 7c0df70bf049e..1d14ff64b9241 100644 --- a/sycl/plugins/hip/pi_hip.cpp +++ b/sycl/plugins/hip/pi_hip.cpp @@ -4426,14 +4426,14 @@ pi_result hip_piextUSMEnqueueMemcpy(pi_queue queue, pi_bool blocking, return result; } -// flags is currently ignored as it is a placeholder for future features pi_result hip_piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_usm_migration_flags flags, pi_uint32 num_events_in_waitlist, const pi_event *events_waitlist, pi_event *event) { - if (!(flags & ~PI_USM_MIGRATION_TBD0)) + // flags is currently unused so fail if set + if (flags != 0) return PI_INVALID_VALUE; assert(queue != nullptr); assert(ptr != nullptr); diff --git a/sycl/plugins/level_zero/pi_level_zero.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp index 061345988c5fa..027ccdf09bb29 100644 --- a/sycl/plugins/level_zero/pi_level_zero.cpp +++ b/sycl/plugins/level_zero/pi_level_zero.cpp @@ -6628,13 +6628,14 @@ pi_result piextUSMEnqueueMemcpy(pi_queue Queue, pi_bool Blocking, void *DstPtr, /// @param NumEventsInWaitlist is the number of events to wait on /// @param EventsWaitlist is an array of events to wait on /// @param Event is the event that represents this operation -// flags is currently ignored as it is a placeholder for future features pi_result piextUSMEnqueuePrefetch(pi_queue Queue, const void *Ptr, size_t Size, pi_usm_migration_flags Flags, pi_uint32 NumEventsInWaitList, const pi_event *EventWaitList, pi_event *Event) { - PI_ASSERT(!(Flags & ~PI_USM_MIGRATION_TBD0), PI_INVALID_VALUE); + + // flags is currently unused so fail if set + PI_ASSERT(Flags == 0, PI_INVALID_VALUE); PI_ASSERT(Queue, PI_INVALID_QUEUE); PI_ASSERT(Event, PI_INVALID_EVENT); diff --git a/sycl/plugins/opencl/pi_opencl.cpp b/sycl/plugins/opencl/pi_opencl.cpp index a61cc445ce7db..59c62716b1845 100644 --- a/sycl/plugins/opencl/pi_opencl.cpp +++ b/sycl/plugins/opencl/pi_opencl.cpp @@ -1005,7 +1005,6 @@ pi_result piextUSMEnqueueMemcpy(pi_queue queue, pi_bool blocking, void *dst_ptr, /// \param num_events_in_waitlist is the number of events to wait on /// \param events_waitlist is an array of events to wait on /// \param event is the event that represents this operation -// flags is currently ignored as it is a placeholder for future features pi_result piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, pi_usm_migration_flags flags, pi_uint32 num_events_in_waitlist, @@ -1014,7 +1013,8 @@ pi_result piextUSMEnqueuePrefetch(pi_queue queue, const void *ptr, size_t size, (void)ptr; (void)size; - if (!(flags & ~PI_USM_MIGRATION_TBD0)) + // flags is currently unused so fail if set + if (flags != 0) return PI_INVALID_VALUE; return cast(clEnqueueMarkerWithWaitList( From 92354777b1e30bed82c224e8dd2d8e86f024fb48 Mon Sep 17 00:00:00 2001 From: "aidan.belton" Date: Wed, 15 Sep 2021 10:08:33 +0100 Subject: [PATCH 5/5] Comment PI_USM_MIGRATION_TBD0 --- sycl/include/CL/sycl/detail/pi.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sycl/include/CL/sycl/detail/pi.h b/sycl/include/CL/sycl/detail/pi.h index 3ab941823e603..9d0669eb6494d 100644 --- a/sycl/include/CL/sycl/detail/pi.h +++ b/sycl/include/CL/sycl/detail/pi.h @@ -1560,6 +1560,9 @@ typedef enum : pi_bitfield { PI_MEM_ALLOC_FLAGS = CL_MEM_ALLOC_FLAGS_INTEL } _pi_usm_mem_properties; +// Flag is used for piProgramUSMEnqueuePrefetch. PI_USM_MIGRATION_TBD0 is a +// placeholder for future developments and should not change the behaviour of +// piProgramUSMEnqueuePrefetch typedef enum : pi_bitfield { PI_USM_MIGRATION_TBD0 = (1 << 0) } _pi_usm_migration_flags;