Skip to content

[SYCL] Specialize atomic fetch_add for floating point types #2765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jan 21, 2021
Merged

[SYCL] Specialize atomic fetch_add for floating point types #2765

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
7dce90a
[SYCL] Specialize atomic fetch_add for floating point types
Nov 11, 2020
28a3a1b
Provide a general macro for float atomics' emulation
Dec 2, 2020
d6a5208
[SYCL][Test] Configure atomic float tests for CUDA/CPU
Dec 9, 2020
3beb65a
Address review comments: fetch_sub in atomic.hpp, RUN lines' cleanup
Jan 13, 2021
a9d8d88
Fix the negation subject in the fetch_sub implementation
Jan 13, 2021
1e2ff47
Remove the "emulated" atomics usage for the HOST tests
Jan 14, 2021
73b5b5d
Mark the tests unsupported on CUDA
Jan 14, 2021
da6edc5
Edit the UNSUPPORTED placement & content
Jan 14, 2021
212aa4d
Update tests to check the IR for spinlock emulation behavior
Jan 15, 2021
0b178bc
Switch to using the emulation by default
Jan 21, 2021
342a179
Revert changes in atomic.hpp - the new implementation can't be used t…
Jan 21, 2021
be71125
Remove double underscore from the macro name
Jan 21, 2021
906c413
Switch to XFAIL for CUDA
Jan 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions sycl/include/CL/__spirv/spirv_ops.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ extern SYCL_EXTERNAL TempRetT __spirv_ImageSampleExplicitLod(SampledType,
extern SYCL_EXTERNAL Type __spirv_AtomicISub( \
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
Type V);
#define __SPIRV_ATOMIC_FADD(AS, Type) \
extern SYCL_EXTERNAL Type __spirv_AtomicFAddEXT( \
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
Type V);
#define __SPIRV_ATOMIC_SMIN(AS, Type) \
extern SYCL_EXTERNAL Type __spirv_AtomicSMin( \
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
Expand Down Expand Up @@ -109,6 +113,7 @@ extern SYCL_EXTERNAL TempRetT __spirv_ImageSampleExplicitLod(SampledType,
Type V);

#define __SPIRV_ATOMIC_FLOAT(AS, Type) \
__SPIRV_ATOMIC_FADD(AS, Type) \
__SPIRV_ATOMIC_LOAD(AS, Type) \
__SPIRV_ATOMIC_STORE(AS, Type) \
__SPIRV_ATOMIC_EXCHANGE(AS, Type)
Expand Down
12 changes: 12 additions & 0 deletions sycl/include/CL/sycl/ONEAPI/atomic_ref.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,11 @@ class atomic_ref_impl<

T fetch_add(T operand, memory_order order = default_read_modify_write_order,
memory_scope scope = default_scope) const noexcept {
// TODO: Remove the "native atomics" macro check once implemented for all
// backends
#if defined(__SYCL_DEVICE_ONLY__) && defined(SYCL_USE_NATIVE_FP_ATOMICS)
return detail::spirv::AtomicFAdd(ptr, scope, order, operand);
#else
auto load_order = detail::getLoadOrder(order);
T expected;
T desired;
Expand All @@ -462,6 +467,7 @@ class atomic_ref_impl<
desired = expected + operand;
} while (!compare_exchange_weak(expected, desired, order, scope));
return expected;
#endif
}

T operator+=(T operand) const noexcept {
Expand All @@ -470,13 +476,19 @@ class atomic_ref_impl<

T fetch_sub(T operand, memory_order order = default_read_modify_write_order,
memory_scope scope = default_scope) const noexcept {
// TODO: Remove the "native atomics" macro check once implemented for all
// backends
#if defined(__SYCL_DEVICE_ONLY__) && defined(SYCL_USE_NATIVE_FP_ATOMICS)
return detail::spirv::AtomicFAdd(ptr, scope, order, -operand);
#else
auto load_order = detail::getLoadOrder(order);
T expected = load(load_order, scope);
T desired;
do {
desired = expected - operand;
} while (!compare_exchange_weak(expected, desired, order, scope));
return expected;
#endif
}

T operator-=(T operand) const noexcept {
Expand Down
10 changes: 10 additions & 0 deletions sycl/include/CL/sycl/detail/spirv.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,16 @@ AtomicISub(multi_ptr<T, AddressSpace> MPtr, ONEAPI::memory_scope Scope,
return __spirv_AtomicISub(Ptr, SPIRVScope, SPIRVOrder, Value);
}

template <typename T, access::address_space AddressSpace>
inline typename detail::enable_if_t<std::is_floating_point<T>::value, T>
AtomicFAdd(multi_ptr<T, AddressSpace> MPtr, ONEAPI::memory_scope Scope,
ONEAPI::memory_order Order, T Value) {
auto *Ptr = MPtr.get();
auto SPIRVOrder = getMemorySemanticsMask(Order);
auto SPIRVScope = getScope(Scope);
return __spirv_AtomicFAddEXT(Ptr, SPIRVScope, SPIRVOrder, Value);
}

template <typename T, access::address_space AddressSpace>
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
AtomicAnd(multi_ptr<T, AddressSpace> MPtr, ONEAPI::memory_scope Scope,
Expand Down
41 changes: 27 additions & 14 deletions sycl/test/atomic_ref/add.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
// TODO: Once NVPTX accepts the __spirv_AtomicF*() IR, remove the XFAIL mark
// XFAIL: cuda

// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -DSYCL_USE_NATIVE_FP_ATOMICS \
// RUN: -fsycl-device-only -S %s -o - | FileCheck %s --check-prefix=CHECK-LLVM
// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -fsycl-device-only -S %s -o - \
// RUN: | FileCheck %s --check-prefix=CHECK-LLVM
// RUN: | FileCheck %s --check-prefix=CHECK-LLVM-EMU
// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -fsycl-targets=%sycl_triple %s -o %t.out
// RUN: %RUN_ON_HOST %t.out

Expand Down Expand Up @@ -167,22 +172,24 @@ void add_test(queue q, size_t N) {
// Floating-point types do not support pre- or post-increment
template <> void add_test<float>(queue q, size_t N) {
add_fetch_test<float>(q, N);
// CHECK-LLVM: declare dso_local spir_func i32
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicLoad
// CHECK-LLVM-SAME: (i32 addrspace(1)*, i32, i32)
// CHECK-LLVM: declare dso_local spir_func i32
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicCompareExchange
// CHECK-LLVM-SAME: (i32 addrspace(1)*, i32, i32, i32, i32, i32)
// CHECK-LLVM: declare dso_local spir_func float
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicFAddEXT
// CHECK-LLVM-SAME: (float addrspace(1)*, i32, i32, float)
// CHECK-LLVM-EMU: declare {{.*}} i32 @{{.*}}__spirv_AtomicLoad
// CHECK-LLVM-EMU-SAME: (i32 addrspace(1)*, i32, i32)
// CHECK-LLVM-EMU: declare {{.*}} i32 @{{.*}}__spirv_AtomicCompareExchange
// CHECK-LLVM-EMU-SAME: (i32 addrspace(1)*, i32, i32, i32, i32, i32)
add_plus_equal_test<float>(q, N);
}
template <> void add_test<double>(queue q, size_t N) {
add_fetch_test<double>(q, N);
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicLoad
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32)
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicCompareExchange
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32, i32, i64, i64)
// CHECK-LLVM: declare dso_local spir_func double
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicFAddEXT
// CHECK-LLVM-SAME: double addrspace(1)*, i32, i32, double)
// CHECK-LLVM-EMU: declare {{.*}} i64 @{{.*}}__spirv_AtomicLoad
// CHECK-LLVM-EMU-SAME: (i64 addrspace(1)*, i32, i32)
// CHECK-LLVM-EMU: declare {{.*}} i64 @{{.*}}__spirv_AtomicCompareExchange
// CHECK-LLVM-EMU-SAME: (i64 addrspace(1)*, i32, i32, i32, i64, i64)
add_plus_equal_test<double>(q, N);
}

Expand Down Expand Up @@ -219,9 +226,15 @@ int main() {
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicIAdd
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32, i64)
add_test<unsigned long long>(q, N);
// The remaining functions have been instantiated earlier
// Floating point-typed functions have been instantiated earlier
add_test<float>(q, N);
add_test<double>(q, N);
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicLoad
// CHECK-LLVM-SAME: i64 addrspace(1)*, i32, i32)
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicCompareExchange
// CHECK-LLVM-SAME: i64 addrspace(1)*, i32, i32, i32, i64, i64)
add_test<char *, ptrdiff_t>(q, N);

std::cout << "Test passed." << std::endl;
Expand Down
41 changes: 28 additions & 13 deletions sycl/test/atomic_ref/sub.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
// TODO: Once NVPTX accepts the __spirv_AtomicF*() IR, remove the XFAIL mark
// XFAIL: cuda

// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -DSYCL_USE_NATIVE_FP_ATOMICS \
// RUN: -fsycl-device-only -S %s -o - | FileCheck %s --check-prefix=CHECK-LLVM
// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -fsycl-device-only -S %s -o - \
// RUN: | FileCheck %s --check-prefix=CHECK-LLVM-EMU
// RUN: %clangxx -fsycl -fsycl-unnamed-lambda -fsycl-targets=%sycl_triple %s -o %t.out
// RUN: %RUN_ON_HOST %t.out

Expand Down Expand Up @@ -165,22 +172,24 @@ void sub_test(queue q, size_t N) {
// Floating-point types do not support pre- or post-decrement
template <> void sub_test<float>(queue q, size_t N) {
sub_fetch_test<float>(q, N);
// CHECK-LLVM: declare dso_local spir_func i32
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicLoad
// CHECK-LLVM-SAME: (i32 addrspace(1)*, i32, i32)
// CHECK-LLVM: declare dso_local spir_func i32
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicCompareExchange
// CHECK-LLVM-SAME: (i32 addrspace(1)*, i32, i32, i32, i32, i32)
// CHECK-LLVM: declare dso_local spir_func float
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicFAddEXT
// CHECK-LLVM-SAME: (float addrspace(1)*, i32, i32, float)
// CHECK-LLVM-EMU: declare {{.*}} i32 @{{.*}}__spirv_AtomicLoad
// CHECK-LLVM-EMU-SAME: (i32 addrspace(1)*, i32, i32)
// CHECK-LLVM-EMU: declare {{.*}} i32 @{{.*}}__spirv_AtomicCompareExchange
// CHECK-LLVM-EMU-SAME: (i32 addrspace(1)*, i32, i32, i32, i32, i32)
sub_plus_equal_test<float>(q, N);
}
template <> void sub_test<double>(queue q, size_t N) {
sub_fetch_test<double>(q, N);
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicLoad
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32)
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicCompareExchange
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32, i32, i64, i64)
// CHECK-LLVM: declare dso_local spir_func double
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicFAddEXT
// CHECK-LLVM-SAME: (double addrspace(1)*, i32, i32, double)
// CHECK-LLVM-EMU: declare {{.*}} i64 @{{.*}}__spirv_AtomicLoad
// CHECK-LLVM-EMU-SAME: (i64 addrspace(1)*, i32, i32)
// CHECK-LLVM-EMU: declare {{.*}} i64 @{{.*}}__spirv_AtomicCompareExchange
// CHECK-LLVM-EMU-SAME: (i64 addrspace(1)*, i32, i32, i32, i64, i64)
sub_plus_equal_test<double>(q, N);
}

Expand Down Expand Up @@ -217,9 +226,15 @@ int main() {
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicISub
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32, i64)
sub_test<unsigned long long>(q, N);
// The remaining functions have been instantiated earlier
// Floating point-typed functions have been instantiated earlier
sub_test<float>(q, N);
sub_test<double>(q, N);
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicLoad
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32)
// CHECK-LLVM: declare dso_local spir_func i64
// CHECK-LLVM-SAME: @_Z{{[0-9]+}}__spirv_AtomicCompareExchange
// CHECK-LLVM-SAME: (i64 addrspace(1)*, i32, i32, i32, i64, i64)
sub_test<char *, ptrdiff_t>(q, N);

std::cout << "Test passed." << std::endl;
Expand Down