Skip to content

Commit 9646680

Browse files
RossBruntonRossBrunton
committed
Replace loader handles with field at start of handle data
This replaces the handle logic in the loader from wrapped pointers to a ddi table at the start of the handle struct itself.
1 parent aea4883 commit 9646680

31 files changed

+941
-3718
lines changed

‎scripts/templates/helper.py

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1295,17 +1295,6 @@ def get_initial_null_set(obj):
12951295
return 'if (nullptr != {0}) {{*{0} = nullptr;}}'.format(lvalue)
12961296
return ""
12971297

1298-
"""
1299-
Public:
1300-
returns true if the function always wraps output pointers in loader handles
1301-
"""
1302-
def always_wrap_outputs(obj):
1303-
cname = obj_traits.class_name(obj)
1304-
return (cname, obj['name']) in [
1305-
('$xProgram', 'Link'),
1306-
('$xProgram', 'LinkExp'),
1307-
]
1308-
13091298
"""
13101299
Private:
13111300
returns the list of parameters, filtering based on desc tags

‎scripts/templates/ldrddi.cpp.mako

Lines changed: 19 additions & 228 deletions
Large diffs are not rendered by default.

‎scripts/templates/ur_interface_loader.cpp.mako

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ from templates import helper as th
2020
//===----------------------------------------------------------------------===//
2121
#include <${n}_api.h>
2222
#include <${n}_ddi.h>
23+
#include <mutex>
2324

2425
#include "ur_interface_loader.hpp"
2526

@@ -68,22 +69,45 @@ ${X}_APIEXPORT ${x}_result_t ${X}_APICALL ${tbl['export']['name']}(
6869
} // extern "C"
6970
#endif
7071

71-
#ifdef UR_STATIC_ADAPTER_${Adapter}
72-
namespace ur::${adapter} {
73-
ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi) {
74-
if (ddi == nullptr) {
72+
namespace {
73+
ur_result_t populateDdiTable(ur_dditable_t *ddi) {
74+
if (ddi == nullptr) {
7575
return UR_RESULT_ERROR_INVALID_NULL_POINTER;
7676
}
7777

7878
ur_result_t result;
7979

80+
#ifdef UR_STATIC_ADAPTER_${Adapter}
81+
#define NAMESPACE_ ::ur::${adapter}
82+
#else
83+
#define NAMESPACE_
84+
#endif
85+
8086
%for tbl in th.get_pfntables(specs, meta, n, tags):
81-
result = ${n}::${adapter}::${tbl['export']['name']}( ${X}_API_VERSION_CURRENT, &ddi->${tbl['name']} );
87+
result = NAMESPACE_::${tbl['export']['name']}( ${X}_API_VERSION_CURRENT, &ddi->${tbl['name']} );
8288
if (result != UR_RESULT_SUCCESS)
8389
return result;
8490
%endfor
8591

92+
#undef NAMESPACE_
93+
8694
return result;
8795
}
8896
}
97+
98+
99+
namespace ur::${adapter} {
100+
${x}_dditable_t *ddi_getter::value() {
101+
static std::once_flag flag;
102+
static ${x}_dditable_t table;
103+
104+
std::call_once(flag, []() { populateDdiTable(&table); });
105+
return &table;
106+
}
107+
108+
#ifdef UR_STATIC_ADAPTER_${Adapter}
109+
ur_result_t urAdapterGetDdiTables(${x}_dditable_t *ddi) {
110+
return populateDdiTable(ddi);
111+
}
89112
#endif
113+
}

‎scripts/templates/ur_interface_loader.hpp.mako

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ from templates import helper as th
1818
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
1919
//
2020
//===----------------------------------------------------------------------===//
21+
#pragma once
22+
2123
#include <${n}_api.h>
2224
#include <${n}_ddi.h>
2325

@@ -36,4 +38,8 @@ ${x}_result_t ${th.make_func_name(n, tags, obj)}(
3638
#ifdef UR_STATIC_ADAPTER_LEVEL_ZERO
3739
ur_result_t urAdapterGetDdiTables(ur_dditable_t *ddi);
3840
#endif
41+
42+
struct ddi_getter {
43+
static ${x}_dditable_t *value();
44+
};
3945
}

‎source/adapters/hip/adapter.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
#include <atomic>
1616
#include <ur_api.h>
1717

18-
struct ur_adapter_handle_t_ {
18+
struct ur_adapter_handle_t_ : ur_object_t_ {
1919
std::atomic<uint32_t> RefCount = 0;
2020
logger::Logger &logger;
2121
ur_adapter_handle_t_();
@@ -40,8 +40,8 @@ class ur_legacy_sink : public logger::Sink {
4040
// through UR entry points.
4141
// https://github.com/oneapi-src/unified-runtime/issues/1330
4242
ur_adapter_handle_t_::ur_adapter_handle_t_()
43-
: logger(
44-
logger::get_logger("hip", /*default_log_level*/ logger::Level::ERR)) {
43+
: ur_object_t_(), logger(logger::get_logger(
44+
"hip", /*default_log_level*/ logger::Level::ERR)) {
4545

4646
if (std::getenv("UR_LOG_HIP") != nullptr)
4747
return;

‎source/adapters/hip/command_buffer.cpp

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ commandHandleReleaseInternal(ur_exp_command_buffer_command_handle_t Command) {
4848

4949
ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_(
5050
ur_context_handle_t hContext, ur_device_handle_t hDevice, bool IsUpdatable)
51-
: Context(hContext), Device(hDevice), IsUpdatable(IsUpdatable),
52-
HIPGraph{nullptr}, HIPGraphExec{nullptr}, RefCountInternal{1},
53-
RefCountExternal{1}, NextSyncPoint{0} {
51+
: ur_object_t_(), Context(hContext), Device(hDevice),
52+
IsUpdatable(IsUpdatable), HIPGraph{nullptr}, HIPGraphExec{nullptr},
53+
RefCountInternal{1}, RefCountExternal{1}, NextSyncPoint{0} {
5454
urContextRetain(hContext);
5555
urDeviceRetain(hDevice);
5656
}
@@ -80,8 +80,9 @@ ur_exp_command_buffer_command_handle_t_::
8080
const size_t *GlobalWorkOffsetPtr, const size_t *GlobalWorkSizePtr,
8181
const size_t *LocalWorkSizePtr, uint32_t NumKernelAlternatives,
8282
ur_kernel_handle_t *KernelAlternatives)
83-
: CommandBuffer(CommandBuffer), Kernel(Kernel), Node(Node), Params(Params),
84-
WorkDim(WorkDim), RefCountInternal(1), RefCountExternal(1) {
83+
: ur_object_t_(), CommandBuffer(CommandBuffer), Kernel(Kernel), Node(Node),
84+
Params(Params), WorkDim(WorkDim), RefCountInternal(1),
85+
RefCountExternal(1) {
8586
CommandBuffer->incrementInternalReferenceCount();
8687

8788
const size_t CopySize = sizeof(size_t) * WorkDim;

‎source/adapters/hip/command_buffer.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
// Struct that stores all the information related to a kernel command in a
4040
// command-buffer, such that the command can be recreated. When handles can
4141
// be returned from other command types this struct will need refactored.
42-
struct ur_exp_command_buffer_command_handle_t_ {
42+
struct ur_exp_command_buffer_command_handle_t_ : ur_object_t_ {
4343
ur_exp_command_buffer_command_handle_t_(
4444
ur_exp_command_buffer_handle_t CommandBuffer, ur_kernel_handle_t Kernel,
4545
hipGraphNode_t Node, hipKernelNodeParams Params, uint32_t WorkDim,
@@ -121,7 +121,7 @@ struct ur_exp_command_buffer_command_handle_t_ {
121121
std::atomic_uint32_t RefCountExternal;
122122
};
123123

124-
struct ur_exp_command_buffer_handle_t_ {
124+
struct ur_exp_command_buffer_handle_t_ : ur_object_t_ {
125125

126126
ur_exp_command_buffer_handle_t_(ur_context_handle_t hContext,
127127
ur_device_handle_t hDevice, bool IsUpdatable);

‎source/adapters/hip/common.hpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,3 +239,8 @@ inline static unsigned getMemoryType(hipPointerAttribute_t hipPointerAttrs) {
239239
return hipPointerAttrs.memoryType;
240240
#endif
241241
}
242+
243+
struct hip_ddi_getter {
244+
static ur_dditable_t *value();
245+
};
246+
using ur_object_t_ = ur_handle_base_t_<hip_ddi_getter>;

‎source/adapters/hip/context.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ typedef void (*ur_context_extended_deleter_t)(void *UserData);
7676
/// between native allocations for devices in the same \c ur_context_handle_t_
7777
/// if necessary.
7878
///
79-
struct ur_context_handle_t_ {
79+
struct ur_context_handle_t_ : ur_object_t_ {
8080

8181
struct deleter_data {
8282
ur_context_extended_deleter_t Function;
@@ -90,7 +90,7 @@ struct ur_context_handle_t_ {
9090
std::atomic_uint32_t RefCount;
9191

9292
ur_context_handle_t_(const ur_device_handle_t *Devs, uint32_t NumDevices)
93-
: Devices{Devs, Devs + NumDevices}, RefCount{1} {
93+
: ur_object_t_(), Devices{Devs, Devs + NumDevices}, RefCount{1} {
9494
for (auto &Dev : Devices) {
9595
urDeviceRetain(Dev);
9696
}

‎source/adapters/hip/device.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
/// Includes an observer pointer to the platform,
1818
/// and implements the reference counting semantics since
1919
/// HIP objects are not refcounted.
20-
struct ur_device_handle_t_ {
20+
struct ur_device_handle_t_ : ur_object_t_ {
2121
private:
2222
using native_type = hipDevice_t;
2323

@@ -38,8 +38,8 @@ struct ur_device_handle_t_ {
3838
public:
3939
ur_device_handle_t_(native_type HipDevice, hipEvent_t EvBase,
4040
ur_platform_handle_t Platform, uint32_t DeviceIndex)
41-
: HIPDevice(HipDevice), RefCount{1}, Platform(Platform), EvBase(EvBase),
42-
DeviceIndex(DeviceIndex) {
41+
: ur_object_t_(), HIPDevice(HipDevice), RefCount{1}, Platform(Platform),
42+
EvBase(EvBase), DeviceIndex(DeviceIndex) {
4343

4444
UR_CHECK_ERROR(hipDeviceGetAttribute(
4545
&MaxWorkGroupSize, hipDeviceAttributeMaxThreadsPerBlock, HIPDevice));

‎source/adapters/hip/event.cpp

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
1919
hipEvent_t EvEnd, hipEvent_t EvQueued,
2020
hipEvent_t EvStart, hipStream_t Stream,
2121
uint32_t StreamToken)
22-
: CommandType{Type}, RefCount{1}, HasOwnership{true},
22+
: ur_object_t_(), CommandType{Type}, RefCount{1}, HasOwnership{true},
2323
HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
2424
StreamToken{StreamToken}, EventId{0}, EvEnd{EvEnd}, EvStart{EvStart},
2525
EvQueued{EvQueued}, Queue{Queue}, Stream{Stream}, Context{Context} {
@@ -29,11 +29,12 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
2929

3030
ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t Context,
3131
hipEvent_t EventNative)
32-
: CommandType{UR_COMMAND_EVENTS_WAIT}, RefCount{1}, HasOwnership{false},
33-
HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
34-
IsInterop{true}, StreamToken{std::numeric_limits<uint32_t>::max()},
35-
EventId{0}, EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr},
36-
Queue{nullptr}, Stream{nullptr}, Context{Context} {
32+
: ur_object_t_(), CommandType{UR_COMMAND_EVENTS_WAIT}, RefCount{1},
33+
HasOwnership{false}, HasBeenWaitedOn{false}, IsRecorded{false},
34+
IsStarted{false}, IsInterop{true},
35+
StreamToken{std::numeric_limits<uint32_t>::max()}, EventId{0},
36+
EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr},
37+
Stream{nullptr}, Context{Context} {
3738
urContextRetain(Context);
3839
}
3940

‎source/adapters/hip/event.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
/// UR Event mapping to hipEvent_t
1616
///
17-
struct ur_event_handle_t_ {
17+
struct ur_event_handle_t_ : ur_object_t_ {
1818
public:
1919
using native_type = hipEvent_t;
2020

‎source/adapters/hip/kernel.hpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
/// A compiler pass converts the UR API local memory model into the
3333
/// HIP shared model. This object simply calculates the total of
3434
/// shared memory, and the initial offsets of each parameter.
35-
struct ur_kernel_handle_t_ {
35+
struct ur_kernel_handle_t_ : ur_object_t_ {
3636
using native_type = hipFunction_t;
3737

3838
native_type Function;
@@ -232,8 +232,9 @@ struct ur_kernel_handle_t_ {
232232
ur_kernel_handle_t_(hipFunction_t Func, hipFunction_t FuncWithOffsetParam,
233233
const char *Name, ur_program_handle_t Program,
234234
ur_context_handle_t Ctxt)
235-
: Function{Func}, FunctionWithOffsetParam{FuncWithOffsetParam},
236-
Name{Name}, Context{Ctxt}, Program{Program}, RefCount{1} {
235+
: ur_object_t_(), Function{Func},
236+
FunctionWithOffsetParam{FuncWithOffsetParam}, Name{Name}, Context{Ctxt},
237+
Program{Program}, RefCount{1} {
237238
assert(Program->getDevice());
238239
UR_CHECK_ERROR(urKernelGetGroupInfo(
239240
this, Program->getDevice(),

‎source/adapters/hip/memory.hpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ struct SurfaceMem {
307307
/// Migrations will occur in both cases if the most recent version of data
308308
/// is on a different device, marked by LastQueueWritingToMemObj->getDevice().
309309
///
310-
struct ur_mem_handle_t_ {
310+
struct ur_mem_handle_t_ : ur_object_t_ {
311311

312312
// TODO: Move as much shared data up as possible
313313
using ur_context = ur_context_handle_t_ *;
@@ -355,9 +355,9 @@ struct ur_mem_handle_t_ {
355355

356356
// Subbuffer constructor
357357
ur_mem_handle_t_(ur_mem Parent, size_t SubBufferOffset)
358-
: Context{Parent->Context}, RefCount{1}, MemFlags{Parent->MemFlags},
359-
HaveMigratedToDeviceSinceLastWrite(Parent->Context->Devices.size(),
360-
false),
358+
: ur_object_t_(), Context{Parent->Context}, RefCount{1},
359+
MemFlags{Parent->MemFlags}, HaveMigratedToDeviceSinceLastWrite(
360+
Parent->Context->Devices.size(), false),
361361
Mem{BufferMem{std::get<BufferMem>(Parent->Mem)}} {
362362
auto &SubBuffer = std::get<BufferMem>(Mem);
363363
SubBuffer.Parent = Parent;

‎source/adapters/hip/physical_mem.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@
1717
/// management.
1818
/// TODO: Implement.
1919
///
20-
struct ur_physical_mem_handle_t_ {
20+
struct ur_physical_mem_handle_t_ : ur_object_t_ {
2121
std::atomic_uint32_t RefCount;
2222

23-
ur_physical_mem_handle_t_() : RefCount(1) {}
23+
ur_physical_mem_handle_t_() : ur_object_t_(), RefCount(1) {}
2424

2525
uint32_t incrementReferenceCount() noexcept { return ++RefCount; }
2626

‎source/adapters/hip/platform.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@
1919
/// available devices since initialization is done
2020
/// when devices are used.
2121
///
22-
struct ur_platform_handle_t_ {
22+
struct ur_platform_handle_t_ : ur_object_t_ {
2323
std::vector<std::unique_ptr<ur_device_handle_t_>> Devices;
2424
};

‎source/adapters/hip/program.hpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
#include "context.hpp"
1818

1919
/// Implementation of UR Program on HIP Module object
20-
struct ur_program_handle_t_ {
20+
struct ur_program_handle_t_ : ur_object_t_ {
2121
using native_type = hipModule_t;
2222
native_type Module;
2323
const char *Binary;
@@ -47,8 +47,9 @@ struct ur_program_handle_t_ {
4747
ur_program_build_status_t BuildStatus = UR_PROGRAM_BUILD_STATUS_NONE;
4848

4949
ur_program_handle_t_(ur_context_handle_t Ctxt, ur_device_handle_t Device)
50-
: Module{nullptr}, Binary{}, BinarySizeInBytes{0}, RefCount{1},
51-
Context{Ctxt}, Device{Device}, KernelReqdWorkGroupSizeMD{} {
50+
: ur_object_t_(), Module{nullptr}, Binary{}, BinarySizeInBytes{0},
51+
RefCount{1}, Context{Ctxt}, Device{Device},
52+
KernelReqdWorkGroupSizeMD{} {
5253
urContextRetain(Context);
5354
urDeviceRetain(Device);
5455
}

‎source/adapters/hip/queue.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ using ur_stream_guard = std::unique_lock<std::mutex>;
1818

1919
/// UR queue mapping on to hipStream_t objects.
2020
///
21-
struct ur_queue_handle_t_ {
21+
struct ur_queue_handle_t_ : ur_object_t_ {
2222
using native_type = hipStream_t;
2323
static constexpr int DefaultNumComputeStreams = 64;
2424
static constexpr int DefaultNumTransferStreams = 16;
@@ -66,7 +66,7 @@ struct ur_queue_handle_t_ {
6666
ur_context_handle_t Context, ur_device_handle_t Device,
6767
unsigned int Flags, ur_queue_flags_t URFlags, int Priority,
6868
bool BackendOwns = true)
69-
: ComputeStreams{std::move(ComputeStreams)},
69+
: ur_object_t_(), ComputeStreams{std::move(ComputeStreams)},
7070
TransferStreams{std::move(TransferStreams)},
7171
DelayCompute(this->ComputeStreams.size(), false),
7272
ComputeAppliedBarrier(this->ComputeStreams.size()),

‎source/adapters/hip/sampler.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@
1717
/// Sampler property layout:
1818
/// | 31 30 ... 6 5 | 4 3 2 | 1 | 0 |
1919
/// | N/A | addressing mode | fiter mode | normalize coords |
20-
struct ur_sampler_handle_t_ {
20+
struct ur_sampler_handle_t_ : ur_object_t_ {
2121
std::atomic_uint32_t RefCount;
2222
uint32_t Props;
2323
ur_context_handle_t Context;
2424

2525
ur_sampler_handle_t_(ur_context_handle_t Context)
26-
: RefCount(1), Props(0), Context(Context) {}
26+
: ur_object_t_(), RefCount(1), Props(0), Context(Context) {}
2727

2828
uint32_t incrementReferenceCount() noexcept { return ++RefCount; }
2929

0 commit comments

Comments
 (0)