Skip to content

Commit 1678894

Browse files
aarongreigaarongreig
authored
Merge branch 'main' into steffen/virtual_mem_adapters
2 parents 0563259 + 67e4d1b commit 1678894

21 files changed

+83
-96
lines changed

‎source/adapters/cuda/command_buffer.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121

2222
ur_exp_command_buffer_handle_t_::ur_exp_command_buffer_handle_t_(
2323
ur_context_handle_t hContext, ur_device_handle_t hDevice)
24-
: Context(hContext),
25-
Device(hDevice), CudaGraph{nullptr}, CudaGraphExec{nullptr}, RefCount{1} {
24+
: Context(hContext), Device(hDevice), CudaGraph{nullptr},
25+
CudaGraphExec{nullptr}, RefCount{1}, NextSyncPoint{0} {
2626
urContextRetain(hContext);
2727
urDeviceRetain(hDevice);
2828
}

‎source/adapters/cuda/command_buffer.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ struct ur_exp_command_buffer_handle_t_ {
184184

185185
void RegisterSyncPoint(ur_exp_command_buffer_sync_point_t SyncPoint,
186186
std::shared_ptr<CUgraphNode> CuNode) {
187-
SyncPoints[SyncPoint] = CuNode;
187+
SyncPoints[SyncPoint] = std::move(CuNode);
188188
NextSyncPoint++;
189189
}
190190

@@ -193,12 +193,12 @@ struct ur_exp_command_buffer_handle_t_ {
193193
}
194194

195195
// Helper to register next sync point
196-
// @param CuNode Node to register as next sycn point
196+
// @param CuNode Node to register as next sync point
197197
// @return Pointer to the sync that registers the Node
198198
ur_exp_command_buffer_sync_point_t
199199
AddSyncPoint(std::shared_ptr<CUgraphNode> CuNode) {
200200
ur_exp_command_buffer_sync_point_t SyncPoint = NextSyncPoint;
201-
RegisterSyncPoint(SyncPoint, CuNode);
201+
RegisterSyncPoint(SyncPoint, std::move(CuNode));
202202
return SyncPoint;
203203
}
204204

‎source/adapters/cuda/device.cpp

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1144,17 +1144,18 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceCreateWithNativeHandle(
11441144
if (Result != UR_RESULT_SUCCESS)
11451145
return Result;
11461146

1147-
ur_platform_handle_t *Plat = static_cast<ur_platform_handle_t *>(
1148-
malloc(NumPlatforms * sizeof(ur_platform_handle_t)));
1149-
Result = urPlatformGet(&AdapterHandle, 1, NumPlatforms, Plat, nullptr);
1147+
std::vector<ur_platform_handle_t> Platforms(NumPlatforms);
1148+
1149+
Result =
1150+
urPlatformGet(&AdapterHandle, 1, NumPlatforms, Platforms.data(), nullptr);
11501151
if (Result != UR_RESULT_SUCCESS)
11511152
return Result;
11521153

11531154
// Iterate through platforms to find device that matches nativeHandle
1154-
for (uint32_t j = 0; j < NumPlatforms; ++j) {
1155-
auto SearchRes =
1156-
std::find_if(begin(Plat[j]->Devices), end(Plat[j]->Devices), IsDevice);
1157-
if (SearchRes != end(Plat[j]->Devices)) {
1155+
for (const auto Platform : Platforms) {
1156+
auto SearchRes = std::find_if(std::begin(Platform->Devices),
1157+
std::end(Platform->Devices), IsDevice);
1158+
if (SearchRes != end(Platform->Devices)) {
11581159
*phDevice = static_cast<ur_device_handle_t>((*SearchRes).get());
11591160
return UR_RESULT_SUCCESS;
11601161
}

‎source/adapters/cuda/event.cpp

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
//===----------------------------------------------------------------------===//
1010

1111
#include "event.hpp"
12-
#include "common.hpp"
1312
#include "context.hpp"
1413
#include "device.hpp"
1514
#include "queue.hpp"
@@ -19,35 +18,25 @@
1918

2019
ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
2120
ur_context_handle_t Context,
22-
ur_queue_handle_t Queue, CUstream Stream,
21+
ur_queue_handle_t Queue,
22+
native_type EvEnd, native_type EvQueued,
23+
native_type EvStart, CUstream Stream,
2324
uint32_t StreamToken)
2425
: CommandType{Type}, RefCount{1}, HasOwnership{true},
2526
HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
26-
StreamToken{StreamToken}, EvEnd{nullptr}, EvStart{nullptr},
27-
EvQueued{nullptr}, Queue{Queue}, Stream{Stream}, Context{Context} {
28-
29-
bool ProfilingEnabled = Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE;
30-
31-
UR_CHECK_ERROR(cuEventCreate(
32-
&EvEnd, ProfilingEnabled ? CU_EVENT_DEFAULT : CU_EVENT_DISABLE_TIMING));
33-
34-
if (ProfilingEnabled) {
35-
UR_CHECK_ERROR(cuEventCreate(&EvQueued, CU_EVENT_DEFAULT));
36-
UR_CHECK_ERROR(cuEventCreate(&EvStart, CU_EVENT_DEFAULT));
37-
}
38-
39-
if (Queue != nullptr) {
40-
urQueueRetain(Queue);
41-
}
27+
StreamToken{StreamToken}, EventID{0}, EvEnd{EvEnd}, EvStart{EvStart},
28+
EvQueued{EvQueued}, Queue{Queue}, Stream{Stream}, Context{Context} {
29+
urQueueRetain(Queue);
4230
urContextRetain(Context);
4331
}
4432

4533
ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t Context,
4634
CUevent EventNative)
4735
: CommandType{UR_COMMAND_EVENTS_WAIT}, RefCount{1}, HasOwnership{false},
4836
HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
49-
StreamToken{std::numeric_limits<uint32_t>::max()}, EvEnd{EventNative},
50-
EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr}, Context{Context} {
37+
StreamToken{std::numeric_limits<uint32_t>::max()}, EventID{0},
38+
EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr},
39+
Stream{nullptr}, Context{Context} {
5140
urContextRetain(Context);
5241
}
5342

‎source/adapters/cuda/event.hpp

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <cuda.h>
1313
#include <ur/ur.hpp>
1414

15+
#include "common.hpp"
1516
#include "queue.hpp"
1617

1718
/// UR Event mapping to CUevent
@@ -82,8 +83,18 @@ struct ur_event_handle_t_ {
8283
static ur_event_handle_t
8384
makeNative(ur_command_t Type, ur_queue_handle_t Queue, CUstream Stream,
8485
uint32_t StreamToken = std::numeric_limits<uint32_t>::max()) {
85-
return new ur_event_handle_t_(Type, Queue->getContext(), Queue, Stream,
86-
StreamToken);
86+
const bool ProfilingEnabled =
87+
Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE;
88+
native_type EvEnd = nullptr, EvQueued = nullptr, EvStart = nullptr;
89+
UR_CHECK_ERROR(cuEventCreate(
90+
&EvEnd, ProfilingEnabled ? CU_EVENT_DEFAULT : CU_EVENT_DISABLE_TIMING));
91+
92+
if (ProfilingEnabled) {
93+
UR_CHECK_ERROR(cuEventCreate(&EvQueued, CU_EVENT_DEFAULT));
94+
UR_CHECK_ERROR(cuEventCreate(&EvStart, CU_EVENT_DEFAULT));
95+
}
96+
return new ur_event_handle_t_(Type, Queue->getContext(), Queue, EvEnd,
97+
EvQueued, EvStart, Stream, StreamToken);
8798
}
8899

89100
static ur_event_handle_t makeWithNative(ur_context_handle_t context,
@@ -99,7 +110,8 @@ struct ur_event_handle_t_ {
99110
// This constructor is private to force programmers to use the makeNative /
100111
// make_user static members in order to create a pi_event for CUDA.
101112
ur_event_handle_t_(ur_command_t Type, ur_context_handle_t Context,
102-
ur_queue_handle_t Queue, CUstream Stream,
113+
ur_queue_handle_t Queue, native_type EvEnd,
114+
native_type EvQueued, native_type EvStart, CUstream Stream,
103115
uint32_t StreamToken);
104116

105117
// This constructor is private to force programmers to use the

‎source/adapters/cuda/image.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ cudaToUrImageChannelFormat(CUarray_format cuda_format,
234234

235235
ur_result_t urTextureCreate(ur_sampler_handle_t hSampler,
236236
const ur_image_desc_t *pImageDesc,
237-
CUDA_RESOURCE_DESC ResourceDesc,
237+
const CUDA_RESOURCE_DESC &ResourceDesc,
238238
ur_exp_image_handle_t *phRetImage) {
239239

240240
try {

‎source/adapters/cuda/memory.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,15 +190,15 @@ struct ur_mem_handle_t_ {
190190
/// Constructs the UR allocation for an unsampled image object
191191
ur_mem_handle_t_(ur_context_handle_t Context, CUarray Array,
192192
CUsurfObject Surf, ur_mem_type_t ImageType)
193-
: Context{Context}, RefCount{1}, MemType{Type::Surface},
193+
: Context{Context}, RefCount{1}, MemType{Type::Surface}, MemFlags{0},
194194
Mem{ImageMem{Array, (void *)Surf, ImageType, nullptr}} {
195195
urContextRetain(Context);
196196
}
197197

198198
/// Constructs the UR allocation for a sampled image object
199199
ur_mem_handle_t_(ur_context_handle_t Context, CUarray Array, CUtexObject Tex,
200200
ur_sampler_handle_t Sampler, ur_mem_type_t ImageType)
201-
: Context{Context}, RefCount{1}, MemType{Type::Texture},
201+
: Context{Context}, RefCount{1}, MemType{Type::Texture}, MemFlags{0},
202202
Mem{ImageMem{Array, (void *)Tex, ImageType, Sampler}} {
203203
urContextRetain(Context);
204204
}

‎source/adapters/cuda/program.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ ur_result_t ur_program_handle_t_::buildProgram(const char *BuildOptions) {
137137

138138
if (!this->BuildOptions.empty()) {
139139
unsigned int MaxRegs;
140-
bool Valid = getMaxRegistersJitOptionValue(BuildOptions, MaxRegs);
140+
const bool Valid =
141+
getMaxRegistersJitOptionValue(this->BuildOptions, MaxRegs);
141142
if (Valid) {
142143
Options.push_back(CU_JIT_MAX_REGISTERS);
143144
OptionVals.push_back(reinterpret_cast<void *>(MaxRegs));

‎source/adapters/cuda/sampler.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ urSamplerCreate(ur_context_handle_t hContext, const ur_sampler_desc_t *pDesc,
1717
std::unique_ptr<ur_sampler_handle_t_> Sampler{
1818
new ur_sampler_handle_t_(hContext)};
1919

20-
if (pDesc && pDesc->stype == UR_STRUCTURE_TYPE_SAMPLER_DESC) {
20+
if (pDesc->stype == UR_STRUCTURE_TYPE_SAMPLER_DESC) {
2121
Sampler->Props |= pDesc->normalizedCoords;
2222
Sampler->Props |= pDesc->filterMode << 1;
2323
Sampler->Props |= pDesc->addressingMode << 2;

‎source/adapters/hip/device.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ struct ur_device_handle_t_ {
3333
: HIPDevice(HipDevice), RefCount{1}, Platform(Platform),
3434
HIPContext(Context), DeviceIndex(DeviceIndex) {}
3535

36-
~ur_device_handle_t_() {
36+
~ur_device_handle_t_() noexcept(false) {
3737
UR_CHECK_ERROR(hipDevicePrimaryCtxRelease(HIPDevice));
3838
}
3939

‎source/adapters/hip/enqueue.cpp

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -987,8 +987,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageRead(
987987
const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) {
988988
UR_ASSERT(hImage->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT);
989989

990-
ur_result_t Result = UR_RESULT_SUCCESS;
991-
992990
ur_lock MemoryMigrationLock{hImage->MemoryMigrationMutex};
993991
auto Device = hQueue->getDevice();
994992
hipStream_t HIPStream = hQueue->getNextTransferStream();
@@ -1039,13 +1037,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageRead(
10391037
UR_CHECK_ERROR(RetImplEvent->start());
10401038
}
10411039

1042-
Result = commonEnqueueMemImageNDCopy(HIPStream, ImgType, AdjustedRegion,
1043-
Array, hipMemoryTypeArray, SrcOffset,
1044-
pDst, hipMemoryTypeHost, nullptr);
1045-
1046-
if (Result != UR_RESULT_SUCCESS) {
1047-
return Result;
1048-
}
1040+
UR_CHECK_ERROR(commonEnqueueMemImageNDCopy(
1041+
HIPStream, ImgType, AdjustedRegion, Array, hipMemoryTypeArray,
1042+
SrcOffset, pDst, hipMemoryTypeHost, nullptr));
10491043

10501044
if (phEvent) {
10511045
UR_CHECK_ERROR(RetImplEvent->record());
@@ -1061,7 +1055,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageRead(
10611055
return UR_RESULT_ERROR_UNKNOWN;
10621056
}
10631057
return UR_RESULT_SUCCESS;
1064-
return Result;
10651058
}
10661059

10671060
UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageWrite(
@@ -1071,15 +1064,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageWrite(
10711064
const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) {
10721065
UR_ASSERT(hImage->isImage(), UR_RESULT_ERROR_INVALID_MEM_OBJECT);
10731066

1074-
ur_result_t Result = UR_RESULT_SUCCESS;
1075-
10761067
try {
10771068
ScopedContext Active(hQueue->getDevice());
10781069
hipStream_t HIPStream = hQueue->getNextTransferStream();
10791070

10801071
if (phEventWaitList) {
1081-
Result = enqueueEventsWait(hQueue, HIPStream, numEventsInWaitList,
1082-
phEventWaitList);
1072+
UR_CHECK_ERROR(enqueueEventsWait(hQueue, HIPStream, numEventsInWaitList,
1073+
phEventWaitList));
10831074
}
10841075

10851076
hipArray *Array =
@@ -1107,13 +1098,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageWrite(
11071098
UR_CHECK_ERROR(RetImplEvent->start());
11081099
}
11091100

1110-
Result = commonEnqueueMemImageNDCopy(HIPStream, ImgType, AdjustedRegion,
1111-
pSrc, hipMemoryTypeHost, nullptr,
1112-
Array, hipMemoryTypeArray, DstOffset);
1113-
1114-
if (Result != UR_RESULT_SUCCESS) {
1115-
return Result;
1116-
}
1101+
UR_CHECK_ERROR(commonEnqueueMemImageNDCopy(
1102+
HIPStream, ImgType, AdjustedRegion, pSrc, hipMemoryTypeHost, nullptr,
1103+
Array, hipMemoryTypeArray, DstOffset));
11171104

11181105
if (phEvent) {
11191106
UR_CHECK_ERROR(RetImplEvent->record());
@@ -1126,8 +1113,6 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageWrite(
11261113
}
11271114

11281115
return UR_RESULT_SUCCESS;
1129-
1130-
return Result;
11311116
}
11321117

11331118
UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageCopy(

‎source/adapters/hip/event.cpp

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
1919
hipStream_t Stream, uint32_t StreamToken)
2020
: CommandType{Type}, RefCount{1}, HasOwnership{true},
2121
HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
22-
StreamToken{StreamToken}, EvEnd{nullptr}, EvStart{nullptr},
22+
StreamToken{StreamToken}, EventId{0}, EvEnd{nullptr}, EvStart{nullptr},
2323
EvQueued{nullptr}, Queue{Queue}, Stream{Stream}, Context{Context} {
2424

2525
bool ProfilingEnabled = Queue->URFlags & UR_QUEUE_FLAG_PROFILING_ENABLE;
@@ -32,18 +32,17 @@ ur_event_handle_t_::ur_event_handle_t_(ur_command_t Type,
3232
UR_CHECK_ERROR(hipEventCreateWithFlags(&EvStart, hipEventDefault));
3333
}
3434

35-
if (Queue != nullptr) {
36-
urQueueRetain(Queue);
37-
}
35+
urQueueRetain(Queue);
3836
urContextRetain(Context);
3937
}
4038

4139
ur_event_handle_t_::ur_event_handle_t_(ur_context_handle_t Context,
4240
hipEvent_t EventNative)
4341
: CommandType{UR_COMMAND_EVENTS_WAIT}, RefCount{1}, HasOwnership{false},
4442
HasBeenWaitedOn{false}, IsRecorded{false}, IsStarted{false},
45-
StreamToken{std::numeric_limits<uint32_t>::max()}, EvEnd{EventNative},
46-
EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr}, Context{Context} {
43+
StreamToken{std::numeric_limits<uint32_t>::max()}, EventId{0},
44+
EvEnd{EventNative}, EvStart{nullptr}, EvQueued{nullptr}, Queue{nullptr},
45+
Stream{nullptr}, Context{Context} {
4746
urContextRetain(Context);
4847
}
4948

@@ -72,7 +71,7 @@ ur_result_t ur_event_handle_t_::start() {
7271
return Result;
7372
}
7473

75-
bool ur_event_handle_t_::isCompleted() const noexcept {
74+
bool ur_event_handle_t_::isCompleted() const {
7675
if (!IsRecorded) {
7776
return false;
7877
}
@@ -225,8 +224,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(ur_event_handle_t hEvent,
225224
return ReturnValue(hEvent->getCommandType());
226225
case UR_EVENT_INFO_REFERENCE_COUNT:
227226
return ReturnValue(hEvent->getReferenceCount());
228-
case UR_EVENT_INFO_COMMAND_EXECUTION_STATUS:
229-
return ReturnValue(hEvent->getExecutionStatus());
227+
case UR_EVENT_INFO_COMMAND_EXECUTION_STATUS: {
228+
try {
229+
return ReturnValue(hEvent->getExecutionStatus());
230+
} catch (ur_result_t Error) {
231+
return Error;
232+
}
233+
}
230234
case UR_EVENT_INFO_CONTEXT:
231235
return ReturnValue(hEvent->getContext());
232236
default:

‎source/adapters/hip/event.hpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,9 @@ struct ur_event_handle_t_ {
4242

4343
bool isStarted() const noexcept { return IsStarted; }
4444

45-
bool isCompleted() const noexcept;
46-
47-
uint32_t getExecutionStatus() const noexcept {
45+
bool isCompleted() const;
4846

47+
uint32_t getExecutionStatus() const {
4948
if (!isRecorded()) {
5049
return UR_EVENT_STATUS_SUBMITTED;
5150
}

‎source/adapters/hip/memory.hpp

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -187,20 +187,16 @@ struct SurfaceMem {
187187
void *HostPtr)
188188
: Arrays(Context->Devices.size(), nullptr),
189189
SurfObjs(Context->Devices.size(), nullptr),
190-
OuterMemStruct{OuterMemStruct},
191-
ImageFormat{ImageFormat}, ImageDesc{ImageDesc}, HostPtr{HostPtr} {
190+
OuterMemStruct{OuterMemStruct}, ImageFormat{ImageFormat},
191+
ImageDesc{ImageDesc}, ArrayDesc{}, HostPtr{HostPtr} {
192192
// We have to use hipArray3DCreate, which has some caveats. The height and
193193
// depth parameters must be set to 0 produce 1D or 2D arrays. image_desc
194194
// gives a minimum value of 1, so we need to convert the answer.
195195
ArrayDesc.NumChannels = 4; // Only support 4 channel image
196196
ArrayDesc.Flags = 0; // No flags required
197197
ArrayDesc.Width = ImageDesc.width;
198-
if (ImageDesc.type == UR_MEM_TYPE_IMAGE1D) {
199-
ArrayDesc.Height = 0;
200-
ArrayDesc.Depth = 0;
201-
} else if (ImageDesc.type == UR_MEM_TYPE_IMAGE2D) {
198+
if (ImageDesc.type == UR_MEM_TYPE_IMAGE2D) {
202199
ArrayDesc.Height = ImageDesc.height;
203-
ArrayDesc.Depth = 0;
204200
} else if (ImageDesc.type == UR_MEM_TYPE_IMAGE3D) {
205201
ArrayDesc.Height = ImageDesc.height;
206202
ArrayDesc.Depth = ImageDesc.depth;
@@ -456,7 +452,7 @@ struct ur_mem_handle_t_ {
456452
urContextRetain(Context);
457453
}
458454

459-
~ur_mem_handle_t_() {
455+
~ur_mem_handle_t_() noexcept(false) {
460456
if (isBuffer() && isSubBuffer()) {
461457
urMemRelease(std::get<BufferMem>(Mem).Parent);
462458
return;
@@ -468,7 +464,7 @@ struct ur_mem_handle_t_ {
468464
return std::holds_alternative<BufferMem>(Mem);
469465
}
470466

471-
bool isSubBuffer() const noexcept {
467+
bool isSubBuffer() const {
472468
return (isBuffer() && (std::get<BufferMem>(Mem).Parent != nullptr));
473469
}
474470

0 commit comments

Comments
 (0)