diff --git a/sycl/test-e2e/Graph/Explicit/regression_accessor_spv.cpp b/sycl/test-e2e/Graph/Explicit/regression_accessor_spv.cpp index c9c2c9171cabd..56f19fe71e63a 100644 --- a/sycl/test-e2e/Graph/Explicit/regression_accessor_spv.cpp +++ b/sycl/test-e2e/Graph/Explicit/regression_accessor_spv.cpp @@ -5,6 +5,9 @@ // REQUIRES: level_zero +// UNSUPPORTED: arch-intel_gpu_pvc +// UNSUPPORTED-TRACKER: https://github.com/intel/llvm/issues/21235 + // Modified version of Update/update_with_indices_accessor_spv.cpp that does // not require the full graph aspect, test was hanging after some changes to // kernel bundles so adding this test for the CI which doesn't support update diff --git a/unified-runtime/include/ur_api.h b/unified-runtime/include/ur_api.h index 2b23e3cd0c59e..c401934937c9e 100644 --- a/unified-runtime/include/ur_api.h +++ b/unified-runtime/include/ur_api.h @@ -10994,226 +10994,150 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( #if !defined(__GNUC__) #pragma endregion #endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for Command-Buffers +// Intel 'oneAPI' Unified Runtime Experimental APIs for device-wide +// synchronization #if !defined(__GNUC__) -#pragma region command_buffer_(experimental) +#pragma region device_wait_(experimental) #endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Device kernel execution capability -typedef uint32_t ur_device_command_buffer_update_capability_flags_t; -typedef enum ur_device_command_buffer_update_capability_flag_t { - /// Device supports updating the kernel arguments in command-buffer - /// commands. - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS = UR_BIT(0), - /// Device supports updating the local work-group size in command-buffer - /// commands. - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE = UR_BIT(1), - /// Device supports updating the global work-group size in command-buffer - /// commands. - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE = UR_BIT(2), - /// Device supports updating the global work offset in command-buffer - /// commands. - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET = - UR_BIT(3), - /// Device supports updating the kernel handle in command-buffer commands. - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE = UR_BIT(4), - /// Device supports updating the event parameters in command-buffer - /// commands. - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS = UR_BIT(5), - /// @cond - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_FORCE_UINT32 = 0x7fffffff - /// @endcond - -} ur_device_command_buffer_update_capability_flag_t; -/// @brief Bit Mask for validating -/// ur_device_command_buffer_update_capability_flags_t -#define UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAGS_MASK 0xffffffc0 +/// @brief Synchronizes with all queues on the device. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hDevice` +/// + `hDevice == nullptr` +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +UR_APIEXPORT ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for dynamic linking +#if !defined(__GNUC__) +#pragma region dynamic_link_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Command-buffer query information type -typedef enum ur_exp_command_buffer_info_t { - /// [uint32_t] Reference count of the command-buffer object. - /// The reference count returned should be considered immediately stale. - /// It is unsuitable for general use in applications. This feature is - /// provided for identifying memory leaks. - UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT = 0, - /// [::ur_exp_command_buffer_desc_t] Returns a ::ur_exp_command_buffer_desc_t - /// with the properties of the command-buffer. Returned values may differ - /// from those passed on construction if the property was ignored by the - /// adapter. - UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR = 1, - /// @cond - UR_EXP_COMMAND_BUFFER_INFO_FORCE_UINT32 = 0x7fffffff - /// @endcond - -} ur_exp_command_buffer_info_t; +/// @brief Creates dynamic links between exported and imported symbols in one or +/// more programs. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point the programs in +/// `phPrograms` will have all external symbols resolved and kernels +/// inside these programs would be ready for use. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPrograms` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If one of the programs in `phPrograms` isn't a valid program +/// object. +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `count == 0` +/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE +/// + If an error occurred while linking `phPrograms`. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the adapter has no means to support the operation. +UR_APIEXPORT ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental API for setting args at kernel +// launch +#if !defined(__GNUC__) +#pragma region enqueue_kernel_launch_with_args_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Command-buffer command query information type -typedef enum ur_exp_command_buffer_command_info_t { - /// [uint32_t] Reference count of the command-buffer object. - /// The reference count returned should be considered immediately stale. - /// It is unsuitable for general use in applications. This feature is - /// provided for identifying memory leaks. - UR_EXP_COMMAND_BUFFER_COMMAND_INFO_REFERENCE_COUNT = 0, +/// @brief What kind of kernel arg is this +typedef enum ur_exp_kernel_arg_type_t { + /// Kernel arg is a value. + UR_EXP_KERNEL_ARG_TYPE_VALUE = 0, + /// Kernel arg is a pointer. + UR_EXP_KERNEL_ARG_TYPE_POINTER = 1, + /// Kernel arg is a memory object. + UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ = 2, + /// Kernel arg is a local allocation. + UR_EXP_KERNEL_ARG_TYPE_LOCAL = 3, + /// Kernel arg is a sampler. + UR_EXP_KERNEL_ARG_TYPE_SAMPLER = 4, /// @cond - UR_EXP_COMMAND_BUFFER_COMMAND_INFO_FORCE_UINT32 = 0x7fffffff + UR_EXP_KERNEL_ARG_TYPE_FORCE_UINT32 = 0x7fffffff /// @endcond -} ur_exp_command_buffer_command_info_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Command-Buffer Descriptor Type -typedef struct ur_exp_command_buffer_desc_t { - /// [in] type of this structure, must be - /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC - ur_structure_type_t stype; - /// [in][optional] pointer to extension-specific structure - const void *pNext; - /// [in] Commands in a finalized command-buffer can be updated. - ur_bool_t isUpdatable; - /// [in] Commands in a command-buffer will execute in-order, explicit - /// sync-point dependencies are ignored. - ur_bool_t isInOrder; - /// [in] Command-buffer profiling is enabled. - ur_bool_t enableProfiling; - -} ur_exp_command_buffer_desc_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief A value that identifies a command inside of a command-buffer, used -/// for -/// defining dependencies between commands in the same command-buffer. -typedef uint32_t ur_exp_command_buffer_sync_point_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Handle of Command-Buffer object -typedef struct ur_exp_command_buffer_handle_t_ *ur_exp_command_buffer_handle_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Handle of a Command-Buffer command -typedef struct ur_exp_command_buffer_command_handle_t_ - *ur_exp_command_buffer_command_handle_t; +} ur_exp_kernel_arg_type_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Descriptor type for updating a kernel command memobj argument. -typedef struct ur_exp_command_buffer_update_memobj_arg_desc_t { - /// [in] type of this structure, must be - /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC - ur_structure_type_t stype; - /// [in][optional] pointer to extension-specific structure - const void *pNext; - /// [in] Argument index. - uint32_t argIndex; - /// [in][optional] Pointer to memory object properties. - const ur_kernel_arg_mem_obj_properties_t *pProperties; - /// [in][optional] Handle of memory object to set at argument index. - ur_mem_handle_t hNewMemObjArg; +/// @brief Mem obj/properties tuple +typedef struct ur_exp_kernel_arg_mem_obj_tuple_t { + /// [in] Handle of a memory object + ur_mem_handle_t hMem; + /// [in] Memory flags to associate with `hMem`. Allowed values are: + /// ::UR_MEM_FLAG_READ_WRITE, ::UR_MEM_FLAG_WRITE_ONLY, + /// ::UR_MEM_FLAG_READ_ONLY. + ur_mem_flags_t flags; -} ur_exp_command_buffer_update_memobj_arg_desc_t; +} ur_exp_kernel_arg_mem_obj_tuple_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Descriptor type for updating a kernel command pointer argument. -typedef struct ur_exp_command_buffer_update_pointer_arg_desc_t { - /// [in] type of this structure, must be - /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC - ur_structure_type_t stype; - /// [in][optional] pointer to extension-specific structure - const void *pNext; - /// [in] Argument index. - uint32_t argIndex; - /// [in][optional] Pointer to USM pointer properties. - const ur_kernel_arg_pointer_properties_t *pProperties; - /// [in][optional] USM pointer to memory location holding the argument - /// value to set at argument index. - const void *pNewPointerArg; +/// @brief Typesafe container for a kernel argument value +typedef union ur_exp_kernel_arg_value_t { + /// [in] argument value represented as matching arg type. + /// The data pointed to will be copied and therefore can be reused on return. + const void *value; + /// [in] Allocation obtained by USM allocation or virtual memory mapping + /// operation, or pointer to a literal value. + const void *pointer; + /// [in] Struct containing a memory object and associated flags. + ur_exp_kernel_arg_mem_obj_tuple_t memObjTuple; + /// [in] Handle of a sampler object. + ur_sampler_handle_t sampler; -} ur_exp_command_buffer_update_pointer_arg_desc_t; +} ur_exp_kernel_arg_value_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Descriptor type for updating a kernel command value argument. -typedef struct ur_exp_command_buffer_update_value_arg_desc_t { +/// @brief Kernel arg properties +typedef struct ur_exp_kernel_arg_properties_t { /// [in] type of this structure, must be - /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC + /// ::UR_STRUCTURE_TYPE_EXP_KERNEL_ARG_PROPERTIES ur_structure_type_t stype; - /// [in][optional] pointer to extension-specific structure - const void *pNext; - /// [in] Argument index. - uint32_t argIndex; - /// [in] Argument size. - size_t argSize; - /// [in][optional] Pointer to value properties. - const ur_kernel_arg_value_properties_t *pProperties; - /// [in][optional] Argument value representing matching kernel arg type to - /// set at argument index. - const void *pNewValueArg; + /// [in,out][optional] pointer to extension-specific structure + void *pNext; + /// [in] type of the kernel arg + ur_exp_kernel_arg_type_t type; + /// [in] index of the kernel arg + uint32_t index; + /// [in] size of the kernel arg + size_t size; + /// [in][tagged_by(type)] Union containing the argument value. + ur_exp_kernel_arg_value_t value; -} ur_exp_command_buffer_update_value_arg_desc_t; +} ur_exp_kernel_arg_properties_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Descriptor type for updating a kernel launch command. -typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t { - /// [in] type of this structure, must be - /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_DESC - ur_structure_type_t stype; - /// [in][optional] pointer to extension-specific structure - const void *pNext; - /// [in] Handle of the command-buffer kernel command to update. - ur_exp_command_buffer_command_handle_t hCommand; - /// [in][optional] The new kernel handle. If this parameter is nullptr, - /// the current kernel handle in `hCommand` - /// will be used. If a kernel handle is passed, it must be a valid kernel - /// alternative as defined in - /// ::urCommandBufferAppendKernelLaunchExp. - ur_kernel_handle_t hNewKernel; - /// [in] Length of pNewMemObjArgList. - uint32_t numNewMemObjArgs; - /// [in] Length of pNewPointerArgList. - uint32_t numNewPointerArgs; - /// [in] Length of pNewValueArgList. - uint32_t numNewValueArgs; - /// [in] Number of work dimensions in the kernel ND-range, from 1-3. - uint32_t newWorkDim; - /// [in][optional][range(0, numNewMemObjArgs)] An array describing the new - /// kernel mem obj arguments for the command. - const ur_exp_command_buffer_update_memobj_arg_desc_t *pNewMemObjArgList; - /// [in][optional][range(0, numNewPointerArgs)] An array describing the - /// new kernel pointer arguments for the command. - const ur_exp_command_buffer_update_pointer_arg_desc_t *pNewPointerArgList; - /// [in][optional][range(0, numNewValueArgs)] An array describing the new - /// kernel value arguments for the command. - const ur_exp_command_buffer_update_value_arg_desc_t *pNewValueArgList; - /// [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned - /// values that describe the offset used - /// to calculate the global ID. If this parameter is nullptr, the current - /// global work offset will be used. This parameter is required if - /// `newWorkDim` is different from the current work dimensions - /// in the command. - size_t *pNewGlobalWorkOffset; - /// [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned - /// values that describe the number of - /// global work-items. If this parameter is nullptr, the current global - /// work size in `hCommand` will be used. - /// This parameter is required if `newWorkDim` is different from the - /// current work dimensions in the command. - size_t *pNewGlobalWorkSize; - /// [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned - /// values that describe the number of - /// work-items that make up a work-group. If `pNewGlobalWorkSize` is set - /// and `pNewLocalWorkSize` is nullptr, - /// then the runtime implementation will choose the local work size. If - /// `pNewGlobalWorkSize` is nullptr and - /// `pNewLocalWorkSize` is nullptr, the current local work size in the - /// command will be used. - size_t *pNewLocalWorkSize; - -} ur_exp_command_buffer_update_kernel_launch_desc_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Create a Command-Buffer object +/// @brief Enqueue a command to execute a kernel /// -/// @details -/// - Create a command-buffer object. +/// @remarks +/// _Analogues_ +/// - **clEnqueueNDRangeKernel** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11221,31 +11145,87 @@ typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t { /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` +/// + `NULL == hQueue` +/// + `NULL == hKernel` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pCommandBufferDesc` -/// + `NULL == phCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns -/// 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP -/// query. +/// + `NULL == pGlobalWorkSize` +/// + `pArgs == NULL && numArgs > 0` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `NULL != pArgs && ::UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type` +/// + `NULL != launchPropList && ::UR_KERNEL_LAUNCH_FLAGS_MASK & +/// launchPropList->flags` +/// - ::UR_RESULT_ERROR_INVALID_QUEUE +/// - ::UR_RESULT_ERROR_INVALID_KERNEL +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_IN_EVENT_LIST_EXEC_STATUS +/// + An event in `phEventWaitList` has ::UR_EVENT_STATUS_ERROR. +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// + `pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || +/// pGlobalWorkSize[2] == 0` +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// + `pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == +/// 0 || pLocalWorkSize[2] == 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGS - "The kernel argument values +/// have not been specified." /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. - ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer); +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If any property in `launchPropList` isn't supported by the device. +UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object + ur_kernel_handle_t hKernel, + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items + uint32_t workDim, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item + const size_t *pGlobalWorkOffset, + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function + const size_t *pGlobalWorkSize, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. + const size_t *pLocalWorkSize, + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for enqueuing timestamp +// recordings +#if !defined(__GNUC__) +#pragma region enqueue_timestamp_recording_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Increment the command-buffer object's reference count. +/// @brief Enqueue a command for recording the device timestamp /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11253,20 +11233,49 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer); +/// + `NULL == hQueue` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phEvent` +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for Inter Process +// Communication +#if !defined(__GNUC__) +#pragma region inter_process_communication_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Decrement the command-buffer object's reference count and delete the -/// command-buffer object if the reference count becomes zero. It is -/// legal to call the entry-point while `hCommandBuffer` is still -/// executing, which will block on completion if the reference count of -/// `hCommandBuffer` becomes zero. +/// @brief Gets an inter-process memory handle for a pointer to device USM +/// memory /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11274,17 +11283,25 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferRetainExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCMemHandleData` +/// + `NULL == pIPCMemHandleDataSizeRet` /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer); +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet); /////////////////////////////////////////////////////////////////////////////// -/// @brief Stop recording on a command-buffer object such that no more commands -/// can be appended and make it ready to enqueue. +/// @brief Releases an inter-process memory handle /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11292,18 +11309,21 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferReleaseExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "If `hCommandBuffer` has already -/// been finalized" +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer); +UR_APIEXPORT ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a kernel execution command to a command-buffer object. +/// @brief Opens an inter-process memory handle to get the corresponding pointer +/// to device USM memory /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11311,82 +11331,32 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferFinalizeExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hKernel` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pGlobalWorkSize` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_KERNEL -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCMemHandleData` +/// + `NULL == ppMem` /// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + `phKernelAlternatives == NULL && numKernelAlternatives > 0` -/// + `phKernelAlternatives != NULL && numKernelAlternatives == 0` -/// + If `phKernelAlternatives` contains `hKernel` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + ipcMemHandleDataSize is not the same as the size of IPC memory +/// handle data /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "phCommand is not NULL and -/// hCommandBuffer is not updatable." -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. - ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. - uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. - const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. - const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. - const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand); +UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM memcpy command to a command-buffer object. +/// @brief Closes an inter-process memory handle /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11394,61 +11364,28 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pSrc` or `pDst` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMem` /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Memory Export Extension APIs +#if !defined(__GNUC__) +#pragma region memory_export_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM fill command to a command-buffer object. +/// @brief Allocate an exportable memory region and return a pointer to that +/// allocation. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11456,65 +11393,44 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` -/// + `NULL == pPattern` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `patternSize == 0 || size == 0` -/// + `patternSize > size` -/// + `size % patternSize != 0` -/// + If `size` is higher than the allocation size of `ptr` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == ppMem` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT +/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `alignment` exceeds largest supported data type by `hDevice` +/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE +/// + `size == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE +/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - +/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory copy command to a command-buffer object. +/// @brief Free an exportable memory allocation. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11522,61 +11438,30 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hSrcMem` -/// + `NULL == hDstMem` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory write command to a command-buffer object. +/// @brief Obtain an exportable handle to a memory allocated with +/// `AllocExportableMemoryExp`.The returned external memory type will be +/// that which was specified upon +/// allocation of the exportable memory (e.g. `opaque_fd` or +/// `win32_nt_handle`). /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11584,60 +11469,66 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMemHandleRet || NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for multi-device compile +#if !defined(__GNUC__) +#pragma region multi_device_compile_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory read command to a command-buffer object. +/// @brief Program operation behavior control flags +typedef uint32_t ur_exp_program_flags_t; +typedef enum ur_exp_program_flag_t { + /// Allow unresolved symbols in the program resulting from the + /// corresponding operation + UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS = UR_BIT(0), + /// @cond + UR_EXP_PROGRAM_FLAG_FORCE_UINT32 = 0x7fffffff + /// @endcond + +} ur_exp_program_flag_t; +/// @brief Bit Mask for validating ur_exp_program_flags_t +#define UR_EXP_PROGRAM_FLAGS_MASK 0xfffffffe + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Produces an executable program from one program, negates need for the +/// linking step. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point, the program passed +/// will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type +/// for each device in `phDevices`. +/// +/// @remarks +/// _Analogues_ +/// - **clBuildProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11645,60 +11536,39 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hProgram` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +/// + `NULL == phDevices` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If `hProgram` isn't a valid program object. +/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE +/// + If an error occurred when building `hProgram`. +UR_APIEXPORT ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory copy command to a command-buffer object. +/// @brief Produces an executable program from one or more programs. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point `hProgram` will +/// contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type +/// for each device in `phDevices`. +/// +/// @remarks +/// _Analogues_ +/// - **clCompileProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11706,69 +11576,45 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hSrcMem` -/// + `NULL == hDstMem` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +/// + `NULL == hProgram` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phDevices` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If `hProgram` isn't a valid program object. +/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE +/// + If an error occurred while compiling `hProgram`. +UR_APIEXPORT ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory write command to a command-buffer object. +/// @brief Produces an executable program from one or more programs. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point the program returned +/// in `phProgram` will contain a binary of the +/// ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in +/// `phDevices`. +/// - If a non-success code is returned, adapters may store a program in +/// `phProgram`. This program should only be used with +/// `::urProgramGetBuildInfo` to get the build log for the failure. +/// Adapters which do not do not support producing build logs must set +/// this value to `nullptr`. +/// +/// @remarks +/// _Analogues_ +/// - **clLinkProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11776,73 +11622,48 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +/// + `NULL == phDevices` +/// + `NULL == phPrograms` +/// + `NULL == phProgram` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If one of the programs in `phPrograms` isn't a valid program +/// object. +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `count == 0` +/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE +/// + If an error occurred while linking `phPrograms`. +UR_APIEXPORT ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for USM Context Memcpy +#if !defined(__GNUC__) +#pragma region usm_context_memcpy_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory read command to a command-buffer object. +/// @brief Perform a synchronous, blocking memcpy operation between USM +/// allocations. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11850,72 +11671,38 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER /// + `NULL == pDst` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. +/// + `NULL == pSrc` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pSrc` or `pDst` +UR_APIEXPORT ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' USM Import/Release Extension APIs +#if !defined(__GNUC__) +#pragma region usm_import_release_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory fill command to a command-buffer object. +/// @brief Import memory into USM +/// +/// @details +/// - Import memory into USM /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11923,69 +11710,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pPattern` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + If `offset + size` results in an out-of-bounds access. -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +UR_APIEXPORT ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size); /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM Prefetch command to a command-buffer object. +/// @brief Release memory from USM /// /// @details -/// - Prefetching may not be supported for all devices or allocation types. -/// If memory prefetching is not supported, the prefetch hint will be -/// ignored. +/// - Release memory from USM /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -11993,67 +11735,62 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_USM_MIGRATION_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pMemory` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +UR_APIEXPORT ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem); +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for USM P2P +#if !defined(__GNUC__) +#pragma region usm_p2p_(experimental) +#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM Advise command to a command-buffer object. +/// @brief Supported peer info +typedef enum ur_exp_peer_info_t { + /// [int] 1 if P2P access is supported otherwise P2P access is not + /// supported. + UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT = 0, + /// [int] 1 if atomic operations are supported over the P2P link, + /// otherwise such operations are not supported. + UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT = 1, + /// @cond + UR_EXP_PEER_INFO_FORCE_UINT32 = 0x7fffffff + /// @endcond + +} ur_exp_peer_info_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Enable access to peer device memory /// /// @details -/// - Not all memory advice hints may be supported for all devices or -/// allocation types. If a memory advice hint is not supported, it will be -/// ignored. +/// - Enables the command device to access and write device memory +/// allocations located on the peer device, provided that a P2P link +/// between the two devices is available. +/// - When Peer Access is successfully enabled, P2P memory accesses are +/// guaranteed to be allowed on the peer device until +/// ::urUsmP2PDisablePeerAccessExp is called. +/// - Note that the function operands may, but aren't guaranteed to, commute +/// for a given adapter: the peer device is not guaranteed to have access +/// to device memory allocations located on the command device. +/// - It is not guaranteed that the commutation relations of the function +/// arguments are identical for peer access and peer copies: For example, +/// for a given adapter the peer device may be able to copy data from the +/// command device, but not access and write the same data on the command +/// device. +/// - Consult the appropriate adapter driver documentation for details of +/// adapter specific behavior and native error codes that may be returned. +/// +/// @remarks +/// _Analogues_ +/// - **cuCtxEnablePeerAccess** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12061,111 +11798,41 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_USM_ADVICE_FLAGS_MASK & advice` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pMemory` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand); - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Function adding work through the native API to be executed -/// immediately. -typedef void (*ur_exp_command_buffer_native_command_function_t)( - /// [in][out] Pointer to data to be passed to callback - void *pUserData); - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Append nodes to the command-buffer through a native backend API -/// -/// @returns +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` /// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pfnNativeCommand` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint); +UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice); /////////////////////////////////////////////////////////////////////////////// -/// @brief Submit a command-buffer for execution on a queue. +/// @brief Disable access to peer device memory +/// +/// @details +/// - Disables the ability of the command device to access and write device +/// memory allocations located on the peer device, provided that a P2P +/// link between the two devices was enabled prior to the call. +/// - Note that the function operands may, but aren't guaranteed to, commute +/// for a given adapter. If, prior to the function call, the peer device +/// had access to device memory allocations on the command device, it is +/// not guaranteed to still have such access following the function +/// return. +/// - It is not guaranteed that the commutation relations of the function +/// arguments are identical for peer access and peer copies: For example +/// for a given adapter, if, prior to the call, the peer device had access +/// to device memory allocations on the command device, the peer device +/// may still, following the function call, be able to copy data from the +/// command device, but not access and write the same data on the command +/// device. +/// - Consult the appropriate adapter driver documentation for details of +/// adapter specific behavior and native error codes that may be returned. +/// +/// @remarks +/// _Analogues_ +/// - **cuCtxDisablePeerAccess** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12173,42 +11840,26 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_QUEUE -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent); +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice); /////////////////////////////////////////////////////////////////////////////// -/// @brief Update a kernel launch command in a finalized command-buffer. +/// @brief Disable access to peer device memory /// /// @details -/// This entry-point is synchronous and may block if the command-buffer is -/// executing when the entry-point is called. On error, the state of the -/// command-buffer commands being updated is undefined. +/// - Queries the peer access capabilities from the command device to the +/// peer device according to the query `propName`. +/// +/// @remarks +/// _Analogues_ +/// - **cuDeviceGetP2PAttribute** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12216,85 +11867,264 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueCommandBufferExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == pUpdateKernelLaunch->hCommand` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pUpdateKernelLaunch` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `numKernelUpdates == 0` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS -/// is not supported by the device, and for any of any element of -/// `pUpdateKernelLaunch` the `numNewMemObjArgs`, `numNewPointerArgs`, -/// or `numNewValueArgs` members are not zero. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is -/// not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is not nullptr. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is -/// not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is nullptr and -/// `pNewGlobalWorkSize` is not nullptr. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewGlobalWorkSize` member is not nullptr -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewGlobalWorkOffset` member is not -/// nullptr. -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `hNewKernel` member is not nullptr. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the `hCommandBuffer`. -/// + If `hCommandBuffer` has not been finalized. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member -/// is not a kernel execution command. -/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member -/// was not created from `hCommandBuffer`. -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` -/// member is less than 1 or greater than 3. -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + If for any element of `pUpdateKernelLaunch` the `hNewKernel` -/// member was not passed to the `hKernel` or `phKernelAlternatives` -/// parameters of ::urCommandBufferAppendKernelLaunchExp when the -/// command was created. -/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` -/// member is different from the current workDim in the `hCommand` -/// member, and `pNewGlobalWorkSize` or `pNewGlobalWorkOffset` are -/// nullptr. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch); +/// + `::UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION +/// + If `propName` is not supported by the adapter. +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `propSize == 0 && pPropValue != NULL` +/// + If `propSize` is less than the real number of bytes needed to +/// return the info. +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `propSize != 0 && pPropValue == NULL` +/// + `pPropValue == NULL && pPropSizeRet == NULL` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. + size_t propSize, + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. + void *pPropValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. + size_t *pPropSizeRet); + +#if !defined(__GNUC__) +#pragma endregion +#endif +// Intel 'oneAPI' Unified Runtime Experimental APIs for Command-Buffers +#if !defined(__GNUC__) +#pragma region command_buffer_(experimental) +#endif +/////////////////////////////////////////////////////////////////////////////// +/// @brief Device kernel execution capability +typedef uint32_t ur_device_command_buffer_update_capability_flags_t; +typedef enum ur_device_command_buffer_update_capability_flag_t { + /// Device supports updating the kernel arguments in command-buffer + /// commands. + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS = UR_BIT(0), + /// Device supports updating the local work-group size in command-buffer + /// commands. + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE = UR_BIT(1), + /// Device supports updating the global work-group size in command-buffer + /// commands. + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE = UR_BIT(2), + /// Device supports updating the global work offset in command-buffer + /// commands. + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET = + UR_BIT(3), + /// Device supports updating the kernel handle in command-buffer commands. + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE = UR_BIT(4), + /// Device supports updating the event parameters in command-buffer + /// commands. + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS = UR_BIT(5), + /// @cond + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_FORCE_UINT32 = 0x7fffffff + /// @endcond + +} ur_device_command_buffer_update_capability_flag_t; +/// @brief Bit Mask for validating +/// ur_device_command_buffer_update_capability_flags_t +#define UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAGS_MASK 0xffffffc0 + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Command-buffer query information type +typedef enum ur_exp_command_buffer_info_t { + /// [uint32_t] Reference count of the command-buffer object. + /// The reference count returned should be considered immediately stale. + /// It is unsuitable for general use in applications. This feature is + /// provided for identifying memory leaks. + UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT = 0, + /// [::ur_exp_command_buffer_desc_t] Returns a ::ur_exp_command_buffer_desc_t + /// with the properties of the command-buffer. Returned values may differ + /// from those passed on construction if the property was ignored by the + /// adapter. + UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR = 1, + /// @cond + UR_EXP_COMMAND_BUFFER_INFO_FORCE_UINT32 = 0x7fffffff + /// @endcond + +} ur_exp_command_buffer_info_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Command-buffer command query information type +typedef enum ur_exp_command_buffer_command_info_t { + /// [uint32_t] Reference count of the command-buffer object. + /// The reference count returned should be considered immediately stale. + /// It is unsuitable for general use in applications. This feature is + /// provided for identifying memory leaks. + UR_EXP_COMMAND_BUFFER_COMMAND_INFO_REFERENCE_COUNT = 0, + /// @cond + UR_EXP_COMMAND_BUFFER_COMMAND_INFO_FORCE_UINT32 = 0x7fffffff + /// @endcond + +} ur_exp_command_buffer_command_info_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Command-Buffer Descriptor Type +typedef struct ur_exp_command_buffer_desc_t { + /// [in] type of this structure, must be + /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC + ur_structure_type_t stype; + /// [in][optional] pointer to extension-specific structure + const void *pNext; + /// [in] Commands in a finalized command-buffer can be updated. + ur_bool_t isUpdatable; + /// [in] Commands in a command-buffer will execute in-order, explicit + /// sync-point dependencies are ignored. + ur_bool_t isInOrder; + /// [in] Command-buffer profiling is enabled. + ur_bool_t enableProfiling; + +} ur_exp_command_buffer_desc_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief A value that identifies a command inside of a command-buffer, used +/// for +/// defining dependencies between commands in the same command-buffer. +typedef uint32_t ur_exp_command_buffer_sync_point_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Handle of Command-Buffer object +typedef struct ur_exp_command_buffer_handle_t_ *ur_exp_command_buffer_handle_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Handle of a Command-Buffer command +typedef struct ur_exp_command_buffer_command_handle_t_ + *ur_exp_command_buffer_command_handle_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Descriptor type for updating a kernel command memobj argument. +typedef struct ur_exp_command_buffer_update_memobj_arg_desc_t { + /// [in] type of this structure, must be + /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_MEMOBJ_ARG_DESC + ur_structure_type_t stype; + /// [in][optional] pointer to extension-specific structure + const void *pNext; + /// [in] Argument index. + uint32_t argIndex; + /// [in][optional] Pointer to memory object properties. + const ur_kernel_arg_mem_obj_properties_t *pProperties; + /// [in][optional] Handle of memory object to set at argument index. + ur_mem_handle_t hNewMemObjArg; + +} ur_exp_command_buffer_update_memobj_arg_desc_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Descriptor type for updating a kernel command pointer argument. +typedef struct ur_exp_command_buffer_update_pointer_arg_desc_t { + /// [in] type of this structure, must be + /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_POINTER_ARG_DESC + ur_structure_type_t stype; + /// [in][optional] pointer to extension-specific structure + const void *pNext; + /// [in] Argument index. + uint32_t argIndex; + /// [in][optional] Pointer to USM pointer properties. + const ur_kernel_arg_pointer_properties_t *pProperties; + /// [in][optional] USM pointer to memory location holding the argument + /// value to set at argument index. + const void *pNewPointerArg; + +} ur_exp_command_buffer_update_pointer_arg_desc_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Descriptor type for updating a kernel command value argument. +typedef struct ur_exp_command_buffer_update_value_arg_desc_t { + /// [in] type of this structure, must be + /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_VALUE_ARG_DESC + ur_structure_type_t stype; + /// [in][optional] pointer to extension-specific structure + const void *pNext; + /// [in] Argument index. + uint32_t argIndex; + /// [in] Argument size. + size_t argSize; + /// [in][optional] Pointer to value properties. + const ur_kernel_arg_value_properties_t *pProperties; + /// [in][optional] Argument value representing matching kernel arg type to + /// set at argument index. + const void *pNewValueArg; + +} ur_exp_command_buffer_update_value_arg_desc_t; + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Descriptor type for updating a kernel launch command. +typedef struct ur_exp_command_buffer_update_kernel_launch_desc_t { + /// [in] type of this structure, must be + /// ::UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_DESC + ur_structure_type_t stype; + /// [in][optional] pointer to extension-specific structure + const void *pNext; + /// [in] Handle of the command-buffer kernel command to update. + ur_exp_command_buffer_command_handle_t hCommand; + /// [in][optional] The new kernel handle. If this parameter is nullptr, + /// the current kernel handle in `hCommand` + /// will be used. If a kernel handle is passed, it must be a valid kernel + /// alternative as defined in + /// ::urCommandBufferAppendKernelLaunchExp. + ur_kernel_handle_t hNewKernel; + /// [in] Length of pNewMemObjArgList. + uint32_t numNewMemObjArgs; + /// [in] Length of pNewPointerArgList. + uint32_t numNewPointerArgs; + /// [in] Length of pNewValueArgList. + uint32_t numNewValueArgs; + /// [in] Number of work dimensions in the kernel ND-range, from 1-3. + uint32_t newWorkDim; + /// [in][optional][range(0, numNewMemObjArgs)] An array describing the new + /// kernel mem obj arguments for the command. + const ur_exp_command_buffer_update_memobj_arg_desc_t *pNewMemObjArgList; + /// [in][optional][range(0, numNewPointerArgs)] An array describing the + /// new kernel pointer arguments for the command. + const ur_exp_command_buffer_update_pointer_arg_desc_t *pNewPointerArgList; + /// [in][optional][range(0, numNewValueArgs)] An array describing the new + /// kernel value arguments for the command. + const ur_exp_command_buffer_update_value_arg_desc_t *pNewValueArgList; + /// [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned + /// values that describe the offset used + /// to calculate the global ID. If this parameter is nullptr, the current + /// global work offset will be used. This parameter is required if + /// `newWorkDim` is different from the current work dimensions + /// in the command. + size_t *pNewGlobalWorkOffset; + /// [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned + /// values that describe the number of + /// global work-items. If this parameter is nullptr, the current global + /// work size in `hCommand` will be used. + /// This parameter is required if `newWorkDim` is different from the + /// current work dimensions in the command. + size_t *pNewGlobalWorkSize; + /// [in][optional][range(0, newWorkDim)] Array of newWorkDim unsigned + /// values that describe the number of + /// work-items that make up a work-group. If `pNewGlobalWorkSize` is set + /// and `pNewLocalWorkSize` is nullptr, + /// then the runtime implementation will choose the local work size. If + /// `pNewGlobalWorkSize` is nullptr and + /// `pNewLocalWorkSize` is nullptr, the current local work size in the + /// command will be used. + size_t *pNewLocalWorkSize; + +} ur_exp_command_buffer_update_kernel_launch_desc_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Get a new event that will be signaled the next time the command in -/// the -/// command-buffer executes. +/// @brief Create a Command-Buffer object /// /// @details -/// It is the users responsibility to release the returned `phSignalEvent`. +/// - Create a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12302,32 +12132,31 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommand` +/// + `NULL == hContext` +/// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phSignalEvent` +/// + `NULL == pCommandBufferDesc` +/// + `NULL == phCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE /// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not -/// supported by the device associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the command-buffer `hCommand` belongs to. -/// + If the command-buffer `hCommand` belongs to has not been -/// finalized. -/// + If no `phEvent` parameter was set on creation of the command -/// associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns +/// 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP +/// query. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent); +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer); /////////////////////////////////////////////////////////////////////////////// -/// @brief Set the list of wait events for a command to depend on to a list of -/// new events. +/// @brief Increment the command-buffer object's reference count. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12335,36 +12164,20 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommand` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not -/// supported by the device associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the command-buffer `hCommand` belongs to. -/// + If the command-buffer `hCommand` belongs to has not been -/// finalized. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// + If `numEventsInWaitList` does not match the number of wait events -/// set when the command associated with `hCommand` was created. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList); +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer); /////////////////////////////////////////////////////////////////////////////// -/// @brief Get command-buffer object information. +/// @brief Decrement the command-buffer object's reference count and delete the +/// command-buffer object if the reference count becomes zero. It is +/// legal to call the entry-point while `hCommandBuffer` is still +/// executing, which will block on completion if the reference count of +/// `hCommandBuffer` becomes zero. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12373,39 +12186,16 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION -/// + If `propName` is not supported by the adapter. -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `propSize == 0 && pPropValue != NULL` -/// + If `propSize` is less than the real number of bytes needed to -/// return the info. -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `propSize != 0 && pPropValue == NULL` -/// + `pPropValue == NULL && pPropSizeRet == NULL` /// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value - size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property - void *pPropValue, - /// [out][optional] bytes returned in command-buffer property - size_t *pPropSizeRet); +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer); /////////////////////////////////////////////////////////////////////////////// -/// @brief Return platform native command-buffer handle. -/// -/// @details -/// - Retrieved native handle can be used for direct interaction with the -/// native platform driver. +/// @brief Stop recording on a command-buffer object such that no more commands +/// can be appended and make it ready to enqueue. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12414,26 +12204,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetInfoExp( /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phNativeCommandBuffer` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the adapter has no underlying equivalent handle. -UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer); +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "If `hCommandBuffer` has already +/// been finalized" +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferFinalizeExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for device-wide -// synchronization -#if !defined(__GNUC__) -#pragma region device_wait_(experimental) -#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Synchronizes with all queues on the device. +/// @brief Append a kernel execution command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12441,29 +12222,82 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hDevice` -/// + `hDevice == nullptr` -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -UR_APIEXPORT ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice); +/// + `NULL == hCommandBuffer` +/// + `NULL == hKernel` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pGlobalWorkSize` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_KERNEL +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + `phKernelAlternatives == NULL && numKernelAlternatives > 0` +/// + `phKernelAlternatives != NULL && numKernelAlternatives == 0` +/// + If `phKernelAlternatives` contains `hKernel` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "phCommand is not NULL and +/// hCommandBuffer is not updatable." +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for dynamic linking -#if !defined(__GNUC__) -#pragma region dynamic_link_(experimental) -#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Creates dynamic links between exported and imported symbols in one or -/// more programs. -/// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point the programs in -/// `phPrograms` will have all external symbols resolved and kernels -/// inside these programs would be ready for use. +/// @brief Append a USM memcpy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12471,36 +12305,61 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceWaitExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phPrograms` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If one of the programs in `phPrograms` isn't a valid program -/// object. +/// + `NULL == pDst` +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `count == 0` -/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE -/// + If an error occurred while linking `phPrograms`. +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pSrc` or `pDst` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. /// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the adapter has no means to support the operation. -UR_APIEXPORT ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms); +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for enqueuing timestamp -// recordings -#if !defined(__GNUC__) -#pragma region enqueue_timestamp_recording_(experimental) -#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Enqueue a command for recording the device timestamp +/// @brief Append a USM fill command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12508,49 +12367,65 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramDynamicLinkExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phEvent` +/// + `NULL == pMemory` +/// + `NULL == pPattern` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `patternSize == 0 || size == 0` +/// + `patternSize > size` +/// + `size % patternSize != 0` +/// + If `size` is higher than the allocation size of `ptr` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT /// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST /// + `phEventWaitList == NULL && numEventsInWaitList > 0` /// + `phEventWaitList != NULL && numEventsInWaitList == 0` -UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent); + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for Inter Process -// Communication -#if !defined(__GNUC__) -#pragma region inter_process_communication_(experimental) -#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Gets an inter-process memory handle for a pointer to device USM -/// memory +/// @brief Append a memory copy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12558,25 +12433,61 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == ppIPCMemHandleData` -/// + `NULL == pIPCMemHandleDataSizeRet` +/// + `NULL == hCommandBuffer` +/// + `NULL == hSrcMem` +/// + `NULL == hDstMem` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet); +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Releases an inter-process memory handle +/// @brief Append a memory write command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12584,21 +12495,60 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCGetMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pIPCMemHandleData` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData); +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Opens an inter-process memory handle to get the corresponding pointer -/// to device USM memory +/// @brief Append a memory read command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12606,32 +12556,60 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCPutMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pIPCMemHandleData` -/// + `NULL == ppMem` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + ipcMemHandleDataSize is not the same as the size of IPC memory -/// handle data +/// + `NULL == pDst` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem); +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Closes an inter-process memory handle +/// @brief Append a rectangular memory copy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12639,73 +12617,69 @@ UR_APIEXPORT ur_result_t UR_APICALL urIPCOpenMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` +/// + `NULL == hCommandBuffer` +/// + `NULL == hSrcMem` +/// + `NULL == hDstMem` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -UR_APIEXPORT ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem); - -#if !defined(__GNUC__) -#pragma endregion -#endif -// Memory Export Extension APIs -#if !defined(__GNUC__) -#pragma region memory_export_(experimental) -#endif -/////////////////////////////////////////////////////////////////////////////// -/// @brief Allocate an exportable memory region and return a pointer to that -/// allocation. -/// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < -/// handleTypeToExport` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == ppMem` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT -/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + If `alignment` exceeds largest supported data type by `hDevice` -/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE -/// + `size == 0` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE -/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - -/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. - size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem); +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Free an exportable memory allocation. +/// @brief Append a rectangular memory write command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12713,30 +12687,73 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem); +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Obtain an exportable handle to a memory allocated with -/// `AllocExportableMemoryExp`.The returned external memory type will be -/// that which was specified upon -/// allocation of the exportable memory (e.g. `opaque_fd` or -/// `win32_nt_handle`). +/// @brief Append a rectangular memory read command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12744,66 +12761,72 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < -/// handleTypeToExport` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemHandleRet || NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// + `NULL == pDst` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -UR_APIEXPORT ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet); - -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for multi-device compile -#if !defined(__GNUC__) -#pragma region multi_device_compile_(experimental) -#endif -/////////////////////////////////////////////////////////////////////////////// -/// @brief Program operation behavior control flags -typedef uint32_t ur_exp_program_flags_t; -typedef enum ur_exp_program_flag_t { - /// Allow unresolved symbols in the program resulting from the - /// corresponding operation - UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS = UR_BIT(0), - /// @cond - UR_EXP_PROGRAM_FLAG_FORCE_UINT32 = 0x7fffffff - /// @endcond - -} ur_exp_program_flag_t; -/// @brief Bit Mask for validating ur_exp_program_flags_t -#define UR_EXP_PROGRAM_FLAGS_MASK 0xfffffffe +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one program, negates need for the -/// linking step. -/// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point, the program passed -/// will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type -/// for each device in `phDevices`. -/// -/// @remarks -/// _Analogues_ -/// - **clBuildProgram** +/// @brief Append a memory fill command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12811,39 +12834,69 @@ typedef enum ur_exp_program_flag_t { /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hProgram` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If `hProgram` isn't a valid program object. -/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE -/// + If an error occurred when building `hProgram`. -UR_APIEXPORT ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions); +/// + `NULL == pPattern` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + If `offset + size` results in an out-of-bounds access. +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one or more programs. +/// @brief Append a USM Prefetch command to a command-buffer object. /// /// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point `hProgram` will -/// contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type -/// for each device in `phDevices`. -/// -/// @remarks -/// _Analogues_ -/// - **clCompileProgram** +/// - Prefetching may not be supported for all devices or allocation types. +/// If memory prefetching is not supported, the prefetch hint will be +/// ignored. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12851,45 +12904,67 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramBuildExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hProgram` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` +/// + `NULL == pMemory` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If `hProgram` isn't a valid program object. -/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE -/// + If an error occurred while compiling `hProgram`. -UR_APIEXPORT ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions); +/// + `::UR_USM_MIGRATION_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pMemory` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one or more programs. +/// @brief Append a USM Advise command to a command-buffer object. /// /// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point the program returned -/// in `phProgram` will contain a binary of the -/// ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in -/// `phDevices`. -/// - If a non-success code is returned, adapters may store a program in -/// `phProgram`. This program should only be used with -/// `::urProgramGetBuildInfo` to get the build log for the failure. -/// Adapters which do not do not support producing build logs must set -/// this value to `nullptr`. -/// -/// @remarks -/// _Analogues_ -/// - **clLinkProgram** +/// - Not all memory advice hints may be supported for all devices or +/// allocation types. If a memory advice hint is not supported, it will be +/// ignored. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12897,48 +12972,69 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCompileExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` -/// + `NULL == phPrograms` -/// + `NULL == phProgram` +/// + `NULL == pMemory` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If one of the programs in `phPrograms` isn't a valid program -/// object. +/// + `::UR_USM_ADVICE_FLAGS_MASK & advice` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `count == 0` -/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE -/// + If an error occurred while linking `phPrograms`. -UR_APIEXPORT ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram); +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pMemory` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for USM Context Memcpy -#if !defined(__GNUC__) -#pragma region usm_context_memcpy_(experimental) -#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Perform a synchronous, blocking memcpy operation between USM -/// allocations. +/// @brief Function adding work through the native API to be executed +/// immediately. +typedef void (*ur_exp_command_buffer_native_command_function_t)( + /// [in][out] Pointer to data to be passed to callback + void *pUserData); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Append nodes to the command-buffer through a native backend API /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12946,38 +13042,41 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramLinkExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// + `NULL == pSrc` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pSrc` or `pDst` -UR_APIEXPORT ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size); +/// + `NULL == pfnNativeCommand` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' USM Import/Release Extension APIs -#if !defined(__GNUC__) -#pragma region usm_import_release_(experimental) -#endif /////////////////////////////////////////////////////////////////////////////// -/// @brief Import memory into USM -/// -/// @details -/// - Import memory into USM +/// @brief Submit a command-buffer for execution on a queue. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -12985,24 +13084,42 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMContextMemcpyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -UR_APIEXPORT ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size); +/// + `NULL == hQueue` +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_QUEUE +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent); /////////////////////////////////////////////////////////////////////////////// -/// @brief Release memory from USM +/// @brief Update a kernel launch command in a finalized command-buffer. /// /// @details -/// - Release memory from USM +/// This entry-point is synchronous and may block if the command-buffer is +/// executing when the entry-point is called. On error, the state of the +/// command-buffer commands being updated is undefined. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -13010,62 +13127,85 @@ UR_APIEXPORT ur_result_t UR_APICALL urUSMImportExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` +/// + `NULL == pUpdateKernelLaunch->hCommand` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -UR_APIEXPORT ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem); - -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental APIs for USM P2P -#if !defined(__GNUC__) -#pragma region usm_p2p_(experimental) -#endif -/////////////////////////////////////////////////////////////////////////////// -/// @brief Supported peer info -typedef enum ur_exp_peer_info_t { - /// [int] 1 if P2P access is supported otherwise P2P access is not - /// supported. - UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT = 0, - /// [int] 1 if atomic operations are supported over the P2P link, - /// otherwise such operations are not supported. - UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT = 1, - /// @cond - UR_EXP_PEER_INFO_FORCE_UINT32 = 0x7fffffff - /// @endcond - -} ur_exp_peer_info_t; +/// + `NULL == pUpdateKernelLaunch` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `numKernelUpdates == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS +/// is not supported by the device, and for any of any element of +/// `pUpdateKernelLaunch` the `numNewMemObjArgs`, `numNewPointerArgs`, +/// or `numNewValueArgs` members are not zero. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is +/// not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is not nullptr. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is +/// not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is nullptr and +/// `pNewGlobalWorkSize` is not nullptr. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewGlobalWorkSize` member is not nullptr +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewGlobalWorkOffset` member is not +/// nullptr. +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `hNewKernel` member is not nullptr. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the `hCommandBuffer`. +/// + If `hCommandBuffer` has not been finalized. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member +/// is not a kernel execution command. +/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member +/// was not created from `hCommandBuffer`. +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` +/// member is less than 1 or greater than 3. +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If for any element of `pUpdateKernelLaunch` the `hNewKernel` +/// member was not passed to the `hKernel` or `phKernelAlternatives` +/// parameters of ::urCommandBufferAppendKernelLaunchExp when the +/// command was created. +/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` +/// member is different from the current workDim in the `hCommand` +/// member, and `pNewGlobalWorkSize` or `pNewGlobalWorkOffset` are +/// nullptr. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch); /////////////////////////////////////////////////////////////////////////////// -/// @brief Enable access to peer device memory +/// @brief Get a new event that will be signaled the next time the command in +/// the +/// command-buffer executes. /// /// @details -/// - Enables the command device to access and write device memory -/// allocations located on the peer device, provided that a P2P link -/// between the two devices is available. -/// - When Peer Access is successfully enabled, P2P memory accesses are -/// guaranteed to be allowed on the peer device until -/// ::urUsmP2PDisablePeerAccessExp is called. -/// - Note that the function operands may, but aren't guaranteed to, commute -/// for a given adapter: the peer device is not guaranteed to have access -/// to device memory allocations located on the command device. -/// - It is not guaranteed that the commutation relations of the function -/// arguments are identical for peer access and peer copies: For example, -/// for a given adapter the peer device may be able to copy data from the -/// command device, but not access and write the same data on the command -/// device. -/// - Consult the appropriate adapter driver documentation for details of -/// adapter specific behavior and native error codes that may be returned. -/// -/// @remarks -/// _Analogues_ -/// - **cuCtxEnablePeerAccess** +/// It is the users responsibility to release the returned `phSignalEvent`. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -13073,68 +13213,69 @@ typedef enum ur_exp_peer_info_t { /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice); +/// + `NULL == hCommand` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phSignalEvent` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not +/// supported by the device associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the command-buffer `hCommand` belongs to. +/// + If the command-buffer `hCommand` belongs to has not been +/// finalized. +/// + If no `phEvent` parameter was set on creation of the command +/// associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent); /////////////////////////////////////////////////////////////////////////////// -/// @brief Disable access to peer device memory -/// -/// @details -/// - Disables the ability of the command device to access and write device -/// memory allocations located on the peer device, provided that a P2P -/// link between the two devices was enabled prior to the call. -/// - Note that the function operands may, but aren't guaranteed to, commute -/// for a given adapter. If, prior to the function call, the peer device -/// had access to device memory allocations on the command device, it is -/// not guaranteed to still have such access following the function -/// return. -/// - It is not guaranteed that the commutation relations of the function -/// arguments are identical for peer access and peer copies: For example -/// for a given adapter, if, prior to the call, the peer device had access -/// to device memory allocations on the command device, the peer device -/// may still, following the function call, be able to copy data from the -/// command device, but not access and write the same data on the command -/// device. -/// - Consult the appropriate adapter driver documentation for details of -/// adapter specific behavior and native error codes that may be returned. -/// -/// @remarks -/// _Analogues_ -/// - **cuCtxDisablePeerAccess** +/// @brief Set the list of wait events for a command to depend on to a list of +/// new events. /// /// @returns /// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` -/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice); +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hCommand` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not +/// supported by the device associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the command-buffer `hCommand` belongs to. +/// + If the command-buffer `hCommand` belongs to has not been +/// finalized. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// + If `numEventsInWaitList` does not match the number of wait events +/// set when the command associated with `hCommand` was created. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList); /////////////////////////////////////////////////////////////////////////////// -/// @brief Disable access to peer device memory -/// -/// @details -/// - Queries the peer access capabilities from the command device to the -/// peer device according to the query `propName`. -/// -/// @remarks -/// _Analogues_ -/// - **cuDeviceGetP2PAttribute** +/// @brief Get command-buffer object information. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -13142,10 +13283,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName` +/// + `::UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -13155,108 +13295,28 @@ UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER /// + `propSize != 0 && pPropValue == NULL` /// + `pPropValue == NULL && pPropSizeRet == NULL` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -UR_APIEXPORT ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet); -#if !defined(__GNUC__) -#pragma endregion -#endif -// Intel 'oneAPI' Unified Runtime Experimental API for setting args at kernel -// launch -#if !defined(__GNUC__) -#pragma region enqueue_kernel_launch_with_args_(experimental) -#endif -/////////////////////////////////////////////////////////////////////////////// -/// @brief What kind of kernel arg is this -typedef enum ur_exp_kernel_arg_type_t { - /// Kernel arg is a value. - UR_EXP_KERNEL_ARG_TYPE_VALUE = 0, - /// Kernel arg is a pointer. - UR_EXP_KERNEL_ARG_TYPE_POINTER = 1, - /// Kernel arg is a memory object. - UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ = 2, - /// Kernel arg is a local allocation. - UR_EXP_KERNEL_ARG_TYPE_LOCAL = 3, - /// Kernel arg is a sampler. - UR_EXP_KERNEL_ARG_TYPE_SAMPLER = 4, - /// @cond - UR_EXP_KERNEL_ARG_TYPE_FORCE_UINT32 = 0x7fffffff - /// @endcond - -} ur_exp_kernel_arg_type_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Mem obj/properties tuple -typedef struct ur_exp_kernel_arg_mem_obj_tuple_t { - /// [in] Handle of a memory object - ur_mem_handle_t hMem; - /// [in] Memory flags to associate with `hMem`. Allowed values are: - /// ::UR_MEM_FLAG_READ_WRITE, ::UR_MEM_FLAG_WRITE_ONLY, - /// ::UR_MEM_FLAG_READ_ONLY. - ur_mem_flags_t flags; - -} ur_exp_kernel_arg_mem_obj_tuple_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Typesafe container for a kernel argument value -typedef union ur_exp_kernel_arg_value_t { - /// [in] argument value represented as matching arg type. - /// The data pointed to will be copied and therefore can be reused on return. - const void *value; - /// [in] Allocation obtained by USM allocation or virtual memory mapping - /// operation, or pointer to a literal value. - const void *pointer; - /// [in] Struct containing a memory object and associated flags. - ur_exp_kernel_arg_mem_obj_tuple_t memObjTuple; - /// [in] Handle of a sampler object. - ur_sampler_handle_t sampler; - -} ur_exp_kernel_arg_value_t; - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Kernel arg properties -typedef struct ur_exp_kernel_arg_properties_t { - /// [in] type of this structure, must be - /// ::UR_STRUCTURE_TYPE_EXP_KERNEL_ARG_PROPERTIES - ur_structure_type_t stype; - /// [in,out][optional] pointer to extension-specific structure - void *pNext; - /// [in] type of the kernel arg - ur_exp_kernel_arg_type_t type; - /// [in] index of the kernel arg - uint32_t index; - /// [in] size of the kernel arg - size_t size; - /// [in][tagged_by(type)] Union containing the argument value. - ur_exp_kernel_arg_value_t value; - -} ur_exp_kernel_arg_properties_t; - /////////////////////////////////////////////////////////////////////////////// -/// @brief Enqueue a command to execute a kernel +/// @brief Return platform native command-buffer handle. /// -/// @remarks -/// _Analogues_ -/// - **clEnqueueNDRangeKernel** +/// @details +/// - Retrieved native handle can be used for direct interaction with the +/// native platform driver. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -13264,76 +13324,16 @@ typedef struct ur_exp_kernel_arg_properties_t { /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` -/// + `NULL == hKernel` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pGlobalWorkSize` -/// + `pArgs == NULL && numArgs > 0` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `NULL != pArgs && ::UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type` -/// + `NULL != launchPropList && ::UR_KERNEL_LAUNCH_FLAGS_MASK & -/// launchPropList->flags` -/// - ::UR_RESULT_ERROR_INVALID_QUEUE -/// - ::UR_RESULT_ERROR_INVALID_KERNEL -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_IN_EVENT_LIST_EXEC_STATUS -/// + An event in `phEventWaitList` has ::UR_EVENT_STATUS_ERROR. -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// + `pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || -/// pGlobalWorkSize[2] == 0` -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// + `pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == -/// 0 || pLocalWorkSize[2] == 0)` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGS - "The kernel argument values -/// have not been specified." -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If any property in `launchPropList` isn't supported by the device. -UR_APIEXPORT ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent); +/// + `NULL == phNativeCommandBuffer` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the adapter has no underlying equivalent handle. +UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer); #if !defined(__GNUC__) #pragma endregion @@ -15382,28 +15382,28 @@ typedef struct ur_enqueue_usm_free_exp_params_t { } ur_enqueue_usm_free_exp_params_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Function parameters for urEnqueueCommandBufferExp +/// @brief Function parameters for urEnqueueTimestampRecordingExp /// @details Each entry is a pointer to the parameter passed to the function; /// allowing the callback the ability to modify the parameter's value -typedef struct ur_enqueue_command_buffer_exp_params_t { +typedef struct ur_enqueue_timestamp_recording_exp_params_t { ur_queue_handle_t *phQueue; - ur_exp_command_buffer_handle_t *phCommandBuffer; + bool *pblocking; uint32_t *pnumEventsInWaitList; const ur_event_handle_t **pphEventWaitList; ur_event_handle_t **pphEvent; -} ur_enqueue_command_buffer_exp_params_t; +} ur_enqueue_timestamp_recording_exp_params_t; /////////////////////////////////////////////////////////////////////////////// -/// @brief Function parameters for urEnqueueTimestampRecordingExp +/// @brief Function parameters for urEnqueueCommandBufferExp /// @details Each entry is a pointer to the parameter passed to the function; /// allowing the callback the ability to modify the parameter's value -typedef struct ur_enqueue_timestamp_recording_exp_params_t { +typedef struct ur_enqueue_command_buffer_exp_params_t { ur_queue_handle_t *phQueue; - bool *pblocking; + ur_exp_command_buffer_handle_t *phCommandBuffer; uint32_t *pnumEventsInWaitList; const ur_event_handle_t **pphEventWaitList; ur_event_handle_t **pphEvent; -} ur_enqueue_timestamp_recording_exp_params_t; +} ur_enqueue_command_buffer_exp_params_t; /////////////////////////////////////////////////////////////////////////////// /// @brief Function parameters for urEnqueueHostTaskExp diff --git a/unified-runtime/include/ur_api_funcs.def b/unified-runtime/include/ur_api_funcs.def index 0c7fd185cb0f7..9c308992a2803 100644 --- a/unified-runtime/include/ur_api_funcs.def +++ b/unified-runtime/include/ur_api_funcs.def @@ -143,8 +143,8 @@ _UR_API(urEnqueueUSMDeviceAllocExp) _UR_API(urEnqueueUSMSharedAllocExp) _UR_API(urEnqueueUSMHostAllocExp) _UR_API(urEnqueueUSMFreeExp) -_UR_API(urEnqueueCommandBufferExp) _UR_API(urEnqueueTimestampRecordingExp) +_UR_API(urEnqueueCommandBufferExp) _UR_API(urEnqueueHostTaskExp) _UR_API(urEnqueueNativeCommandExp) _UR_API(urEnqueueGraphExp) diff --git a/unified-runtime/include/ur_ddi.h b/unified-runtime/include/ur_ddi.h index 667b0600db0db..3b0b054d3ba55 100644 --- a/unified-runtime/include/ur_ddi.h +++ b/unified-runtime/include/ur_ddi.h @@ -1187,18 +1187,18 @@ typedef ur_result_t(UR_APICALL *ur_pfnEnqueueUSMFreeExp_t)( ur_queue_handle_t, ur_usm_pool_handle_t, void *, uint32_t, const ur_event_handle_t *, ur_event_handle_t *); -/////////////////////////////////////////////////////////////////////////////// -/// @brief Function-pointer for urEnqueueCommandBufferExp -typedef ur_result_t(UR_APICALL *ur_pfnEnqueueCommandBufferExp_t)( - ur_queue_handle_t, ur_exp_command_buffer_handle_t, uint32_t, - const ur_event_handle_t *, ur_event_handle_t *); - /////////////////////////////////////////////////////////////////////////////// /// @brief Function-pointer for urEnqueueTimestampRecordingExp typedef ur_result_t(UR_APICALL *ur_pfnEnqueueTimestampRecordingExp_t)( ur_queue_handle_t, bool, uint32_t, const ur_event_handle_t *, ur_event_handle_t *); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Function-pointer for urEnqueueCommandBufferExp +typedef ur_result_t(UR_APICALL *ur_pfnEnqueueCommandBufferExp_t)( + ur_queue_handle_t, ur_exp_command_buffer_handle_t, uint32_t, + const ur_event_handle_t *, ur_event_handle_t *); + /////////////////////////////////////////////////////////////////////////////// /// @brief Function-pointer for urEnqueueHostTaskExp typedef ur_result_t(UR_APICALL *ur_pfnEnqueueHostTaskExp_t)( @@ -1228,8 +1228,8 @@ typedef struct ur_enqueue_exp_dditable_t { ur_pfnEnqueueUSMSharedAllocExp_t pfnUSMSharedAllocExp; ur_pfnEnqueueUSMHostAllocExp_t pfnUSMHostAllocExp; ur_pfnEnqueueUSMFreeExp_t pfnUSMFreeExp; - ur_pfnEnqueueCommandBufferExp_t pfnCommandBufferExp; ur_pfnEnqueueTimestampRecordingExp_t pfnTimestampRecordingExp; + ur_pfnEnqueueCommandBufferExp_t pfnCommandBufferExp; ur_pfnEnqueueHostTaskExp_t pfnHostTaskExp; ur_pfnEnqueueNativeCommandExp_t pfnNativeCommandExp; ur_pfnEnqueueGraphExp_t pfnGraphExp; diff --git a/unified-runtime/include/ur_print.h b/unified-runtime/include/ur_print.h index a367339986b91..64ee9276e31dd 100644 --- a/unified-runtime/include/ur_print.h +++ b/unified-runtime/include/ur_print.h @@ -1353,6 +1353,56 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintExpImageCopyRegion( const struct ur_exp_image_copy_region_t params, char *buffer, const size_t buff_size, size_t *out_size); +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_exp_kernel_arg_type_t enum +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL +urPrintExpKernelArgType(enum ur_exp_kernel_arg_type_t value, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_exp_kernel_arg_mem_obj_tuple_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintExpKernelArgMemObjTuple( + const struct ur_exp_kernel_arg_mem_obj_tuple_t params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_exp_kernel_arg_properties_t struct +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL urPrintExpKernelArgProperties( + const struct ur_exp_kernel_arg_properties_t params, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_exp_program_flag_t enum +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL +urPrintExpProgramFlags(enum ur_exp_program_flag_t value, char *buffer, + const size_t buff_size, size_t *out_size); + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_exp_peer_info_t enum +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// - `buff_size < out_size` +UR_APIEXPORT ur_result_t UR_APICALL +urPrintExpPeerInfo(enum ur_exp_peer_info_t value, char *buffer, + const size_t buff_size, size_t *out_size); + /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_device_command_buffer_update_capability_flag_t enum /// @returns @@ -1435,56 +1485,6 @@ urPrintExpCommandBufferUpdateKernelLaunchDesc( const struct ur_exp_command_buffer_update_kernel_launch_desc_t params, char *buffer, const size_t buff_size, size_t *out_size); -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_program_flag_t enum -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL -urPrintExpProgramFlags(enum ur_exp_program_flag_t value, char *buffer, - const size_t buff_size, size_t *out_size); - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_peer_info_t enum -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL -urPrintExpPeerInfo(enum ur_exp_peer_info_t value, char *buffer, - const size_t buff_size, size_t *out_size); - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_kernel_arg_type_t enum -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL -urPrintExpKernelArgType(enum ur_exp_kernel_arg_type_t value, char *buffer, - const size_t buff_size, size_t *out_size); - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_kernel_arg_mem_obj_tuple_t struct -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL urPrintExpKernelArgMemObjTuple( - const struct ur_exp_kernel_arg_mem_obj_tuple_t params, char *buffer, - const size_t buff_size, size_t *out_size); - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_kernel_arg_properties_t struct -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL urPrintExpKernelArgProperties( - const struct ur_exp_kernel_arg_properties_t params, char *buffer, - const size_t buff_size, size_t *out_size); - /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_exp_host_task_flag_t enum /// @returns @@ -2886,24 +2886,24 @@ UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueUsmFreeExpParams( const size_t buff_size, size_t *out_size); /////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_enqueue_command_buffer_exp_params_t struct +/// @brief Print ur_enqueue_timestamp_recording_exp_params_t struct /// @returns /// - ::UR_RESULT_SUCCESS /// - ::UR_RESULT_ERROR_INVALID_SIZE /// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueCommandBufferExpParams( - const struct ur_enqueue_command_buffer_exp_params_t *params, char *buffer, - const size_t buff_size, size_t *out_size); +UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueTimestampRecordingExpParams( + const struct ur_enqueue_timestamp_recording_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size); /////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_enqueue_timestamp_recording_exp_params_t struct +/// @brief Print ur_enqueue_command_buffer_exp_params_t struct /// @returns /// - ::UR_RESULT_SUCCESS /// - ::UR_RESULT_ERROR_INVALID_SIZE /// - `buff_size < out_size` -UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueTimestampRecordingExpParams( - const struct ur_enqueue_timestamp_recording_exp_params_t *params, - char *buffer, const size_t buff_size, size_t *out_size); +UR_APIEXPORT ur_result_t UR_APICALL urPrintEnqueueCommandBufferExpParams( + const struct ur_enqueue_command_buffer_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size); /////////////////////////////////////////////////////////////////////////////// /// @brief Print ur_enqueue_host_task_exp_params_t struct diff --git a/unified-runtime/include/ur_print.hpp b/unified-runtime/include/ur_print.hpp index 156b27813e09c..4875cc73856ec 100644 --- a/unified-runtime/include/ur_print.hpp +++ b/unified-runtime/include/ur_print.hpp @@ -250,6 +250,18 @@ template <> inline ur_result_t printFlag(std::ostream &os, uint32_t flag); +inline ur_result_t printUnion(std::ostream &os, + const union ur_exp_kernel_arg_value_t params, + const enum ur_exp_kernel_arg_type_t tag); + +template <> +inline ur_result_t printFlag(std::ostream &os, + uint32_t flag); + +template <> +inline ur_result_t printTagged(std::ostream &os, const void *ptr, + ur_exp_peer_info_t value, size_t size); + template <> inline ur_result_t printFlag(std::ostream &os, @@ -263,18 +275,6 @@ inline ur_result_t printTagged(std::ostream &os, const void *ptr, ur_exp_command_buffer_command_info_t value, size_t size); -template <> -inline ur_result_t printFlag(std::ostream &os, - uint32_t flag); - -template <> -inline ur_result_t printTagged(std::ostream &os, const void *ptr, - ur_exp_peer_info_t value, size_t size); - -inline ur_result_t printUnion(std::ostream &os, - const union ur_exp_kernel_arg_value_t params, - const enum ur_exp_kernel_arg_type_t tag); - template <> inline ur_result_t printFlag(std::ostream &os, uint32_t flag); @@ -590,6 +590,18 @@ inline std::ostream &operator<<( inline std::ostream & operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_image_copy_region_t params); +inline std::ostream &operator<<(std::ostream &os, + enum ur_exp_kernel_arg_type_t value); +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_exp_kernel_arg_mem_obj_tuple_t params); +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_exp_kernel_arg_properties_t params); +inline std::ostream &operator<<(std::ostream &os, + enum ur_exp_program_flag_t value); +inline std::ostream &operator<<(std::ostream &os, + enum ur_exp_peer_info_t value); inline std::ostream & operator<<(std::ostream &os, enum ur_device_command_buffer_update_capability_flag_t value); @@ -614,18 +626,6 @@ inline std::ostream &operator<<( inline std::ostream & operator<<(std::ostream &os, [[maybe_unused]] const struct ur_exp_command_buffer_update_kernel_launch_desc_t params); -inline std::ostream &operator<<(std::ostream &os, - enum ur_exp_program_flag_t value); -inline std::ostream &operator<<(std::ostream &os, - enum ur_exp_peer_info_t value); -inline std::ostream &operator<<(std::ostream &os, - enum ur_exp_kernel_arg_type_t value); -inline std::ostream &operator<<( - std::ostream &os, - [[maybe_unused]] const struct ur_exp_kernel_arg_mem_obj_tuple_t params); -inline std::ostream & -operator<<(std::ostream &os, - [[maybe_unused]] const struct ur_exp_kernel_arg_properties_t params); inline std::ostream &operator<<(std::ostream &os, enum ur_exp_host_task_flag_t value); inline std::ostream & @@ -12030,31 +12030,149 @@ operator<<(std::ostream &os, const struct ur_exp_image_copy_region_t params) { return os; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the -/// ur_device_command_buffer_update_capability_flag_t type +/// @brief Print operator for the ur_exp_kernel_arg_type_t type /// @returns /// std::ostream & -inline std::ostream & -operator<<(std::ostream &os, - enum ur_device_command_buffer_update_capability_flag_t value) { +inline std::ostream &operator<<(std::ostream &os, + enum ur_exp_kernel_arg_type_t value) { switch (value) { - case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS: - os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS"; + case UR_EXP_KERNEL_ARG_TYPE_VALUE: + os << "UR_EXP_KERNEL_ARG_TYPE_VALUE"; break; - case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE: - os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE"; + case UR_EXP_KERNEL_ARG_TYPE_POINTER: + os << "UR_EXP_KERNEL_ARG_TYPE_POINTER"; break; - case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE: - os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE"; + case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: + os << "UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ"; break; - case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET: - os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET"; + case UR_EXP_KERNEL_ARG_TYPE_LOCAL: + os << "UR_EXP_KERNEL_ARG_TYPE_LOCAL"; break; - case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE: - os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE"; + case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: + os << "UR_EXP_KERNEL_ARG_TYPE_SAMPLER"; break; - case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS: - os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS"; + default: + os << "unknown enumerator"; + break; + } + return os; +} +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_exp_kernel_arg_mem_obj_tuple_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + const struct ur_exp_kernel_arg_mem_obj_tuple_t params) { + os << "(struct ur_exp_kernel_arg_mem_obj_tuple_t){"; + + os << ".hMem = "; + + ur::details::printPtr(os, (params.hMem)); + + os << ", "; + os << ".flags = "; + + ur::details::printFlag(os, (params.flags)); + + os << "}"; + return os; +} +namespace ur::details { + +/////////////////////////////////////////////////////////////////////////////// +// @brief Print ur_exp_kernel_arg_value_t union +inline ur_result_t printUnion(std::ostream &os, + const union ur_exp_kernel_arg_value_t params, + const enum ur_exp_kernel_arg_type_t tag) { + os << "(union ur_exp_kernel_arg_value_t){"; + + switch (tag) { + case UR_EXP_KERNEL_ARG_TYPE_VALUE: + + os << ".value = "; + + ur::details::printPtr(os, (params.value)); + + break; + case UR_EXP_KERNEL_ARG_TYPE_POINTER: + + os << ".pointer = "; + + ur::details::printPtr(os, (params.pointer)); + + break; + case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: + + os << ".memObjTuple = "; + + os << (params.memObjTuple); + + break; + case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: + + os << ".sampler = "; + + ur::details::printPtr(os, (params.sampler)); + + break; + default: + os << ""; + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + os << "}"; + return UR_RESULT_SUCCESS; +} +} // namespace ur::details +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_exp_kernel_arg_properties_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + const struct ur_exp_kernel_arg_properties_t params) { + os << "(struct ur_exp_kernel_arg_properties_t){"; + + os << ".stype = "; + + os << (params.stype); + + os << ", "; + os << ".pNext = "; + + ur::details::printStruct(os, (params.pNext)); + + os << ", "; + os << ".type = "; + + os << (params.type); + + os << ", "; + os << ".index = "; + + os << (params.index); + + os << ", "; + os << ".size = "; + + os << (params.size); + + os << ", "; + os << ".value = "; + ur::details::printUnion(os, (params.value), params.type); + + os << "}"; + return os; +} +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_exp_program_flag_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<(std::ostream &os, + enum ur_exp_program_flag_t value) { + switch (value) { + case UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS: + os << "UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS"; break; default: os << "unknown enumerator"; @@ -12065,64 +12183,193 @@ operator<<(std::ostream &os, namespace ur::details { /////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_device_command_buffer_update_capability_flag_t flag +/// @brief Print ur_exp_program_flag_t flag template <> -inline ur_result_t -printFlag(std::ostream &os, - uint32_t flag) { +inline ur_result_t printFlag(std::ostream &os, + uint32_t flag) { uint32_t val = flag; bool first = true; - if ((val & - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS) == - (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS) { - val ^= (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS; - if (!first) { - os << " | "; - } else { - first = false; - } - os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS; - } - - if ((val & UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE) == - (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE) { - val ^= (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE; + if ((val & UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS) == + (uint32_t)UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS) { + val ^= (uint32_t)UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS; if (!first) { os << " | "; } else { first = false; } - os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE; + os << UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS; } - - if ((val & - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE) == - (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE) { - val ^= (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE; + if (val != 0) { + std::bitset<32> bits(val); if (!first) { os << " | "; - } else { - first = false; } - os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE; + os << "unknown bit flags " << bits; + } else if (first) { + os << "0"; } - - if ((val & - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET) == - (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET) { - val ^= (uint32_t) - UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET; - if (!first) { - os << " | "; - } else { + return UR_RESULT_SUCCESS; +} +} // namespace ur::details +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the ur_exp_peer_info_t type +/// @returns +/// std::ostream & +inline std::ostream &operator<<(std::ostream &os, + enum ur_exp_peer_info_t value) { + switch (value) { + case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT: + os << "UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT"; + break; + case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT: + os << "UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT"; + break; + default: + os << "unknown enumerator"; + break; + } + return os; +} +namespace ur::details { +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_exp_peer_info_t enum value +template <> +inline ur_result_t printTagged(std::ostream &os, const void *ptr, + ur_exp_peer_info_t value, size_t size) { + if (ptr == NULL) { + return printPtr(os, ptr); + } + + switch (value) { + case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT: { + const int *tptr = (const int *)ptr; + if (sizeof(int) > size) { + os << "invalid size (is: " << size << ", expected: >=" << sizeof(int) + << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; + case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT: { + const int *tptr = (const int *)ptr; + if (sizeof(int) > size) { + os << "invalid size (is: " << size << ", expected: >=" << sizeof(int) + << ")"; + return UR_RESULT_ERROR_INVALID_SIZE; + } + os << (const void *)(tptr) << " ("; + + os << *tptr; + + os << ")"; + } break; + default: + os << "unknown enumerator"; + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + return UR_RESULT_SUCCESS; +} +} // namespace ur::details + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print operator for the +/// ur_device_command_buffer_update_capability_flag_t type +/// @returns +/// std::ostream & +inline std::ostream & +operator<<(std::ostream &os, + enum ur_device_command_buffer_update_capability_flag_t value) { + switch (value) { + case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS: + os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS"; + break; + case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE: + os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE"; + break; + case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE: + os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE"; + break; + case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET: + os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET"; + break; + case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE: + os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE"; + break; + case UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS: + os << "UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS"; + break; + default: + os << "unknown enumerator"; + break; + } + return os; +} + +namespace ur::details { +/////////////////////////////////////////////////////////////////////////////// +/// @brief Print ur_device_command_buffer_update_capability_flag_t flag +template <> +inline ur_result_t +printFlag(std::ostream &os, + uint32_t flag) { + uint32_t val = flag; + bool first = true; + + if ((val & + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS) == + (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS) { + val ^= (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS; + if (!first) { + os << " | "; + } else { + first = false; + } + os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS; + } + + if ((val & UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE) == + (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE) { + val ^= (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE; + if (!first) { + os << " | "; + } else { + first = false; + } + os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE; + } + + if ((val & + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE) == + (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE) { + val ^= (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE; + if (!first) { + os << " | "; + } else { + first = false; + } + os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE; + } + + if ((val & + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET) == + (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET) { + val ^= (uint32_t) + UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET; + if (!first) { + os << " | "; + } else { first = false; } os << UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET; @@ -12577,253 +12824,6 @@ inline std::ostream &operator<<( return os; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_exp_program_flag_t type -/// @returns -/// std::ostream & -inline std::ostream &operator<<(std::ostream &os, - enum ur_exp_program_flag_t value) { - switch (value) { - case UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS: - os << "UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS"; - break; - default: - os << "unknown enumerator"; - break; - } - return os; -} - -namespace ur::details { -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_program_flag_t flag -template <> -inline ur_result_t printFlag(std::ostream &os, - uint32_t flag) { - uint32_t val = flag; - bool first = true; - - if ((val & UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS) == - (uint32_t)UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS) { - val ^= (uint32_t)UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS; - if (!first) { - os << " | "; - } else { - first = false; - } - os << UR_EXP_PROGRAM_FLAG_ALLOW_UNRESOLVED_SYMBOLS; - } - if (val != 0) { - std::bitset<32> bits(val); - if (!first) { - os << " | "; - } - os << "unknown bit flags " << bits; - } else if (first) { - os << "0"; - } - return UR_RESULT_SUCCESS; -} -} // namespace ur::details -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_exp_peer_info_t type -/// @returns -/// std::ostream & -inline std::ostream &operator<<(std::ostream &os, - enum ur_exp_peer_info_t value) { - switch (value) { - case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT: - os << "UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT"; - break; - case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT: - os << "UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT"; - break; - default: - os << "unknown enumerator"; - break; - } - return os; -} -namespace ur::details { -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print ur_exp_peer_info_t enum value -template <> -inline ur_result_t printTagged(std::ostream &os, const void *ptr, - ur_exp_peer_info_t value, size_t size) { - if (ptr == NULL) { - return printPtr(os, ptr); - } - - switch (value) { - case UR_EXP_PEER_INFO_UR_PEER_ACCESS_SUPPORT: { - const int *tptr = (const int *)ptr; - if (sizeof(int) > size) { - os << "invalid size (is: " << size << ", expected: >=" << sizeof(int) - << ")"; - return UR_RESULT_ERROR_INVALID_SIZE; - } - os << (const void *)(tptr) << " ("; - - os << *tptr; - - os << ")"; - } break; - case UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT: { - const int *tptr = (const int *)ptr; - if (sizeof(int) > size) { - os << "invalid size (is: " << size << ", expected: >=" << sizeof(int) - << ")"; - return UR_RESULT_ERROR_INVALID_SIZE; - } - os << (const void *)(tptr) << " ("; - - os << *tptr; - - os << ")"; - } break; - default: - os << "unknown enumerator"; - return UR_RESULT_ERROR_INVALID_ENUMERATION; - } - return UR_RESULT_SUCCESS; -} -} // namespace ur::details - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_exp_kernel_arg_type_t type -/// @returns -/// std::ostream & -inline std::ostream &operator<<(std::ostream &os, - enum ur_exp_kernel_arg_type_t value) { - switch (value) { - case UR_EXP_KERNEL_ARG_TYPE_VALUE: - os << "UR_EXP_KERNEL_ARG_TYPE_VALUE"; - break; - case UR_EXP_KERNEL_ARG_TYPE_POINTER: - os << "UR_EXP_KERNEL_ARG_TYPE_POINTER"; - break; - case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: - os << "UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ"; - break; - case UR_EXP_KERNEL_ARG_TYPE_LOCAL: - os << "UR_EXP_KERNEL_ARG_TYPE_LOCAL"; - break; - case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: - os << "UR_EXP_KERNEL_ARG_TYPE_SAMPLER"; - break; - default: - os << "unknown enumerator"; - break; - } - return os; -} -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_exp_kernel_arg_mem_obj_tuple_t type -/// @returns -/// std::ostream & -inline std::ostream & -operator<<(std::ostream &os, - const struct ur_exp_kernel_arg_mem_obj_tuple_t params) { - os << "(struct ur_exp_kernel_arg_mem_obj_tuple_t){"; - - os << ".hMem = "; - - ur::details::printPtr(os, (params.hMem)); - - os << ", "; - os << ".flags = "; - - ur::details::printFlag(os, (params.flags)); - - os << "}"; - return os; -} -namespace ur::details { - -/////////////////////////////////////////////////////////////////////////////// -// @brief Print ur_exp_kernel_arg_value_t union -inline ur_result_t printUnion(std::ostream &os, - const union ur_exp_kernel_arg_value_t params, - const enum ur_exp_kernel_arg_type_t tag) { - os << "(union ur_exp_kernel_arg_value_t){"; - - switch (tag) { - case UR_EXP_KERNEL_ARG_TYPE_VALUE: - - os << ".value = "; - - ur::details::printPtr(os, (params.value)); - - break; - case UR_EXP_KERNEL_ARG_TYPE_POINTER: - - os << ".pointer = "; - - ur::details::printPtr(os, (params.pointer)); - - break; - case UR_EXP_KERNEL_ARG_TYPE_MEM_OBJ: - - os << ".memObjTuple = "; - - os << (params.memObjTuple); - - break; - case UR_EXP_KERNEL_ARG_TYPE_SAMPLER: - - os << ".sampler = "; - - ur::details::printPtr(os, (params.sampler)); - - break; - default: - os << ""; - return UR_RESULT_ERROR_INVALID_ENUMERATION; - } - os << "}"; - return UR_RESULT_SUCCESS; -} -} // namespace ur::details -/////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_exp_kernel_arg_properties_t type -/// @returns -/// std::ostream & -inline std::ostream & -operator<<(std::ostream &os, - const struct ur_exp_kernel_arg_properties_t params) { - os << "(struct ur_exp_kernel_arg_properties_t){"; - - os << ".stype = "; - - os << (params.stype); - - os << ", "; - os << ".pNext = "; - - ur::details::printStruct(os, (params.pNext)); - - os << ", "; - os << ".type = "; - - os << (params.type); - - os << ", "; - os << ".index = "; - - os << (params.index); - - os << ", "; - os << ".size = "; - - os << (params.size); - - os << ", "; - os << ".value = "; - ur::details::printUnion(os, (params.value), params.type); - - os << "}"; - return os; -} -/////////////////////////////////////////////////////////////////////////////// /// @brief Print operator for the ur_exp_host_task_flag_t type /// @returns /// std::ostream & @@ -18040,22 +18040,23 @@ inline std::ostream &operator<<( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_enqueue_command_buffer_exp_params_t type +/// @brief Print operator for the ur_enqueue_timestamp_recording_exp_params_t +/// type /// @returns /// std::ostream & -inline std::ostream & -operator<<(std::ostream &os, - [[maybe_unused]] const struct ur_enqueue_command_buffer_exp_params_t - *params) { +inline std::ostream &operator<<( + std::ostream &os, + [[maybe_unused]] const struct ur_enqueue_timestamp_recording_exp_params_t + *params) { os << ".hQueue = "; ur::details::printPtr(os, *(params->phQueue)); os << ", "; - os << ".hCommandBuffer = "; + os << ".blocking = "; - ur::details::printPtr(os, *(params->phCommandBuffer)); + os << *(params->pblocking); os << ", "; os << ".numEventsInWaitList = "; @@ -18087,23 +18088,22 @@ operator<<(std::ostream &os, } /////////////////////////////////////////////////////////////////////////////// -/// @brief Print operator for the ur_enqueue_timestamp_recording_exp_params_t -/// type +/// @brief Print operator for the ur_enqueue_command_buffer_exp_params_t type /// @returns /// std::ostream & -inline std::ostream &operator<<( - std::ostream &os, - [[maybe_unused]] const struct ur_enqueue_timestamp_recording_exp_params_t - *params) { +inline std::ostream & +operator<<(std::ostream &os, + [[maybe_unused]] const struct ur_enqueue_command_buffer_exp_params_t + *params) { os << ".hQueue = "; ur::details::printPtr(os, *(params->phQueue)); os << ", "; - os << ".blocking = "; + os << ".hCommandBuffer = "; - os << *(params->pblocking); + ur::details::printPtr(os, *(params->phCommandBuffer)); os << ", "; os << ".numEventsInWaitList = "; @@ -22672,12 +22672,12 @@ inline ur_result_t UR_APICALL printFunctionParams(std::ostream &os, case UR_FUNCTION_ENQUEUE_USM_FREE_EXP: { os << (const struct ur_enqueue_usm_free_exp_params_t *)params; } break; - case UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP: { - os << (const struct ur_enqueue_command_buffer_exp_params_t *)params; - } break; case UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP: { os << (const struct ur_enqueue_timestamp_recording_exp_params_t *)params; } break; + case UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP: { + os << (const struct ur_enqueue_command_buffer_exp_params_t *)params; + } break; case UR_FUNCTION_ENQUEUE_HOST_TASK_EXP: { os << (const struct ur_enqueue_host_task_exp_params_t *)params; } break; diff --git a/unified-runtime/scripts/core/exp-command-buffer.yml b/unified-runtime/scripts/core/exp-command-buffer.yml index 432354b89f5e7..e347fdc52c645 100644 --- a/unified-runtime/scripts/core/exp-command-buffer.yml +++ b/unified-runtime/scripts/core/exp-command-buffer.yml @@ -10,7 +10,7 @@ --- #-------------------------------------------------------------------------- type: header desc: "Intel $OneApi Unified Runtime Experimental APIs for Command-Buffers" -ordinal: "99" +ordinal: "100" --- #-------------------------------------------------------------------------- type: enum extend: true diff --git a/unified-runtime/scripts/core/exp-enqueue-kernel-launch-with-args.yml b/unified-runtime/scripts/core/exp-enqueue-kernel-launch-with-args.yml index 937dc6e3fe8f5..fc8e56bd5dee3 100644 --- a/unified-runtime/scripts/core/exp-enqueue-kernel-launch-with-args.yml +++ b/unified-runtime/scripts/core/exp-enqueue-kernel-launch-with-args.yml @@ -10,7 +10,7 @@ --- #-------------------------------------------------------------------------- type: header desc: "Intel $OneApi Unified Runtime Experimental API for setting args at kernel launch" -ordinal: "100" +ordinal: "99" --- #-------------------------------------------------------------------------- type: enum desc: "What kind of kernel arg is this" diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp index d5154018cfada..c423aae2d0c02 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.cpp @@ -231,9 +231,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( pDdiTable->pfnUSMSharedAllocExp = ur::level_zero::urEnqueueUSMSharedAllocExp; pDdiTable->pfnUSMHostAllocExp = ur::level_zero::urEnqueueUSMHostAllocExp; pDdiTable->pfnUSMFreeExp = ur::level_zero::urEnqueueUSMFreeExp; - pDdiTable->pfnCommandBufferExp = ur::level_zero::urEnqueueCommandBufferExp; pDdiTable->pfnTimestampRecordingExp = ur::level_zero::urEnqueueTimestampRecordingExp; + pDdiTable->pfnCommandBufferExp = ur::level_zero::urEnqueueCommandBufferExp; pDdiTable->pfnHostTaskExp = ur::level_zero::urEnqueueHostTaskExp; pDdiTable->pfnNativeCommandExp = ur::level_zero::urEnqueueNativeCommandExp; pDdiTable->pfnGraphExp = ur::level_zero::urEnqueueGraphExp; diff --git a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp index 7cb148e1c51bb..8ddf614891c97 100644 --- a/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp +++ b/unified-runtime/source/adapters/level_zero/ur_interface_loader.hpp @@ -626,6 +626,70 @@ ur_result_t urBindlessImagesSignalExternalSemaphoreExp( ur_queue_handle_t hQueue, ur_exp_external_semaphore_handle_t hSemaphore, bool hasSignalValue, uint64_t signalValue, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urDeviceWaitExp(ur_device_handle_t hDevice); +ur_result_t urProgramDynamicLinkExp(ur_context_handle_t hContext, + uint32_t count, + const ur_program_handle_t *phPrograms); +ur_result_t urEnqueueKernelLaunchWithArgsExp( + ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, + const ur_kernel_launch_ext_properties_t *launchPropList, + uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, + ur_event_handle_t *phEvent); +ur_result_t urEnqueueTimestampRecordingExp( + ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); +ur_result_t urIPCGetMemHandleExp(ur_context_handle_t hContext, void *pMem, + void **ppIPCMemHandleData, + size_t *pIPCMemHandleDataSizeRet); +ur_result_t urIPCPutMemHandleExp(ur_context_handle_t hContext, + void *pIPCMemHandleData); +ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + void *pIPCMemHandleData, + size_t ipcMemHandleDataSize, void **ppMem); +ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urMemoryExportAllocExportableMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, + size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); +ur_result_t urMemoryExportFreeExportableMemoryExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + void *pMem); +ur_result_t urMemoryExportExportMemoryHandleExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_mem_type_t handleTypeToExport, void *pMem, + void *pMemHandleRet); +ur_result_t urProgramBuildExp(ur_program_handle_t hProgram, uint32_t numDevices, + ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramCompileExp(ur_program_handle_t hProgram, + uint32_t numDevices, + ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, + const char *pOptions); +ur_result_t urProgramLinkExp(ur_context_handle_t hContext, uint32_t numDevices, + ur_device_handle_t *phDevices, + ur_exp_program_flags_t flags, uint32_t count, + const ur_program_handle_t *phPrograms, + const char *pOptions, + ur_program_handle_t *phProgram); +ur_result_t urUSMContextMemcpyExp(ur_context_handle_t hContext, void *pDst, + const void *pSrc, size_t size); +ur_result_t urUSMImportExp(ur_context_handle_t hContext, void *pMem, + size_t size); +ur_result_t urUSMReleaseExp(ur_context_handle_t hContext, void *pMem); +ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice); +ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice); +ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, + ur_device_handle_t peerDevice, + ur_exp_peer_info_t propName, + size_t propSize, void *pPropValue, + size_t *pPropSizeRet); ur_result_t urCommandBufferCreateExp(ur_context_handle_t hContext, ur_device_handle_t hDevice, @@ -767,70 +831,6 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, ur_result_t urCommandBufferGetNativeHandleExp(ur_exp_command_buffer_handle_t hCommandBuffer, ur_native_handle_t *phNativeCommandBuffer); -ur_result_t urDeviceWaitExp(ur_device_handle_t hDevice); -ur_result_t urProgramDynamicLinkExp(ur_context_handle_t hContext, - uint32_t count, - const ur_program_handle_t *phPrograms); -ur_result_t urEnqueueTimestampRecordingExp( - ur_queue_handle_t hQueue, bool blocking, uint32_t numEventsInWaitList, - const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent); -ur_result_t urIPCGetMemHandleExp(ur_context_handle_t hContext, void *pMem, - void **ppIPCMemHandleData, - size_t *pIPCMemHandleDataSizeRet); -ur_result_t urIPCPutMemHandleExp(ur_context_handle_t hContext, - void *pIPCMemHandleData); -ur_result_t urIPCOpenMemHandleExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - void *pIPCMemHandleData, - size_t ipcMemHandleDataSize, void **ppMem); -ur_result_t urIPCCloseMemHandleExp(ur_context_handle_t hContext, void *pMem); -ur_result_t urMemoryExportAllocExportableMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t alignment, - size_t size, ur_exp_external_mem_type_t handleTypeToExport, void **ppMem); -ur_result_t urMemoryExportFreeExportableMemoryExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - void *pMem); -ur_result_t urMemoryExportExportMemoryHandleExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_mem_type_t handleTypeToExport, void *pMem, - void *pMemHandleRet); -ur_result_t urProgramBuildExp(ur_program_handle_t hProgram, uint32_t numDevices, - ur_device_handle_t *phDevices, - ur_exp_program_flags_t flags, - const char *pOptions); -ur_result_t urProgramCompileExp(ur_program_handle_t hProgram, - uint32_t numDevices, - ur_device_handle_t *phDevices, - ur_exp_program_flags_t flags, - const char *pOptions); -ur_result_t urProgramLinkExp(ur_context_handle_t hContext, uint32_t numDevices, - ur_device_handle_t *phDevices, - ur_exp_program_flags_t flags, uint32_t count, - const ur_program_handle_t *phPrograms, - const char *pOptions, - ur_program_handle_t *phProgram); -ur_result_t urUSMContextMemcpyExp(ur_context_handle_t hContext, void *pDst, - const void *pSrc, size_t size); -ur_result_t urUSMImportExp(ur_context_handle_t hContext, void *pMem, - size_t size); -ur_result_t urUSMReleaseExp(ur_context_handle_t hContext, void *pMem); -ur_result_t urUsmP2PEnablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice); -ur_result_t urUsmP2PDisablePeerAccessExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice); -ur_result_t urUsmP2PPeerAccessGetInfoExp(ur_device_handle_t commandDevice, - ur_device_handle_t peerDevice, - ur_exp_peer_info_t propName, - size_t propSize, void *pPropValue, - size_t *pPropSizeRet); -ur_result_t urEnqueueKernelLaunchWithArgsExp( - ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, - const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, - const size_t *pLocalWorkSize, uint32_t numArgs, - const ur_exp_kernel_arg_properties_t *pArgs, - const ur_kernel_launch_ext_properties_t *launchPropList, - uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, - ur_event_handle_t *phEvent); ur_result_t urEnqueueHostTaskExp( ur_queue_handle_t hQueue, ur_exp_host_task_function_t pfnHostTask, void *data, const ur_exp_host_task_properties_t *pProperties, diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp index 40511cc1e1abb..cfe540648231e 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_api.cpp @@ -424,12 +424,18 @@ ur_result_t urBindlessImagesSignalExternalSemaphoreExp( } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueCommandBufferExp( - ur_queue_handle_t hQueue, ur_exp_command_buffer_handle_t hCommandBuffer, +ur_result_t urEnqueueKernelLaunchWithArgsExp( + ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, + const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, + const size_t *pLocalWorkSize, uint32_t numArgs, + const ur_exp_kernel_arg_properties_t *pArgs, + const ur_kernel_launch_ext_properties_t *launchPropList, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueCommandBufferExp( - hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); + return hQueue->get().enqueueKernelLaunchWithArgsExp( + hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, + numArgs, pArgs, launchPropList, numEventsInWaitList, phEventWaitList, + phEvent); } catch (...) { return exceptionToResult(std::current_exception()); } @@ -441,18 +447,12 @@ ur_result_t urEnqueueTimestampRecordingExp( } catch (...) { return exceptionToResult(std::current_exception()); } -ur_result_t urEnqueueKernelLaunchWithArgsExp( - ur_queue_handle_t hQueue, ur_kernel_handle_t hKernel, uint32_t workDim, - const size_t *pGlobalWorkOffset, const size_t *pGlobalWorkSize, - const size_t *pLocalWorkSize, uint32_t numArgs, - const ur_exp_kernel_arg_properties_t *pArgs, - const ur_kernel_launch_ext_properties_t *launchPropList, +ur_result_t urEnqueueCommandBufferExp( + ur_queue_handle_t hQueue, ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) try { - return hQueue->get().enqueueKernelLaunchWithArgsExp( - hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, pLocalWorkSize, - numArgs, pArgs, launchPropList, numEventsInWaitList, phEventWaitList, - phEvent); + return hQueue->get().enqueueCommandBufferExp( + hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); } catch (...) { return exceptionToResult(std::current_exception()); } diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp index 3cb8d73fc704d..d72aad98b7075 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_api.hpp @@ -156,18 +156,18 @@ struct ur_queue_t_ : ur_queue_extensions { virtual ur_result_t bindlessImagesSignalExternalSemaphoreExp( ur_exp_external_semaphore_handle_t, bool, uint64_t, uint32_t, const ur_event_handle_t *, ur_event_handle_t *) = 0; - virtual ur_result_t enqueueCommandBufferExp(ur_exp_command_buffer_handle_t, - uint32_t, - const ur_event_handle_t *, - ur_event_handle_t *) = 0; - virtual ur_result_t enqueueTimestampRecordingExp(bool, uint32_t, - const ur_event_handle_t *, - ur_event_handle_t *) = 0; virtual ur_result_t enqueueKernelLaunchWithArgsExp( ur_kernel_handle_t, uint32_t, const size_t *, const size_t *, const size_t *, uint32_t, const ur_exp_kernel_arg_properties_t *, const ur_kernel_launch_ext_properties_t *, uint32_t, const ur_event_handle_t *, ur_event_handle_t *) = 0; + virtual ur_result_t enqueueTimestampRecordingExp(bool, uint32_t, + const ur_event_handle_t *, + ur_event_handle_t *) = 0; + virtual ur_result_t enqueueCommandBufferExp(ur_exp_command_buffer_handle_t, + uint32_t, + const ur_event_handle_t *, + ur_event_handle_t *) = 0; virtual ur_result_t enqueueHostTaskExp(ur_exp_host_task_function_t, void *, const ur_exp_host_task_properties_t *, uint32_t, const ur_event_handle_t *, diff --git a/unified-runtime/source/adapters/mock/ur_mockddi.cpp b/unified-runtime/source/adapters/mock/ur_mockddi.cpp index 4a6a8c50f4467..8b8f4d4138cff 100644 --- a/unified-runtime/source/adapters/mock/ur_mockddi.cpp +++ b/unified-runtime/source/adapters/mock/ur_mockddi.cpp @@ -9488,23 +9488,16 @@ __urdlllocal ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferCreateExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. - ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer) try { +/// @brief Intercept function for urDeviceWaitExp +__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_create_exp_params_t params = { - &hContext, &hDevice, &pCommandBufferDesc, &phCommandBuffer}; + ur_device_wait_exp_params_t params = {&hDevice}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urCommandBufferCreateExp")); + mock::getCallbacks().get_before_callback("urDeviceWaitExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9513,13 +9506,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urCommandBufferCreateExp")); + mock::getCallbacks().get_replace_callback("urDeviceWaitExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - *phCommandBuffer = - mock::createDummyHandle(); result = UR_RESULT_SUCCESS; } @@ -9528,7 +9519,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urCommandBufferCreateExp")); + mock::getCallbacks().get_after_callback("urDeviceWaitExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9539,16 +9530,21 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferRetainExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) try { +/// @brief Intercept function for urProgramDynamicLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_retain_exp_params_t params = {&hCommandBuffer}; + ur_program_dynamic_link_exp_params_t params = {&hContext, &count, + &phPrograms}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urCommandBufferRetainExp")); + mock::getCallbacks().get_before_callback("urProgramDynamicLinkExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9557,12 +9553,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urCommandBufferRetainExp")); + mock::getCallbacks().get_replace_callback("urProgramDynamicLinkExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - mock::retainDummyHandle(hCommandBuffer); result = UR_RESULT_SUCCESS; } @@ -9571,7 +9566,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urCommandBufferRetainExp")); + mock::getCallbacks().get_after_callback("urProgramDynamicLinkExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9582,16 +9577,59 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferReleaseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) try { +/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp +__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object + ur_kernel_handle_t hKernel, + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items + uint32_t workDim, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item + const size_t *pGlobalWorkOffset, + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function + const size_t *pGlobalWorkSize, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. + const size_t *pLocalWorkSize, + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_release_exp_params_t params = {&hCommandBuffer}; + ur_enqueue_kernel_launch_with_args_exp_params_t params = { + &hQueue, &hKernel, + &workDim, &pGlobalWorkOffset, + &pGlobalWorkSize, &pLocalWorkSize, + &numArgs, &pArgs, + &launchPropList, &numEventsInWaitList, + &phEventWaitList, &phEvent}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urCommandBufferReleaseExp")); + mock::getCallbacks().get_before_callback( + "urEnqueueKernelLaunchWithArgsExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9600,12 +9638,16 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urCommandBufferReleaseExp")); + mock::getCallbacks().get_replace_callback( + "urEnqueueKernelLaunchWithArgsExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - mock::releaseDummyHandle(hCommandBuffer); + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -9614,7 +9656,8 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urCommandBufferReleaseExp")); + mock::getCallbacks().get_after_callback( + "urEnqueueKernelLaunchWithArgsExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9625,16 +9668,40 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferFinalizeExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) try { +/// @brief Intercept function for urEnqueueTimestampRecordingExp +__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_finalize_exp_params_t params = {&hCommandBuffer}; + ur_enqueue_timestamp_recording_exp_params_t params = { + &hQueue, &blocking, &numEventsInWaitList, &phEventWaitList, &phEvent}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urCommandBufferFinalizeExp")); + mock::getCallbacks().get_before_callback( + "urEnqueueTimestampRecordingExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9643,11 +9710,13 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urCommandBufferFinalizeExp")); + mock::getCallbacks().get_replace_callback( + "urEnqueueTimestampRecordingExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + *phEvent = mock::createDummyHandle(); result = UR_RESULT_SUCCESS; } @@ -9656,7 +9725,8 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urCommandBufferFinalizeExp")); + mock::getCallbacks().get_after_callback( + "urEnqueueTimestampRecordingExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9667,73 +9737,23 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. - ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. - uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. - const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. - const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. - const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urIPCGetMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_kernel_launch_exp_params_t params = { - &hCommandBuffer, - &hKernel, - &workDim, - &pGlobalWorkOffset, - &pGlobalWorkSize, - &pLocalWorkSize, - &numKernelAlternatives, - &phKernelAlternatives, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_ipc_get_mem_handle_exp_params_t params = { + &hContext, &pMem, &ppIPCMemHandleData, &pIPCMemHandleDataSizeRet}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendKernelLaunchExp")); + mock::getCallbacks().get_before_callback("urIPCGetMemHandleExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9742,21 +9762,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendKernelLaunchExp")); + mock::getCallbacks().get_replace_callback("urIPCGetMemHandleExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -9765,8 +9775,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendKernelLaunchExp")); + mock::getCallbacks().get_after_callback("urIPCGetMemHandleExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9777,53 +9786,18 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urIPCPutMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_usm_memcpy_exp_params_t params = { - &hCommandBuffer, - &pDst, - &pSrc, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_ipc_put_mem_handle_exp_params_t params = {&hContext, &pIPCMemHandleData}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendUSMMemcpyExp")); + mock::getCallbacks().get_before_callback("urIPCPutMemHandleExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9832,21 +9806,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendUSMMemcpyExp")); + mock::getCallbacks().get_replace_callback("urIPCPutMemHandleExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -9855,8 +9819,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendUSMMemcpyExp")); + mock::getCallbacks().get_after_callback("urIPCPutMemHandleExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9867,56 +9830,26 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urIPCOpenMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_usm_fill_exp_params_t params = { - &hCommandBuffer, - &pMemory, - &pPattern, - &patternSize, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_ipc_open_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCMemHandleData, &ipcMemHandleDataSize, &ppMem}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendUSMFillExp")); + mock::getCallbacks().get_before_callback("urIPCOpenMemHandleExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -9925,21 +9858,55 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendUSMFillExp")); + mock::getCallbacks().get_replace_callback("urIPCOpenMemHandleExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urIPCOpenMemHandleExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCCloseMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem) try { + ur_result_t result = UR_RESULT_SUCCESS; + + ur_ipc_close_mem_handle_exp_params_t params = {&hContext, &pMem}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urIPCCloseMemHandleExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urIPCCloseMemHandleExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + result = UR_RESULT_SUCCESS; } @@ -9948,8 +9915,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendUSMFillExp")); + mock::getCallbacks().get_after_callback("urIPCCloseMemHandleExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -9960,59 +9926,29 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_copy_exp_params_t params = { - &hCommandBuffer, - &hSrcMem, - &hDstMem, - &srcOffset, - &dstOffset, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_memory_export_alloc_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &alignment, &size, &handleTypeToExport, &ppMem}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferCopyExp")); + "urMemoryExportAllocExportableMemoryExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10022,20 +9958,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferCopyExp")); + "urMemoryExportAllocExportableMemoryExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10045,7 +9972,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferCopyExp")); + "urMemoryExportAllocExportableMemoryExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10056,56 +9983,22 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_write_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &offset, - &size, - &pSrc, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_memory_export_free_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &pMem}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferWriteExp")); + "urMemoryExportFreeExportableMemoryExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10115,20 +10008,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferWriteExp")); + "urMemoryExportFreeExportableMemoryExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10138,7 +10022,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferWriteExp")); + "urMemoryExportFreeExportableMemoryExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10149,56 +10033,27 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_read_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &offset, - &size, - &pDst, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_memory_export_export_memory_handle_exp_params_t params = { + &hContext, &hDevice, &handleTypeToExport, &pMem, &pMemHandleRet}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferReadExp")); + "urMemoryExportExportMemoryHandleExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10208,20 +10063,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferReadExp")); + "urMemoryExportExportMemoryHandleExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10231,7 +10077,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferReadExp")); + "urMemoryExportExportMemoryHandleExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10242,71 +10088,25 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urProgramBuildExp +__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_copy_rect_exp_params_t params = { - &hCommandBuffer, - &hSrcMem, - &hDstMem, - &srcOrigin, - &dstOrigin, - ®ion, - &srcRowPitch, - &srcSlicePitch, - &dstRowPitch, - &dstSlicePitch, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_program_build_exp_params_t params = {&hProgram, &numDevices, &phDevices, + &flags, &pOptions}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferCopyRectExp")); + mock::getCallbacks().get_before_callback("urProgramBuildExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10315,21 +10115,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferCopyRectExp")); + mock::getCallbacks().get_replace_callback("urProgramBuildExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10338,8 +10128,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferCopyRectExp")); + mock::getCallbacks().get_after_callback("urProgramBuildExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10350,74 +10139,25 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urProgramCompileExp +__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_write_rect_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &bufferOffset, - &hostOffset, - ®ion, - &bufferRowPitch, - &bufferSlicePitch, - &hostRowPitch, - &hostSlicePitch, - &pSrc, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_program_compile_exp_params_t params = {&hProgram, &numDevices, &phDevices, + &flags, &pOptions}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferWriteRectExp")); + mock::getCallbacks().get_before_callback("urProgramCompileExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10426,21 +10166,73 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferWriteRectExp")); + mock::getCallbacks().get_replace_callback("urProgramCompileExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); + result = UR_RESULT_SUCCESS; + } + + if (result != UR_RESULT_SUCCESS) { + return result; + } + + auto afterCallback = reinterpret_cast( + mock::getCallbacks().get_after_callback("urProgramCompileExp")); + if (afterCallback) { + return afterCallback(¶ms); + } + + return result; +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urProgramLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram) try { + ur_result_t result = UR_RESULT_SUCCESS; + if (nullptr != phProgram) { + *phProgram = nullptr; + } + + ur_program_link_exp_params_t params = {&hContext, &numDevices, &phDevices, + &flags, &count, &phPrograms, + &pOptions, &phProgram}; + + auto beforeCallback = reinterpret_cast( + mock::getCallbacks().get_before_callback("urProgramLinkExp")); + if (beforeCallback) { + result = beforeCallback(¶ms); + if (result != UR_RESULT_SUCCESS) { + return result; } + } + + auto replaceCallback = reinterpret_cast( + mock::getCallbacks().get_replace_callback("urProgramLinkExp")); + if (replaceCallback) { + result = replaceCallback(¶ms); + } else { + + *phProgram = mock::createDummyHandle(); result = UR_RESULT_SUCCESS; } @@ -10449,8 +10241,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferWriteRectExp")); + mock::getCallbacks().get_after_callback("urProgramLinkExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10461,73 +10252,23 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. +/// @brief Intercept function for urUSMContextMemcpyExp +__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_read_rect_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &bufferOffset, - &hostOffset, - ®ion, - &bufferRowPitch, - &bufferSlicePitch, - &hostRowPitch, - &hostSlicePitch, - &pDst, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_usm_context_memcpy_exp_params_t params = {&hContext, &pDst, &pSrc, &size}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferReadRectExp")); + mock::getCallbacks().get_before_callback("urUSMContextMemcpyExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10536,21 +10277,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferReadRectExp")); + mock::getCallbacks().get_replace_callback("urUSMContextMemcpyExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10559,8 +10290,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferReadRectExp")); + mock::getCallbacks().get_after_callback("urUSMContextMemcpyExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10571,59 +10301,20 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urUSMImportExp +__urdlllocal ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_mem_buffer_fill_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &pPattern, - &patternSize, - &offset, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_usm_import_exp_params_t params = {&hContext, &pMem, &size}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendMemBufferFillExp")); + mock::getCallbacks().get_before_callback("urUSMImportExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10632,21 +10323,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendMemBufferFillExp")); + mock::getCallbacks().get_replace_callback("urUSMImportExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10655,8 +10336,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendMemBufferFillExp")); + mock::getCallbacks().get_after_callback("urUSMImportExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10667,53 +10347,18 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urUSMReleaseExp +__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_usm_prefetch_exp_params_t params = { - &hCommandBuffer, - &pMemory, - &size, - &flags, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_usm_release_exp_params_t params = {&hContext, &pMem}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendUSMPrefetchExp")); + mock::getCallbacks().get_before_callback("urUSMReleaseExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10722,21 +10367,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendUSMPrefetchExp")); + mock::getCallbacks().get_replace_callback("urUSMReleaseExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10745,8 +10380,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendUSMPrefetchExp")); + mock::getCallbacks().get_after_callback("urUSMReleaseExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10757,53 +10391,19 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { +/// @brief Intercept function for urUsmP2PEnablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_usm_advise_exp_params_t params = { - &hCommandBuffer, - &pMemory, - &size, - &advice, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_usm_p2p_enable_peer_access_exp_params_t params = {&commandDevice, + &peerDevice}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendUSMAdviseExp")); + mock::getCallbacks().get_before_callback("urUsmP2PEnablePeerAccessExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10812,21 +10412,11 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendUSMAdviseExp")); + mock::getCallbacks().get_replace_callback("urUsmP2PEnablePeerAccessExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } - // optional output handle - if (phCommand) { - *phCommand = - mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10835,8 +10425,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendUSMAdviseExp")); + mock::getCallbacks().get_after_callback("urUsmP2PEnablePeerAccessExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10847,38 +10436,19 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendNativeCommandExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint) try { +/// @brief Intercept function for urUsmP2PDisablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_append_native_command_exp_params_t params = { - &hCommandBuffer, &pfnNativeCommand, &pData, - &hChildCommandBuffer, &numSyncPointsInWaitList, &pSyncPointWaitList, - &pSyncPoint}; + ur_usm_p2p_disable_peer_access_exp_params_t params = {&commandDevice, + &peerDevice}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferAppendNativeCommandExp")); + mock::getCallbacks().get_before_callback("urUsmP2PDisablePeerAccessExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10888,7 +10458,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urCommandBufferAppendNativeCommandExp")); + "urUsmP2PDisablePeerAccessExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { @@ -10901,8 +10471,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferAppendNativeCommandExp")); + mock::getCallbacks().get_after_callback("urUsmP2PDisablePeerAccessExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10913,32 +10482,34 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueCommandBufferExp -__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) try { +/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. + size_t propSize, + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. + void *pPropValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. + size_t *pPropSizeRet) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_enqueue_command_buffer_exp_params_t params = {&hQueue, &hCommandBuffer, - &numEventsInWaitList, - &phEventWaitList, &phEvent}; + ur_usm_p2p_peer_access_get_info_exp_params_t params = { + &commandDevice, &peerDevice, &propName, + &propSize, &pPropValue, &pPropSizeRet}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urEnqueueCommandBufferExp")); + mock::getCallbacks().get_before_callback("urUsmP2PPeerAccessGetInfoExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -10947,15 +10518,12 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urEnqueueCommandBufferExp")); + mock::getCallbacks().get_replace_callback( + "urUsmP2PPeerAccessGetInfoExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } result = UR_RESULT_SUCCESS; } @@ -10964,7 +10532,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urEnqueueCommandBufferExp")); + mock::getCallbacks().get_after_callback("urUsmP2PPeerAccessGetInfoExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -10975,24 +10543,23 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch) try { +/// @brief Intercept function for urCommandBufferCreateExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_update_kernel_launch_exp_params_t params = { - &hCommandBuffer, &numKernelUpdates, &pUpdateKernelLaunch}; + ur_command_buffer_create_exp_params_t params = { + &hContext, &hDevice, &pCommandBufferDesc, &phCommandBuffer}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferUpdateKernelLaunchExp")); + mock::getCallbacks().get_before_callback("urCommandBufferCreateExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11001,12 +10568,13 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferUpdateKernelLaunchExp")); + mock::getCallbacks().get_replace_callback("urCommandBufferCreateExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + *phCommandBuffer = + mock::createDummyHandle(); result = UR_RESULT_SUCCESS; } @@ -11015,8 +10583,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferUpdateKernelLaunchExp")); + mock::getCallbacks().get_after_callback("urCommandBufferCreateExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11027,20 +10594,16 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateSignalEventExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent) try { +/// @brief Intercept function for urCommandBufferRetainExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_update_signal_event_exp_params_t params = {&hCommand, - &phSignalEvent}; + ur_command_buffer_retain_exp_params_t params = {&hCommandBuffer}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferUpdateSignalEventExp")); + mock::getCallbacks().get_before_callback("urCommandBufferRetainExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11049,13 +10612,12 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferUpdateSignalEventExp")); + mock::getCallbacks().get_replace_callback("urCommandBufferRetainExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - *phSignalEvent = mock::createDummyHandle(); + mock::retainDummyHandle(hCommandBuffer); result = UR_RESULT_SUCCESS; } @@ -11064,8 +10626,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferUpdateSignalEventExp")); + mock::getCallbacks().get_after_callback("urCommandBufferRetainExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11076,24 +10637,16 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList) try { +/// @brief Intercept function for urCommandBufferReleaseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_update_wait_events_exp_params_t params = { - &hCommand, &numEventsInWaitList, &phEventWaitList}; + ur_command_buffer_release_exp_params_t params = {&hCommandBuffer}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback( - "urCommandBufferUpdateWaitEventsExp")); + mock::getCallbacks().get_before_callback("urCommandBufferReleaseExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11102,12 +10655,12 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urCommandBufferUpdateWaitEventsExp")); + mock::getCallbacks().get_replace_callback("urCommandBufferReleaseExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + mock::releaseDummyHandle(hCommandBuffer); result = UR_RESULT_SUCCESS; } @@ -11116,8 +10669,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback( - "urCommandBufferUpdateWaitEventsExp")); + mock::getCallbacks().get_after_callback("urCommandBufferReleaseExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11128,26 +10680,16 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetInfoExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value - size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property - void *pPropValue, - /// [out][optional] bytes returned in command-buffer property - size_t *pPropSizeRet) try { +/// @brief Intercept function for urCommandBufferFinalizeExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_get_info_exp_params_t params = { - &hCommandBuffer, &propName, &propSize, &pPropValue, &pPropSizeRet}; + ur_command_buffer_finalize_exp_params_t params = {&hCommandBuffer}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urCommandBufferGetInfoExp")); + mock::getCallbacks().get_before_callback("urCommandBufferFinalizeExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11156,7 +10698,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urCommandBufferGetInfoExp")); + mock::getCallbacks().get_replace_callback("urCommandBufferFinalizeExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { @@ -11169,7 +10711,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urCommandBufferGetInfoExp")); + mock::getCallbacks().get_after_callback("urCommandBufferFinalizeExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11180,20 +10722,73 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetNativeHandleExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. +/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( + /// [in] Handle of the command-buffer object. ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer) try { + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_command_buffer_get_native_handle_exp_params_t params = { - &hCommandBuffer, &phNativeCommandBuffer}; + ur_command_buffer_append_kernel_launch_exp_params_t params = { + &hCommandBuffer, + &hKernel, + &workDim, + &pGlobalWorkOffset, + &pGlobalWorkSize, + &pLocalWorkSize, + &numKernelAlternatives, + &phKernelAlternatives, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urCommandBufferGetNativeHandleExp")); + "urCommandBufferAppendKernelLaunchExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11203,13 +10798,20 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urCommandBufferGetNativeHandleExp")); + "urCommandBufferAppendKernelLaunchExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - *phNativeCommandBuffer = - reinterpret_cast(hCommandBuffer); + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11219,7 +10821,7 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urCommandBufferGetNativeHandleExp")); + "urCommandBufferAppendKernelLaunchExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11230,16 +10832,53 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urDeviceWaitExp -__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice) try { +/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_device_wait_exp_params_t params = {&hDevice}; + ur_command_buffer_append_usm_memcpy_exp_params_t params = { + &hCommandBuffer, + &pDst, + &pSrc, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urDeviceWaitExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendUSMMemcpyExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11248,58 +10887,21 @@ __urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urDeviceWaitExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendUSMMemcpyExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - result = UR_RESULT_SUCCESS; - } - - if (result != UR_RESULT_SUCCESS) { - return result; - } - - auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urDeviceWaitExp")); - if (afterCallback) { - return afterCallback(¶ms); - } - - return result; -} catch (...) { - return exceptionToResult(std::current_exception()); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramDynamicLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) try { - ur_result_t result = UR_RESULT_SUCCESS; - - ur_program_dynamic_link_exp_params_t params = {&hContext, &count, - &phPrograms}; - - auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urProgramDynamicLinkExp")); - if (beforeCallback) { - result = beforeCallback(¶ms); - if (result != UR_RESULT_SUCCESS) { - return result; + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); } - } - - auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urProgramDynamicLinkExp")); - if (replaceCallback) { - result = replaceCallback(¶ms); - } else { - result = UR_RESULT_SUCCESS; } @@ -11308,7 +10910,8 @@ __urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urProgramDynamicLinkExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendUSMMemcpyExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11319,40 +10922,56 @@ __urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueTimestampRecordingExp -__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// @brief Intercept function for urCommandBufferAppendUSMFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent) try { + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_enqueue_timestamp_recording_exp_params_t params = { - &hQueue, &blocking, &numEventsInWaitList, &phEventWaitList, &phEvent}; + ur_command_buffer_append_usm_fill_exp_params_t params = { + &hCommandBuffer, + &pMemory, + &pPattern, + &patternSize, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urEnqueueTimestampRecordingExp")); + "urCommandBufferAppendUSMFillExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11362,12 +10981,20 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urEnqueueTimestampRecordingExp")); + "urCommandBufferAppendUSMFillExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - *phEvent = mock::createDummyHandle(); + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11377,7 +11004,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urEnqueueTimestampRecordingExp")); + "urCommandBufferAppendUSMFillExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11388,23 +11015,59 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCGetMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_ipc_get_mem_handle_exp_params_t params = { - &hContext, &pMem, &ppIPCMemHandleData, &pIPCMemHandleDataSizeRet}; + ur_command_buffer_append_mem_buffer_copy_exp_params_t params = { + &hCommandBuffer, + &hSrcMem, + &hDstMem, + &srcOffset, + &dstOffset, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urIPCGetMemHandleExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendMemBufferCopyExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11413,11 +11076,21 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urIPCGetMemHandleExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendMemBufferCopyExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11426,7 +11099,8 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urIPCGetMemHandleExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendMemBufferCopyExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11437,18 +11111,56 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCPutMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_ipc_put_mem_handle_exp_params_t params = {&hContext, &pIPCMemHandleData}; + ur_command_buffer_append_mem_buffer_write_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &offset, + &size, + &pSrc, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urIPCPutMemHandleExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendMemBufferWriteExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11457,11 +11169,21 @@ __urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urIPCPutMemHandleExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendMemBufferWriteExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11470,7 +11192,8 @@ __urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urIPCPutMemHandleExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendMemBufferWriteExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11481,26 +11204,56 @@ __urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCOpenMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_ipc_open_mem_handle_exp_params_t params = { - &hContext, &hDevice, &pIPCMemHandleData, &ipcMemHandleDataSize, &ppMem}; + ur_command_buffer_append_mem_buffer_read_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &offset, + &size, + &pDst, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urIPCOpenMemHandleExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendMemBufferReadExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11509,11 +11262,21 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urIPCOpenMemHandleExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendMemBufferReadExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11522,7 +11285,8 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urIPCOpenMemHandleExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendMemBufferReadExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11533,18 +11297,71 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCCloseMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_ipc_close_mem_handle_exp_params_t params = {&hContext, &pMem}; + ur_command_buffer_append_mem_buffer_copy_rect_exp_params_t params = { + &hCommandBuffer, + &hSrcMem, + &hDstMem, + &srcOrigin, + &dstOrigin, + ®ion, + &srcRowPitch, + &srcSlicePitch, + &dstRowPitch, + &dstSlicePitch, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urIPCCloseMemHandleExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendMemBufferCopyRectExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11553,11 +11370,21 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urIPCCloseMemHandleExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendMemBufferCopyRectExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11566,7 +11393,8 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urIPCCloseMemHandleExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendMemBufferCopyRectExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11577,29 +11405,74 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. - size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_memory_export_alloc_exportable_memory_exp_params_t params = { - &hContext, &hDevice, &alignment, &size, &handleTypeToExport, &ppMem}; + ur_command_buffer_append_mem_buffer_write_rect_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &bufferOffset, + &hostOffset, + ®ion, + &bufferRowPitch, + &bufferSlicePitch, + &hostRowPitch, + &hostSlicePitch, + &pSrc, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urMemoryExportAllocExportableMemoryExp")); + "urCommandBufferAppendMemBufferWriteRectExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11609,11 +11482,20 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urMemoryExportAllocExportableMemoryExp")); + "urCommandBufferAppendMemBufferWriteRectExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11623,7 +11505,7 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urMemoryExportAllocExportableMemoryExp")); + "urCommandBufferAppendMemBufferWriteRectExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11634,22 +11516,73 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_memory_export_free_exportable_memory_exp_params_t params = { - &hContext, &hDevice, &pMem}; + ur_command_buffer_append_mem_buffer_read_rect_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &bufferOffset, + &hostOffset, + ®ion, + &bufferRowPitch, + &bufferSlicePitch, + &hostRowPitch, + &hostSlicePitch, + &pDst, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urMemoryExportFreeExportableMemoryExp")); + "urCommandBufferAppendMemBufferReadRectExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11659,11 +11592,20 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urMemoryExportFreeExportableMemoryExp")); + "urCommandBufferAppendMemBufferReadRectExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11673,7 +11615,7 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urMemoryExportFreeExportableMemoryExp")); + "urCommandBufferAppendMemBufferReadRectExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11684,27 +11626,59 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportExportMemoryHandleExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet) try { +/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_memory_export_export_memory_handle_exp_params_t params = { - &hContext, &hDevice, &handleTypeToExport, &pMem, &pMemHandleRet}; + ur_command_buffer_append_mem_buffer_fill_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &pPattern, + &patternSize, + &offset, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urMemoryExportExportMemoryHandleExp")); + "urCommandBufferAppendMemBufferFillExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11714,11 +11688,20 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urMemoryExportExportMemoryHandleExp")); + "urCommandBufferAppendMemBufferFillExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11728,7 +11711,7 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urMemoryExportExportMemoryHandleExp")); + "urCommandBufferAppendMemBufferFillExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11739,25 +11722,53 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramBuildExp -__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) try { +/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_program_build_exp_params_t params = {&hProgram, &numDevices, &phDevices, - &flags, &pOptions}; + ur_command_buffer_append_usm_prefetch_exp_params_t params = { + &hCommandBuffer, + &pMemory, + &size, + &flags, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urProgramBuildExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendUSMPrefetchExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11766,62 +11777,21 @@ __urdlllocal ur_result_t UR_APICALL urProgramBuildExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urProgramBuildExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendUSMPrefetchExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - result = UR_RESULT_SUCCESS; - } - - if (result != UR_RESULT_SUCCESS) { - return result; - } - - auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urProgramBuildExp")); - if (afterCallback) { - return afterCallback(¶ms); - } - - return result; -} catch (...) { - return exceptionToResult(std::current_exception()); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramCompileExp -__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) try { - ur_result_t result = UR_RESULT_SUCCESS; - - ur_program_compile_exp_params_t params = {&hProgram, &numDevices, &phDevices, - &flags, &pOptions}; - - auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urProgramCompileExp")); - if (beforeCallback) { - result = beforeCallback(¶ms); - if (result != UR_RESULT_SUCCESS) { - return result; + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); } - } - - auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urProgramCompileExp")); - if (replaceCallback) { - result = replaceCallback(¶ms); - } else { - result = UR_RESULT_SUCCESS; } @@ -11830,7 +11800,8 @@ __urdlllocal ur_result_t UR_APICALL urProgramCompileExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urProgramCompileExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendUSMPrefetchExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11841,35 +11812,53 @@ __urdlllocal ur_result_t UR_APICALL urProgramCompileExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram) try { +/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { ur_result_t result = UR_RESULT_SUCCESS; - if (nullptr != phProgram) { - *phProgram = nullptr; - } - ur_program_link_exp_params_t params = {&hContext, &numDevices, &phDevices, - &flags, &count, &phPrograms, - &pOptions, &phProgram}; + ur_command_buffer_append_usm_advise_exp_params_t params = { + &hCommandBuffer, + &pMemory, + &size, + &advice, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urProgramLinkExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendUSMAdviseExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11878,12 +11867,21 @@ __urdlllocal ur_result_t UR_APICALL urProgramLinkExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urProgramLinkExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendUSMAdviseExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - *phProgram = mock::createDummyHandle(); + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } + // optional output handle + if (phCommand) { + *phCommand = + mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11892,7 +11890,8 @@ __urdlllocal ur_result_t UR_APICALL urProgramLinkExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urProgramLinkExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendUSMAdviseExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11903,23 +11902,38 @@ __urdlllocal ur_result_t UR_APICALL urProgramLinkExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMContextMemcpyExp -__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size) try { +/// @brief Intercept function for urCommandBufferAppendNativeCommandExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_usm_context_memcpy_exp_params_t params = {&hContext, &pDst, &pSrc, &size}; + ur_command_buffer_append_native_command_exp_params_t params = { + &hCommandBuffer, &pfnNativeCommand, &pData, + &hChildCommandBuffer, &numSyncPointsInWaitList, &pSyncPointWaitList, + &pSyncPoint}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urUSMContextMemcpyExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferAppendNativeCommandExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11928,7 +11942,8 @@ __urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urUSMContextMemcpyExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferAppendNativeCommandExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { @@ -11941,7 +11956,8 @@ __urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urUSMContextMemcpyExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferAppendNativeCommandExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11952,20 +11968,32 @@ __urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMImportExp -__urdlllocal ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size) try { +/// @brief Intercept function for urEnqueueCommandBufferExp +__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_usm_import_exp_params_t params = {&hContext, &pMem, &size}; + ur_enqueue_command_buffer_exp_params_t params = {&hQueue, &hCommandBuffer, + &numEventsInWaitList, + &phEventWaitList, &phEvent}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urUSMImportExp")); + mock::getCallbacks().get_before_callback("urEnqueueCommandBufferExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -11974,11 +12002,15 @@ __urdlllocal ur_result_t UR_APICALL urUSMImportExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urUSMImportExp")); + mock::getCallbacks().get_replace_callback("urEnqueueCommandBufferExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + // optional output handle + if (phEvent) { + *phEvent = mock::createDummyHandle(); + } result = UR_RESULT_SUCCESS; } @@ -11987,7 +12019,7 @@ __urdlllocal ur_result_t UR_APICALL urUSMImportExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urUSMImportExp")); + mock::getCallbacks().get_after_callback("urEnqueueCommandBufferExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -11998,18 +12030,24 @@ __urdlllocal ur_result_t UR_APICALL urUSMImportExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMReleaseExp -__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem) try { +/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_usm_release_exp_params_t params = {&hContext, &pMem}; + ur_command_buffer_update_kernel_launch_exp_params_t params = { + &hCommandBuffer, &numKernelUpdates, &pUpdateKernelLaunch}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urUSMReleaseExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferUpdateKernelLaunchExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -12018,7 +12056,8 @@ __urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urUSMReleaseExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferUpdateKernelLaunchExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { @@ -12031,7 +12070,8 @@ __urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urUSMReleaseExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferUpdateKernelLaunchExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -12042,19 +12082,20 @@ __urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PEnablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) try { +/// @brief Intercept function for urCommandBufferUpdateSignalEventExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_usm_p2p_enable_peer_access_exp_params_t params = {&commandDevice, - &peerDevice}; + ur_command_buffer_update_signal_event_exp_params_t params = {&hCommand, + &phSignalEvent}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urUsmP2PEnablePeerAccessExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferUpdateSignalEventExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -12063,11 +12104,13 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback("urUsmP2PEnablePeerAccessExp")); + mock::getCallbacks().get_replace_callback( + "urCommandBufferUpdateSignalEventExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { + *phSignalEvent = mock::createDummyHandle(); result = UR_RESULT_SUCCESS; } @@ -12076,7 +12119,8 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urUsmP2PEnablePeerAccessExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferUpdateSignalEventExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -12087,19 +12131,24 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PDisablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) try { +/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_usm_p2p_disable_peer_access_exp_params_t params = {&commandDevice, - &peerDevice}; + ur_command_buffer_update_wait_events_exp_params_t params = { + &hCommand, &numEventsInWaitList, &phEventWaitList}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urUsmP2PDisablePeerAccessExp")); + mock::getCallbacks().get_before_callback( + "urCommandBufferUpdateWaitEventsExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -12109,7 +12158,7 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urUsmP2PDisablePeerAccessExp")); + "urCommandBufferUpdateWaitEventsExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { @@ -12122,7 +12171,8 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urUsmP2PDisablePeerAccessExp")); + mock::getCallbacks().get_after_callback( + "urCommandBufferUpdateWaitEventsExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -12133,34 +12183,26 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// @brief Intercept function for urCommandBufferGetInfoExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_usm_p2p_peer_access_get_info_exp_params_t params = { - &commandDevice, &peerDevice, &propName, - &propSize, &pPropValue, &pPropSizeRet}; + ur_command_buffer_get_info_exp_params_t params = { + &hCommandBuffer, &propName, &propSize, &pPropValue, &pPropSizeRet}; auto beforeCallback = reinterpret_cast( - mock::getCallbacks().get_before_callback("urUsmP2PPeerAccessGetInfoExp")); + mock::getCallbacks().get_before_callback("urCommandBufferGetInfoExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -12169,8 +12211,7 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } auto replaceCallback = reinterpret_cast( - mock::getCallbacks().get_replace_callback( - "urUsmP2PPeerAccessGetInfoExp")); + mock::getCallbacks().get_replace_callback("urCommandBufferGetInfoExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { @@ -12183,7 +12224,7 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } auto afterCallback = reinterpret_cast( - mock::getCallbacks().get_after_callback("urUsmP2PPeerAccessGetInfoExp")); + mock::getCallbacks().get_after_callback("urCommandBufferGetInfoExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -12194,59 +12235,20 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp -__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) try { +/// @brief Intercept function for urCommandBufferGetNativeHandleExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer) try { ur_result_t result = UR_RESULT_SUCCESS; - ur_enqueue_kernel_launch_with_args_exp_params_t params = { - &hQueue, &hKernel, - &workDim, &pGlobalWorkOffset, - &pGlobalWorkSize, &pLocalWorkSize, - &numArgs, &pArgs, - &launchPropList, &numEventsInWaitList, - &phEventWaitList, &phEvent}; + ur_command_buffer_get_native_handle_exp_params_t params = { + &hCommandBuffer, &phNativeCommandBuffer}; auto beforeCallback = reinterpret_cast( mock::getCallbacks().get_before_callback( - "urEnqueueKernelLaunchWithArgsExp")); + "urCommandBufferGetNativeHandleExp")); if (beforeCallback) { result = beforeCallback(¶ms); if (result != UR_RESULT_SUCCESS) { @@ -12256,15 +12258,13 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( auto replaceCallback = reinterpret_cast( mock::getCallbacks().get_replace_callback( - "urEnqueueKernelLaunchWithArgsExp")); + "urCommandBufferGetNativeHandleExp")); if (replaceCallback) { result = replaceCallback(¶ms); } else { - // optional output handle - if (phEvent) { - *phEvent = mock::createDummyHandle(); - } + *phNativeCommandBuffer = + reinterpret_cast(hCommandBuffer); result = UR_RESULT_SUCCESS; } @@ -12274,7 +12274,7 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( auto afterCallback = reinterpret_cast( mock::getCallbacks().get_after_callback( - "urEnqueueKernelLaunchWithArgsExp")); + "urCommandBufferGetNativeHandleExp")); if (afterCallback) { return afterCallback(¶ms); } @@ -13373,10 +13373,10 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( pDdiTable->pfnUSMFreeExp = driver::urEnqueueUSMFreeExp; - pDdiTable->pfnCommandBufferExp = driver::urEnqueueCommandBufferExp; - pDdiTable->pfnTimestampRecordingExp = driver::urEnqueueTimestampRecordingExp; + pDdiTable->pfnCommandBufferExp = driver::urEnqueueCommandBufferExp; + pDdiTable->pfnHostTaskExp = driver::urEnqueueHostTaskExp; pDdiTable->pfnNativeCommandExp = driver::urEnqueueNativeCommandExp; diff --git a/unified-runtime/source/common/logger/ur_sinks.hpp b/unified-runtime/source/common/logger/ur_sinks.hpp index fdae2c5f2477f..c8fcf586cc628 100644 --- a/unified-runtime/source/common/logger/ur_sinks.hpp +++ b/unified-runtime/source/common/logger/ur_sinks.hpp @@ -43,17 +43,17 @@ class Sink { std::string message = buffer.str(); -// This is a temporary workaround, where UR adapter is teardowned -// before the UR loader, which will result in access violation when we use print -// function as the overrided print function was already released with the UR -// adapter. -// TODO: Change adapters to use a common sink class in the loader instead of -// using thier own sink class that inherit from logger::Sink. - if (isTearDowned) { - std::cerr << message; - } else { - print(level, message); - } + // This is a temporary workaround, where UR adapter is teardowned + // before the UR loader, which will result in access violation when we use + // print function as the overrided print function was already released with + // the UR adapter. + // TODO: Change adapters to use a common sink class in the loader instead of + // using thier own sink class that inherit from logger::Sink. + if (isTearDowned) { + std::cerr << message; + } else { + print(level, message); + } } void setFileLine(bool fileline) { add_fileline = fileline; } @@ -187,9 +187,7 @@ class StderrSink : public Sink { this->flush_level = flush_lvl; } - ~StderrSink() { - logger::isTearDowned = true; - } + ~StderrSink() { logger::isTearDowned = true; } }; class FileSink : public Sink { diff --git a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp index 283edbe0884c3..eb9db99ad20c0 100644 --- a/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp +++ b/unified-runtime/source/loader/layers/tracing/ur_trcddi.cpp @@ -8004,42 +8004,72 @@ __urdlllocal ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferCreateExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. +/// @brief Intercept function for urDeviceWaitExp +__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice) { + auto pfnWaitExp = getContext()->urDdiTable.DeviceExp.pfnWaitExp; + + if (nullptr == pfnWaitExp) + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + + ur_device_wait_exp_params_t params = {&hDevice}; + uint64_t instance = getContext()->notify_begin(UR_FUNCTION_DEVICE_WAIT_EXP, + "urDeviceWaitExp", ¶ms); + + auto &logger = getContext()->logger; + UR_LOG_L(logger, INFO, " ---> urDeviceWaitExp\n"); + + ur_result_t result = pfnWaitExp(hDevice); + + getContext()->notify_end(UR_FUNCTION_DEVICE_WAIT_EXP, "urDeviceWaitExp", + ¶ms, &result, instance); + + if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { + std::ostringstream args_str; + ur::extras::printFunctionParams(args_str, UR_FUNCTION_DEVICE_WAIT_EXP, + ¶ms); + UR_LOG_L(logger, INFO, " <--- urDeviceWaitExp({}) -> {};\n", + args_str.str(), result); + } + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urProgramDynamicLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer) { - auto pfnCreateExp = getContext()->urDdiTable.CommandBufferExp.pfnCreateExp; + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms) { + auto pfnDynamicLinkExp = + getContext()->urDdiTable.ProgramExp.pfnDynamicLinkExp; - if (nullptr == pfnCreateExp) + if (nullptr == pfnDynamicLinkExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_create_exp_params_t params = { - &hContext, &hDevice, &pCommandBufferDesc, &phCommandBuffer}; - uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_CREATE_EXP, - "urCommandBufferCreateExp", ¶ms); + ur_program_dynamic_link_exp_params_t params = {&hContext, &count, + &phPrograms}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_PROGRAM_DYNAMIC_LINK_EXP, "urProgramDynamicLinkExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferCreateExp\n"); + UR_LOG_L(logger, INFO, " ---> urProgramDynamicLinkExp\n"); - ur_result_t result = - pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); + ur_result_t result = pfnDynamicLinkExp(hContext, count, phPrograms); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_CREATE_EXP, - "urCommandBufferCreateExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_PROGRAM_DYNAMIC_LINK_EXP, + "urProgramDynamicLinkExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_CREATE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urCommandBufferCreateExp({}) -> {};\n", + args_str, UR_FUNCTION_PROGRAM_DYNAMIC_LINK_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urProgramDynamicLinkExp({}) -> {};\n", args_str.str(), result); } @@ -8047,34 +8077,81 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferRetainExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - auto pfnRetainExp = getContext()->urDdiTable.CommandBufferExp.pfnRetainExp; +/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp +__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object + ur_kernel_handle_t hKernel, + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items + uint32_t workDim, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item + const size_t *pGlobalWorkOffset, + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function + const size_t *pGlobalWorkSize, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. + const size_t *pLocalWorkSize, + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { + auto pfnKernelLaunchWithArgsExp = + getContext()->urDdiTable.EnqueueExp.pfnKernelLaunchWithArgsExp; - if (nullptr == pfnRetainExp) + if (nullptr == pfnKernelLaunchWithArgsExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_retain_exp_params_t params = {&hCommandBuffer}; - uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_RETAIN_EXP, - "urCommandBufferRetainExp", ¶ms); + ur_enqueue_kernel_launch_with_args_exp_params_t params = { + &hQueue, &hKernel, + &workDim, &pGlobalWorkOffset, + &pGlobalWorkSize, &pLocalWorkSize, + &numArgs, &pArgs, + &launchPropList, &numEventsInWaitList, + &phEventWaitList, &phEvent}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_ENQUEUE_KERNEL_LAUNCH_WITH_ARGS_EXP, + "urEnqueueKernelLaunchWithArgsExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferRetainExp\n"); + UR_LOG_L(logger, INFO, " ---> urEnqueueKernelLaunchWithArgsExp\n"); - ur_result_t result = pfnRetainExp(hCommandBuffer); + ur_result_t result = pfnKernelLaunchWithArgsExp( + hQueue, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numArgs, pArgs, launchPropList, numEventsInWaitList, + phEventWaitList, phEvent); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_RETAIN_EXP, - "urCommandBufferRetainExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_ENQUEUE_KERNEL_LAUNCH_WITH_ARGS_EXP, + "urEnqueueKernelLaunchWithArgsExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_RETAIN_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urCommandBufferRetainExp({}) -> {};\n", + args_str, UR_FUNCTION_ENQUEUE_KERNEL_LAUNCH_WITH_ARGS_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urEnqueueKernelLaunchWithArgsExp({}) -> {};\n", args_str.str(), result); } @@ -8082,34 +8159,60 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferReleaseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - auto pfnReleaseExp = getContext()->urDdiTable.CommandBufferExp.pfnReleaseExp; +/// @brief Intercept function for urEnqueueTimestampRecordingExp +__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent) { + auto pfnTimestampRecordingExp = + getContext()->urDdiTable.EnqueueExp.pfnTimestampRecordingExp; - if (nullptr == pfnReleaseExp) + if (nullptr == pfnTimestampRecordingExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_release_exp_params_t params = {&hCommandBuffer}; + ur_enqueue_timestamp_recording_exp_params_t params = { + &hQueue, &blocking, &numEventsInWaitList, &phEventWaitList, &phEvent}; uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_RELEASE_EXP, - "urCommandBufferReleaseExp", ¶ms); + getContext()->notify_begin(UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP, + "urEnqueueTimestampRecordingExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferReleaseExp\n"); + UR_LOG_L(logger, INFO, " ---> urEnqueueTimestampRecordingExp\n"); - ur_result_t result = pfnReleaseExp(hCommandBuffer); + ur_result_t result = pfnTimestampRecordingExp( + hQueue, blocking, numEventsInWaitList, phEventWaitList, phEvent); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_RELEASE_EXP, - "urCommandBufferReleaseExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP, + "urEnqueueTimestampRecordingExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_RELEASE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urCommandBufferReleaseExp({}) -> {};\n", + args_str, UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urEnqueueTimestampRecordingExp({}) -> {};\n", args_str.str(), result); } @@ -8117,35 +8220,40 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferFinalizeExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - auto pfnFinalizeExp = - getContext()->urDdiTable.CommandBufferExp.pfnFinalizeExp; +/// @brief Intercept function for urIPCGetMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet) { + auto pfnGetMemHandleExp = getContext()->urDdiTable.IPCExp.pfnGetMemHandleExp; - if (nullptr == pfnFinalizeExp) + if (nullptr == pfnGetMemHandleExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_finalize_exp_params_t params = {&hCommandBuffer}; - uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_FINALIZE_EXP, - "urCommandBufferFinalizeExp", ¶ms); + ur_ipc_get_mem_handle_exp_params_t params = { + &hContext, &pMem, &ppIPCMemHandleData, &pIPCMemHandleDataSizeRet}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_IPC_GET_MEM_HANDLE_EXP, "urIPCGetMemHandleExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferFinalizeExp\n"); + UR_LOG_L(logger, INFO, " ---> urIPCGetMemHandleExp\n"); - ur_result_t result = pfnFinalizeExp(hCommandBuffer); + ur_result_t result = pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, + pIPCMemHandleDataSizeRet); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_FINALIZE_EXP, - "urCommandBufferFinalizeExp", ¶ms, &result, - instance); + getContext()->notify_end(UR_FUNCTION_IPC_GET_MEM_HANDLE_EXP, + "urIPCGetMemHandleExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_FINALIZE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urCommandBufferFinalizeExp({}) -> {};\n", + args_str, UR_FUNCTION_IPC_GET_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCGetMemHandleExp({}) -> {};\n", args_str.str(), result); } @@ -8153,96 +8261,34 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. - ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. - uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. - const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. - const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. - const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendKernelLaunchExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendKernelLaunchExp; +/// @brief Intercept function for urIPCPutMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData) { + auto pfnPutMemHandleExp = getContext()->urDdiTable.IPCExp.pfnPutMemHandleExp; - if (nullptr == pfnAppendKernelLaunchExp) + if (nullptr == pfnPutMemHandleExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_kernel_launch_exp_params_t params = { - &hCommandBuffer, - &hKernel, - &workDim, - &pGlobalWorkOffset, - &pGlobalWorkSize, - &pLocalWorkSize, - &numKernelAlternatives, - &phKernelAlternatives, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_ipc_put_mem_handle_exp_params_t params = {&hContext, &pIPCMemHandleData}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_KERNEL_LAUNCH_EXP, - "urCommandBufferAppendKernelLaunchExp", ¶ms); + UR_FUNCTION_IPC_PUT_MEM_HANDLE_EXP, "urIPCPutMemHandleExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendKernelLaunchExp\n"); + UR_LOG_L(logger, INFO, " ---> urIPCPutMemHandleExp\n"); - ur_result_t result = pfnAppendKernelLaunchExp( - hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnPutMemHandleExp(hContext, pIPCMemHandleData); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_KERNEL_LAUNCH_EXP, - "urCommandBufferAppendKernelLaunchExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_IPC_PUT_MEM_HANDLE_EXP, + "urIPCPutMemHandleExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_KERNEL_LAUNCH_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendKernelLaunchExp({}) -> {};\n", + args_str, UR_FUNCTION_IPC_PUT_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCPutMemHandleExp({}) -> {};\n", args_str.str(), result); } @@ -8250,75 +8296,44 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMMemcpyExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMMemcpyExp; +/// @brief Intercept function for urIPCOpenMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem) { + auto pfnOpenMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenMemHandleExp; - if (nullptr == pfnAppendUSMMemcpyExp) + if (nullptr == pfnOpenMemHandleExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_usm_memcpy_exp_params_t params = { - &hCommandBuffer, - &pDst, - &pSrc, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_ipc_open_mem_handle_exp_params_t params = { + &hContext, &hDevice, &pIPCMemHandleData, &ipcMemHandleDataSize, &ppMem}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_MEMCPY_EXP, - "urCommandBufferAppendUSMMemcpyExp", ¶ms); + UR_FUNCTION_IPC_OPEN_MEM_HANDLE_EXP, "urIPCOpenMemHandleExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMMemcpyExp\n"); + UR_LOG_L(logger, INFO, " ---> urIPCOpenMemHandleExp\n"); - ur_result_t result = pfnAppendUSMMemcpyExp( - hCommandBuffer, pDst, pSrc, size, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, + ipcMemHandleDataSize, ppMem); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_MEMCPY_EXP, - "urCommandBufferAppendUSMMemcpyExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_IPC_OPEN_MEM_HANDLE_EXP, + "urIPCOpenMemHandleExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_MEMCPY_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendUSMMemcpyExp({}) -> {};\n", + args_str, UR_FUNCTION_IPC_OPEN_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCOpenMemHandleExp({}) -> {};\n", args_str.str(), result); } @@ -8326,78 +8341,36 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMFillExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMFillExp; +/// @brief Intercept function for urIPCCloseMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem) { + auto pfnCloseMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnCloseMemHandleExp; - if (nullptr == pfnAppendUSMFillExp) + if (nullptr == pfnCloseMemHandleExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_usm_fill_exp_params_t params = { - &hCommandBuffer, - &pMemory, - &pPattern, - &patternSize, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; - uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_FILL_EXP, - "urCommandBufferAppendUSMFillExp", ¶ms); + ur_ipc_close_mem_handle_exp_params_t params = {&hContext, &pMem}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP, "urIPCCloseMemHandleExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMFillExp\n"); + UR_LOG_L(logger, INFO, " ---> urIPCCloseMemHandleExp\n"); - ur_result_t result = pfnAppendUSMFillExp( - hCommandBuffer, pMemory, pPattern, patternSize, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnCloseMemHandleExp(hContext, pMem); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_FILL_EXP, - "urCommandBufferAppendUSMFillExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP, + "urIPCCloseMemHandleExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_FILL_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendUSMFillExp({}) -> {};\n", + args_str, UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urIPCCloseMemHandleExp({}) -> {};\n", args_str.str(), result); } @@ -8405,82 +8378,50 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferCopyExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyExp; + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { + auto pfnAllocExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; - if (nullptr == pfnAppendMemBufferCopyExp) + if (nullptr == pfnAllocExportableMemoryExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_copy_exp_params_t params = { - &hCommandBuffer, - &hSrcMem, - &hDstMem, - &srcOffset, - &dstOffset, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_memory_export_alloc_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &alignment, &size, &handleTypeToExport, &ppMem}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_EXP, - "urCommandBufferAppendMemBufferCopyExp", ¶ms); + UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, + "urMemoryExportAllocExportableMemoryExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferCopyExp\n"); + UR_LOG_L(logger, INFO, " ---> urMemoryExportAllocExportableMemoryExp\n"); - ur_result_t result = pfnAppendMemBufferCopyExp( - hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnAllocExportableMemoryExp( + hContext, hDevice, alignment, size, handleTypeToExport, ppMem); getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_EXP, - "urCommandBufferAppendMemBufferCopyExp", ¶ms, &result, instance); + UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, + "urMemoryExportAllocExportableMemoryExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_EXP, + args_str, UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferCopyExp({}) -> {};\n", + " <--- urMemoryExportAllocExportableMemoryExp({}) -> {};\n", args_str.str(), result); } @@ -8488,79 +8429,42 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferWriteExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteExp; +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { + auto pfnFreeExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; - if (nullptr == pfnAppendMemBufferWriteExp) + if (nullptr == pfnFreeExportableMemoryExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_write_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &offset, - &size, - &pSrc, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_memory_export_free_exportable_memory_exp_params_t params = { + &hContext, &hDevice, &pMem}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_EXP, - "urCommandBufferAppendMemBufferWriteExp", ¶ms); + UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, + "urMemoryExportFreeExportableMemoryExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferWriteExp\n"); + UR_LOG_L(logger, INFO, " ---> urMemoryExportFreeExportableMemoryExp\n"); - ur_result_t result = pfnAppendMemBufferWriteExp( - hCommandBuffer, hBuffer, offset, size, pSrc, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnFreeExportableMemoryExp(hContext, hDevice, pMem); - getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_EXP, - "urCommandBufferAppendMemBufferWriteExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, + "urMemoryExportFreeExportableMemoryExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_EXP, + args_str, UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferWriteExp({}) -> {};\n", + " <--- urMemoryExportFreeExportableMemoryExp({}) -> {};\n", args_str.str(), result); } @@ -8568,79 +8472,47 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferReadExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadExp; +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { + auto pfnExportMemoryHandleExp = + getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; - if (nullptr == pfnAppendMemBufferReadExp) + if (nullptr == pfnExportMemoryHandleExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_read_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &offset, - &size, - &pDst, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_memory_export_export_memory_handle_exp_params_t params = { + &hContext, &hDevice, &handleTypeToExport, &pMem, &pMemHandleRet}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_EXP, - "urCommandBufferAppendMemBufferReadExp", ¶ms); + UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, + "urMemoryExportExportMemoryHandleExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferReadExp\n"); + UR_LOG_L(logger, INFO, " ---> urMemoryExportExportMemoryHandleExp\n"); - ur_result_t result = pfnAppendMemBufferReadExp( - hCommandBuffer, hBuffer, offset, size, pDst, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnExportMemoryHandleExp( + hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); - getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_EXP, - "urCommandBufferAppendMemBufferReadExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, + "urMemoryExportExportMemoryHandleExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_EXP, - ¶ms); + args_str, UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferReadExp({}) -> {};\n", + " <--- urMemoryExportExportMemoryHandleExp({}) -> {};\n", args_str.str(), result); } @@ -8648,95 +8520,42 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferCopyRectExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyRectExp; +/// @brief Intercept function for urProgramBuildExp +__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { + auto pfnBuildExp = getContext()->urDdiTable.ProgramExp.pfnBuildExp; - if (nullptr == pfnAppendMemBufferCopyRectExp) + if (nullptr == pfnBuildExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_copy_rect_exp_params_t params = { - &hCommandBuffer, - &hSrcMem, - &hDstMem, - &srcOrigin, - &dstOrigin, - ®ion, - &srcRowPitch, - &srcSlicePitch, - &dstRowPitch, - &dstSlicePitch, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_RECT_EXP, - "urCommandBufferAppendMemBufferCopyRectExp", ¶ms); + ur_program_build_exp_params_t params = {&hProgram, &numDevices, &phDevices, + &flags, &pOptions}; + uint64_t instance = getContext()->notify_begin(UR_FUNCTION_PROGRAM_BUILD_EXP, + "urProgramBuildExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferCopyRectExp\n"); + UR_LOG_L(logger, INFO, " ---> urProgramBuildExp\n"); - ur_result_t result = pfnAppendMemBufferCopyRectExp( - hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, - srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = + pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); - getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_RECT_EXP, - "urCommandBufferAppendMemBufferCopyRectExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_PROGRAM_BUILD_EXP, "urProgramBuildExp", + ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_RECT_EXP, - ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferCopyRectExp({}) -> {};\n", + ur::extras::printFunctionParams(args_str, UR_FUNCTION_PROGRAM_BUILD_EXP, + ¶ms); + UR_LOG_L(logger, INFO, " <--- urProgramBuildExp({}) -> {};\n", args_str.str(), result); } @@ -8744,99 +8563,42 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferWriteRectExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteRectExp; +/// @brief Intercept function for urProgramCompileExp +__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { + auto pfnCompileExp = getContext()->urDdiTable.ProgramExp.pfnCompileExp; - if (nullptr == pfnAppendMemBufferWriteRectExp) + if (nullptr == pfnCompileExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_write_rect_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &bufferOffset, - &hostOffset, - ®ion, - &bufferRowPitch, - &bufferSlicePitch, - &hostRowPitch, - &hostSlicePitch, - &pSrc, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_program_compile_exp_params_t params = {&hProgram, &numDevices, &phDevices, + &flags, &pOptions}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_RECT_EXP, - "urCommandBufferAppendMemBufferWriteRectExp", ¶ms); + UR_FUNCTION_PROGRAM_COMPILE_EXP, "urProgramCompileExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, - " ---> urCommandBufferAppendMemBufferWriteRectExp\n"); + UR_LOG_L(logger, INFO, " ---> urProgramCompileExp\n"); - ur_result_t result = pfnAppendMemBufferWriteRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = + pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); - getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_RECT_EXP, - "urCommandBufferAppendMemBufferWriteRectExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_PROGRAM_COMPILE_EXP, + "urProgramCompileExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_RECT_EXP, - ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferWriteRectExp({}) -> {};\n", + ur::extras::printFunctionParams(args_str, UR_FUNCTION_PROGRAM_COMPILE_EXP, + ¶ms); + UR_LOG_L(logger, INFO, " <--- urProgramCompileExp({}) -> {};\n", args_str.str(), result); } @@ -8844,97 +8606,52 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferReadRectExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadRectExp; +/// @brief Intercept function for urProgramLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram) { + if (nullptr != phProgram) { + *phProgram = nullptr; + } + auto pfnLinkExp = getContext()->urDdiTable.ProgramExp.pfnLinkExp; - if (nullptr == pfnAppendMemBufferReadRectExp) + if (nullptr == pfnLinkExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_read_rect_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &bufferOffset, - &hostOffset, - ®ion, - &bufferRowPitch, - &bufferSlicePitch, - &hostRowPitch, - &hostSlicePitch, - &pDst, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_RECT_EXP, - "urCommandBufferAppendMemBufferReadRectExp", ¶ms); + ur_program_link_exp_params_t params = {&hContext, &numDevices, &phDevices, + &flags, &count, &phPrograms, + &pOptions, &phProgram}; + uint64_t instance = getContext()->notify_begin(UR_FUNCTION_PROGRAM_LINK_EXP, + "urProgramLinkExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferReadRectExp\n"); + UR_LOG_L(logger, INFO, " ---> urProgramLinkExp\n"); - ur_result_t result = pfnAppendMemBufferReadRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnLinkExp(hContext, numDevices, phDevices, flags, count, + phPrograms, pOptions, phProgram); - getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_RECT_EXP, - "urCommandBufferAppendMemBufferReadRectExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_PROGRAM_LINK_EXP, "urProgramLinkExp", + ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_RECT_EXP, - ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferReadRectExp({}) -> {};\n", + ur::extras::printFunctionParams(args_str, UR_FUNCTION_PROGRAM_LINK_EXP, + ¶ms); + UR_LOG_L(logger, INFO, " <--- urProgramLinkExp({}) -> {};\n", args_str.str(), result); } @@ -8942,82 +8659,40 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferFillExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferFillExp; +/// @brief Intercept function for urUSMContextMemcpyExp +__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. + void *pDst, + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size) { + auto pfnContextMemcpyExp = + getContext()->urDdiTable.USMExp.pfnContextMemcpyExp; - if (nullptr == pfnAppendMemBufferFillExp) + if (nullptr == pfnContextMemcpyExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_mem_buffer_fill_exp_params_t params = { - &hCommandBuffer, - &hBuffer, - &pPattern, - &patternSize, - &offset, - &size, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; + ur_usm_context_memcpy_exp_params_t params = {&hContext, &pDst, &pSrc, &size}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_FILL_EXP, - "urCommandBufferAppendMemBufferFillExp", ¶ms); + UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP, "urUSMContextMemcpyExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferFillExp\n"); + UR_LOG_L(logger, INFO, " ---> urUSMContextMemcpyExp\n"); - ur_result_t result = pfnAppendMemBufferFillExp( - hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnContextMemcpyExp(hContext, pDst, pSrc, size); - getContext()->notify_end( - UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_FILL_EXP, - "urCommandBufferAppendMemBufferFillExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP, + "urUSMContextMemcpyExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_FILL_EXP, - ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendMemBufferFillExp({}) -> {};\n", + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urUSMContextMemcpyExp({}) -> {};\n", args_str.str(), result); } @@ -9025,75 +8700,36 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMPrefetchExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMPrefetchExp; +/// @brief Intercept function for urUSMImportExp +__urdlllocal ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size) { + auto pfnImportExp = getContext()->urDdiTable.USMExp.pfnImportExp; - if (nullptr == pfnAppendUSMPrefetchExp) + if (nullptr == pfnImportExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_usm_prefetch_exp_params_t params = { - &hCommandBuffer, - &pMemory, - &size, - &flags, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP, - "urCommandBufferAppendUSMPrefetchExp", ¶ms); + ur_usm_import_exp_params_t params = {&hContext, &pMem, &size}; + uint64_t instance = getContext()->notify_begin(UR_FUNCTION_USM_IMPORT_EXP, + "urUSMImportExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMPrefetchExp\n"); + UR_LOG_L(logger, INFO, " ---> urUSMImportExp\n"); - ur_result_t result = pfnAppendUSMPrefetchExp( - hCommandBuffer, pMemory, size, flags, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnImportExp(hContext, pMem, size); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP, - "urCommandBufferAppendUSMPrefetchExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_USM_IMPORT_EXP, "urUSMImportExp", + ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendUSMPrefetchExp({}) -> {};\n", + ur::extras::printFunctionParams(args_str, UR_FUNCTION_USM_IMPORT_EXP, + ¶ms); + UR_LOG_L(logger, INFO, " <--- urUSMImportExp({}) -> {};\n", args_str.str(), result); } @@ -9101,75 +8737,34 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMAdviseExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMAdviseExp; +/// @brief Intercept function for urUSMReleaseExp +__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem) { + auto pfnReleaseExp = getContext()->urDdiTable.USMExp.pfnReleaseExp; - if (nullptr == pfnAppendUSMAdviseExp) + if (nullptr == pfnReleaseExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_usm_advise_exp_params_t params = { - &hCommandBuffer, - &pMemory, - &size, - &advice, - &numSyncPointsInWaitList, - &pSyncPointWaitList, - &numEventsInWaitList, - &phEventWaitList, - &pSyncPoint, - &phEvent, - &phCommand}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP, - "urCommandBufferAppendUSMAdviseExp", ¶ms); + ur_usm_release_exp_params_t params = {&hContext, &pMem}; + uint64_t instance = getContext()->notify_begin(UR_FUNCTION_USM_RELEASE_EXP, + "urUSMReleaseExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMAdviseExp\n"); + UR_LOG_L(logger, INFO, " ---> urUSMReleaseExp\n"); - ur_result_t result = pfnAppendUSMAdviseExp( - hCommandBuffer, pMemory, size, advice, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnReleaseExp(hContext, pMem); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP, - "urCommandBufferAppendUSMAdviseExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_USM_RELEASE_EXP, "urUSMReleaseExp", + ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendUSMAdviseExp({}) -> {};\n", + ur::extras::printFunctionParams(args_str, UR_FUNCTION_USM_RELEASE_EXP, + ¶ms); + UR_LOG_L(logger, INFO, " <--- urUSMReleaseExp({}) -> {};\n", args_str.str(), result); } @@ -9177,60 +8772,38 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendNativeCommandExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint) { - auto pfnAppendNativeCommandExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendNativeCommandExp; +/// @brief Intercept function for urUsmP2PEnablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { + auto pfnEnablePeerAccessExp = + getContext()->urDdiTable.UsmP2PExp.pfnEnablePeerAccessExp; - if (nullptr == pfnAppendNativeCommandExp) + if (nullptr == pfnEnablePeerAccessExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_append_native_command_exp_params_t params = { - &hCommandBuffer, &pfnNativeCommand, &pData, - &hChildCommandBuffer, &numSyncPointsInWaitList, &pSyncPointWaitList, - &pSyncPoint}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_APPEND_NATIVE_COMMAND_EXP, - "urCommandBufferAppendNativeCommandExp", ¶ms); + ur_usm_p2p_enable_peer_access_exp_params_t params = {&commandDevice, + &peerDevice}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP, + "urUsmP2PEnablePeerAccessExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendNativeCommandExp\n"); + UR_LOG_L(logger, INFO, " ---> urUsmP2PEnablePeerAccessExp\n"); - ur_result_t result = pfnAppendNativeCommandExp( - hCommandBuffer, pfnNativeCommand, pData, hChildCommandBuffer, - numSyncPointsInWaitList, pSyncPointWaitList, pSyncPoint); + ur_result_t result = pfnEnablePeerAccessExp(commandDevice, peerDevice); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_NATIVE_COMMAND_EXP, - "urCommandBufferAppendNativeCommandExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP, + "urUsmP2PEnablePeerAccessExp", ¶ms, &result, + instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_NATIVE_COMMAND_EXP, - ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferAppendNativeCommandExp({}) -> {};\n", + args_str, UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urUsmP2PEnablePeerAccessExp({}) -> {};\n", args_str.str(), result); } @@ -9238,52 +8811,38 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueCommandBufferExp -__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { - auto pfnCommandBufferExp = - getContext()->urDdiTable.EnqueueExp.pfnCommandBufferExp; +/// @brief Intercept function for urUsmP2PDisablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { + auto pfnDisablePeerAccessExp = + getContext()->urDdiTable.UsmP2PExp.pfnDisablePeerAccessExp; - if (nullptr == pfnCommandBufferExp) + if (nullptr == pfnDisablePeerAccessExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_enqueue_command_buffer_exp_params_t params = {&hQueue, &hCommandBuffer, - &numEventsInWaitList, - &phEventWaitList, &phEvent}; + ur_usm_p2p_disable_peer_access_exp_params_t params = {&commandDevice, + &peerDevice}; uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP, - "urEnqueueCommandBufferExp", ¶ms); + getContext()->notify_begin(UR_FUNCTION_USM_P2P_DISABLE_PEER_ACCESS_EXP, + "urUsmP2PDisablePeerAccessExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urEnqueueCommandBufferExp\n"); + UR_LOG_L(logger, INFO, " ---> urUsmP2PDisablePeerAccessExp\n"); - ur_result_t result = pfnCommandBufferExp( - hQueue, hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); + ur_result_t result = pfnDisablePeerAccessExp(commandDevice, peerDevice); - getContext()->notify_end(UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP, - "urEnqueueCommandBufferExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_USM_P2P_DISABLE_PEER_ACCESS_EXP, + "urUsmP2PDisablePeerAccessExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urEnqueueCommandBufferExp({}) -> {};\n", + args_str, UR_FUNCTION_USM_P2P_DISABLE_PEER_ACCESS_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urUsmP2PDisablePeerAccessExp({}) -> {};\n", args_str.str(), result); } @@ -9291,44 +8850,54 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch) { - auto pfnUpdateKernelLaunchExp = - getContext()->urDdiTable.CommandBufferExp.pfnUpdateKernelLaunchExp; +/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. + size_t propSize, + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. + void *pPropValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. + size_t *pPropSizeRet) { + auto pfnPeerAccessGetInfoExp = + getContext()->urDdiTable.UsmP2PExp.pfnPeerAccessGetInfoExp; - if (nullptr == pfnUpdateKernelLaunchExp) + if (nullptr == pfnPeerAccessGetInfoExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_update_kernel_launch_exp_params_t params = { - &hCommandBuffer, &numKernelUpdates, &pUpdateKernelLaunch}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP, - "urCommandBufferUpdateKernelLaunchExp", ¶ms); + ur_usm_p2p_peer_access_get_info_exp_params_t params = { + &commandDevice, &peerDevice, &propName, + &propSize, &pPropValue, &pPropSizeRet}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_USM_P2P_PEER_ACCESS_GET_INFO_EXP, + "urUsmP2PPeerAccessGetInfoExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferUpdateKernelLaunchExp\n"); + UR_LOG_L(logger, INFO, " ---> urUsmP2PPeerAccessGetInfoExp\n"); - ur_result_t result = pfnUpdateKernelLaunchExp( - hCommandBuffer, numKernelUpdates, pUpdateKernelLaunch); + ur_result_t result = pfnPeerAccessGetInfoExp( + commandDevice, peerDevice, propName, propSize, pPropValue, pPropSizeRet); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP, - "urCommandBufferUpdateKernelLaunchExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_USM_P2P_PEER_ACCESS_GET_INFO_EXP, + "urUsmP2PPeerAccessGetInfoExp", ¶ms, &result, + instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferUpdateKernelLaunchExp({}) -> {};\n", + args_str, UR_FUNCTION_USM_P2P_PEER_ACCESS_GET_INFO_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urUsmP2PPeerAccessGetInfoExp({}) -> {};\n", args_str.str(), result); } @@ -9336,39 +8905,42 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateSignalEventExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent) { - auto pfnUpdateSignalEventExp = - getContext()->urDdiTable.CommandBufferExp.pfnUpdateSignalEventExp; +/// @brief Intercept function for urCommandBufferCreateExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer) { + auto pfnCreateExp = getContext()->urDdiTable.CommandBufferExp.pfnCreateExp; - if (nullptr == pfnUpdateSignalEventExp) + if (nullptr == pfnCreateExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_update_signal_event_exp_params_t params = {&hCommand, - &phSignalEvent}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP, - "urCommandBufferUpdateSignalEventExp", ¶ms); + ur_command_buffer_create_exp_params_t params = { + &hContext, &hDevice, &pCommandBufferDesc, &phCommandBuffer}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_CREATE_EXP, + "urCommandBufferCreateExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferUpdateSignalEventExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferCreateExp\n"); - ur_result_t result = pfnUpdateSignalEventExp(hCommand, phSignalEvent); + ur_result_t result = + pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP, - "urCommandBufferUpdateSignalEventExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_CREATE_EXP, + "urCommandBufferCreateExp", ¶ms, &result, + instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferUpdateSignalEventExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_CREATE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urCommandBufferCreateExp({}) -> {};\n", args_str.str(), result); } @@ -9376,44 +8948,34 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList) { - auto pfnUpdateWaitEventsExp = - getContext()->urDdiTable.CommandBufferExp.pfnUpdateWaitEventsExp; +/// @brief Intercept function for urCommandBufferRetainExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { + auto pfnRetainExp = getContext()->urDdiTable.CommandBufferExp.pfnRetainExp; - if (nullptr == pfnUpdateWaitEventsExp) + if (nullptr == pfnRetainExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_update_wait_events_exp_params_t params = { - &hCommand, &numEventsInWaitList, &phEventWaitList}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_UPDATE_WAIT_EVENTS_EXP, - "urCommandBufferUpdateWaitEventsExp", ¶ms); + ur_command_buffer_retain_exp_params_t params = {&hCommandBuffer}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_RETAIN_EXP, + "urCommandBufferRetainExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferUpdateWaitEventsExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferRetainExp\n"); - ur_result_t result = - pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); + ur_result_t result = pfnRetainExp(hCommandBuffer); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_UPDATE_WAIT_EVENTS_EXP, - "urCommandBufferUpdateWaitEventsExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_RETAIN_EXP, + "urCommandBufferRetainExp", ¶ms, &result, + instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_UPDATE_WAIT_EVENTS_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferUpdateWaitEventsExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_RETAIN_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urCommandBufferRetainExp({}) -> {};\n", args_str.str(), result); } @@ -9421,45 +8983,34 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetInfoExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value - size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property - void *pPropValue, - /// [out][optional] bytes returned in command-buffer property - size_t *pPropSizeRet) { - auto pfnGetInfoExp = getContext()->urDdiTable.CommandBufferExp.pfnGetInfoExp; +/// @brief Intercept function for urCommandBufferReleaseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { + auto pfnReleaseExp = getContext()->urDdiTable.CommandBufferExp.pfnReleaseExp; - if (nullptr == pfnGetInfoExp) + if (nullptr == pfnReleaseExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_get_info_exp_params_t params = { - &hCommandBuffer, &propName, &propSize, &pPropValue, &pPropSizeRet}; + ur_command_buffer_release_exp_params_t params = {&hCommandBuffer}; uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP, - "urCommandBufferGetInfoExp", ¶ms); + getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_RELEASE_EXP, + "urCommandBufferReleaseExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferGetInfoExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferReleaseExp\n"); - ur_result_t result = pfnGetInfoExp(hCommandBuffer, propName, propSize, - pPropValue, pPropSizeRet); + ur_result_t result = pfnReleaseExp(hCommandBuffer); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP, - "urCommandBufferGetInfoExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_RELEASE_EXP, + "urCommandBufferReleaseExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urCommandBufferGetInfoExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_RELEASE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urCommandBufferReleaseExp({}) -> {};\n", args_str.str(), result); } @@ -9467,40 +9018,35 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetNativeHandleExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer) { - auto pfnGetNativeHandleExp = - getContext()->urDdiTable.CommandBufferExp.pfnGetNativeHandleExp; +/// @brief Intercept function for urCommandBufferFinalizeExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { + auto pfnFinalizeExp = + getContext()->urDdiTable.CommandBufferExp.pfnFinalizeExp; - if (nullptr == pfnGetNativeHandleExp) + if (nullptr == pfnFinalizeExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_command_buffer_get_native_handle_exp_params_t params = { - &hCommandBuffer, &phNativeCommandBuffer}; - uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_COMMAND_BUFFER_GET_NATIVE_HANDLE_EXP, - "urCommandBufferGetNativeHandleExp", ¶ms); + ur_command_buffer_finalize_exp_params_t params = {&hCommandBuffer}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_FINALIZE_EXP, + "urCommandBufferFinalizeExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urCommandBufferGetNativeHandleExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferFinalizeExp\n"); - ur_result_t result = - pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); + ur_result_t result = pfnFinalizeExp(hCommandBuffer); - getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_GET_NATIVE_HANDLE_EXP, - "urCommandBufferGetNativeHandleExp", ¶ms, - &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_FINALIZE_EXP, + "urCommandBufferFinalizeExp", ¶ms, &result, + instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { - std::ostringstream args_str; - ur::extras::printFunctionParams( - args_str, UR_FUNCTION_COMMAND_BUFFER_GET_NATIVE_HANDLE_EXP, ¶ms); - UR_LOG_L(logger, INFO, - " <--- urCommandBufferGetNativeHandleExp({}) -> {};\n", + std::ostringstream args_str; + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_COMMAND_BUFFER_FINALIZE_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urCommandBufferFinalizeExp({}) -> {};\n", args_str.str(), result); } @@ -9508,32 +9054,96 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urDeviceWaitExp -__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice) { - auto pfnWaitExp = getContext()->urDdiTable.DeviceExp.pfnWaitExp; +/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendKernelLaunchExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendKernelLaunchExp; - if (nullptr == pfnWaitExp) + if (nullptr == pfnAppendKernelLaunchExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_device_wait_exp_params_t params = {&hDevice}; - uint64_t instance = getContext()->notify_begin(UR_FUNCTION_DEVICE_WAIT_EXP, - "urDeviceWaitExp", ¶ms); + ur_command_buffer_append_kernel_launch_exp_params_t params = { + &hCommandBuffer, + &hKernel, + &workDim, + &pGlobalWorkOffset, + &pGlobalWorkSize, + &pLocalWorkSize, + &numKernelAlternatives, + &phKernelAlternatives, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_COMMAND_BUFFER_APPEND_KERNEL_LAUNCH_EXP, + "urCommandBufferAppendKernelLaunchExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urDeviceWaitExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendKernelLaunchExp\n"); - ur_result_t result = pfnWaitExp(hDevice); + ur_result_t result = pfnAppendKernelLaunchExp( + hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_DEVICE_WAIT_EXP, "urDeviceWaitExp", - ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_KERNEL_LAUNCH_EXP, + "urCommandBufferAppendKernelLaunchExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams(args_str, UR_FUNCTION_DEVICE_WAIT_EXP, - ¶ms); - UR_LOG_L(logger, INFO, " <--- urDeviceWaitExp({}) -> {};\n", + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_KERNEL_LAUNCH_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendKernelLaunchExp({}) -> {};\n", args_str.str(), result); } @@ -9541,39 +9151,75 @@ __urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramDynamicLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) { - auto pfnDynamicLinkExp = - getContext()->urDdiTable.ProgramExp.pfnDynamicLinkExp; +/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMMemcpyExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMMemcpyExp; - if (nullptr == pfnDynamicLinkExp) + if (nullptr == pfnAppendUSMMemcpyExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_program_dynamic_link_exp_params_t params = {&hContext, &count, - &phPrograms}; + ur_command_buffer_append_usm_memcpy_exp_params_t params = { + &hCommandBuffer, + &pDst, + &pSrc, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_PROGRAM_DYNAMIC_LINK_EXP, "urProgramDynamicLinkExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_MEMCPY_EXP, + "urCommandBufferAppendUSMMemcpyExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urProgramDynamicLinkExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMMemcpyExp\n"); - ur_result_t result = pfnDynamicLinkExp(hContext, count, phPrograms); + ur_result_t result = pfnAppendUSMMemcpyExp( + hCommandBuffer, pDst, pSrc, size, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_PROGRAM_DYNAMIC_LINK_EXP, - "urProgramDynamicLinkExp", ¶ms, &result, - instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_MEMCPY_EXP, + "urCommandBufferAppendUSMMemcpyExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_PROGRAM_DYNAMIC_LINK_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urProgramDynamicLinkExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_MEMCPY_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendUSMMemcpyExp({}) -> {};\n", args_str.str(), result); } @@ -9581,60 +9227,78 @@ __urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueTimestampRecordingExp -__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// @brief Intercept function for urCommandBufferAppendUSMFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent) { - auto pfnTimestampRecordingExp = - getContext()->urDdiTable.EnqueueExp.pfnTimestampRecordingExp; + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMFillExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMFillExp; - if (nullptr == pfnTimestampRecordingExp) + if (nullptr == pfnAppendUSMFillExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_enqueue_timestamp_recording_exp_params_t params = { - &hQueue, &blocking, &numEventsInWaitList, &phEventWaitList, &phEvent}; + ur_command_buffer_append_usm_fill_exp_params_t params = { + &hCommandBuffer, + &pMemory, + &pPattern, + &patternSize, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP, - "urEnqueueTimestampRecordingExp", ¶ms); + getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_FILL_EXP, + "urCommandBufferAppendUSMFillExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urEnqueueTimestampRecordingExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMFillExp\n"); - ur_result_t result = pfnTimestampRecordingExp( - hQueue, blocking, numEventsInWaitList, phEventWaitList, phEvent); + ur_result_t result = pfnAppendUSMFillExp( + hCommandBuffer, pMemory, pPattern, patternSize, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP, - "urEnqueueTimestampRecordingExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_FILL_EXP, + "urCommandBufferAppendUSMFillExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_ENQUEUE_TIMESTAMP_RECORDING_EXP, ¶ms); + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_FILL_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urEnqueueTimestampRecordingExp({}) -> {};\n", + " <--- urCommandBufferAppendUSMFillExp({}) -> {};\n", args_str.str(), result); } @@ -9642,40 +9306,82 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCGetMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet) { - auto pfnGetMemHandleExp = getContext()->urDdiTable.IPCExp.pfnGetMemHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferCopyExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyExp; - if (nullptr == pfnGetMemHandleExp) + if (nullptr == pfnAppendMemBufferCopyExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_ipc_get_mem_handle_exp_params_t params = { - &hContext, &pMem, &ppIPCMemHandleData, &pIPCMemHandleDataSizeRet}; + ur_command_buffer_append_mem_buffer_copy_exp_params_t params = { + &hCommandBuffer, + &hSrcMem, + &hDstMem, + &srcOffset, + &dstOffset, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_IPC_GET_MEM_HANDLE_EXP, "urIPCGetMemHandleExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_EXP, + "urCommandBufferAppendMemBufferCopyExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urIPCGetMemHandleExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferCopyExp\n"); - ur_result_t result = pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, - pIPCMemHandleDataSizeRet); + ur_result_t result = pfnAppendMemBufferCopyExp( + hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_IPC_GET_MEM_HANDLE_EXP, - "urIPCGetMemHandleExp", ¶ms, &result, instance); + getContext()->notify_end( + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_EXP, + "urCommandBufferAppendMemBufferCopyExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_IPC_GET_MEM_HANDLE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urIPCGetMemHandleExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendMemBufferCopyExp({}) -> {};\n", args_str.str(), result); } @@ -9683,34 +9389,79 @@ __urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCPutMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData) { - auto pfnPutMemHandleExp = getContext()->urDdiTable.IPCExp.pfnPutMemHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferWriteExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteExp; - if (nullptr == pfnPutMemHandleExp) + if (nullptr == pfnAppendMemBufferWriteExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_ipc_put_mem_handle_exp_params_t params = {&hContext, &pIPCMemHandleData}; + ur_command_buffer_append_mem_buffer_write_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &offset, + &size, + &pSrc, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_IPC_PUT_MEM_HANDLE_EXP, "urIPCPutMemHandleExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_EXP, + "urCommandBufferAppendMemBufferWriteExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urIPCPutMemHandleExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferWriteExp\n"); - ur_result_t result = pfnPutMemHandleExp(hContext, pIPCMemHandleData); + ur_result_t result = pfnAppendMemBufferWriteExp( + hCommandBuffer, hBuffer, offset, size, pSrc, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_IPC_PUT_MEM_HANDLE_EXP, - "urIPCPutMemHandleExp", ¶ms, &result, instance); + getContext()->notify_end( + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_EXP, + "urCommandBufferAppendMemBufferWriteExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_IPC_PUT_MEM_HANDLE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urIPCPutMemHandleExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendMemBufferWriteExp({}) -> {};\n", args_str.str(), result); } @@ -9718,44 +9469,79 @@ __urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCOpenMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem) { - auto pfnOpenMemHandleExp = - getContext()->urDdiTable.IPCExp.pfnOpenMemHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferReadExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadExp; - if (nullptr == pfnOpenMemHandleExp) + if (nullptr == pfnAppendMemBufferReadExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_ipc_open_mem_handle_exp_params_t params = { - &hContext, &hDevice, &pIPCMemHandleData, &ipcMemHandleDataSize, &ppMem}; + ur_command_buffer_append_mem_buffer_read_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &offset, + &size, + &pDst, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_IPC_OPEN_MEM_HANDLE_EXP, "urIPCOpenMemHandleExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_EXP, + "urCommandBufferAppendMemBufferReadExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urIPCOpenMemHandleExp\n"); - - ur_result_t result = pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, - ipcMemHandleDataSize, ppMem); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferReadExp\n"); - getContext()->notify_end(UR_FUNCTION_IPC_OPEN_MEM_HANDLE_EXP, - "urIPCOpenMemHandleExp", ¶ms, &result, instance); + ur_result_t result = pfnAppendMemBufferReadExp( + hCommandBuffer, hBuffer, offset, size, pDst, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); + + getContext()->notify_end( + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_EXP, + "urCommandBufferAppendMemBufferReadExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_IPC_OPEN_MEM_HANDLE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urIPCOpenMemHandleExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendMemBufferReadExp({}) -> {};\n", args_str.str(), result); } @@ -9763,36 +9549,95 @@ __urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCCloseMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem) { - auto pfnCloseMemHandleExp = - getContext()->urDdiTable.IPCExp.pfnCloseMemHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferCopyRectExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyRectExp; - if (nullptr == pfnCloseMemHandleExp) + if (nullptr == pfnAppendMemBufferCopyRectExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_ipc_close_mem_handle_exp_params_t params = {&hContext, &pMem}; + ur_command_buffer_append_mem_buffer_copy_rect_exp_params_t params = { + &hCommandBuffer, + &hSrcMem, + &hDstMem, + &srcOrigin, + &dstOrigin, + ®ion, + &srcRowPitch, + &srcSlicePitch, + &dstRowPitch, + &dstSlicePitch, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP, "urIPCCloseMemHandleExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_RECT_EXP, + "urCommandBufferAppendMemBufferCopyRectExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urIPCCloseMemHandleExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferCopyRectExp\n"); - ur_result_t result = pfnCloseMemHandleExp(hContext, pMem); + ur_result_t result = pfnAppendMemBufferCopyRectExp( + hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, + srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP, - "urIPCCloseMemHandleExp", ¶ms, &result, - instance); + getContext()->notify_end( + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_RECT_EXP, + "urCommandBufferAppendMemBufferCopyRectExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_IPC_CLOSE_MEM_HANDLE_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urIPCCloseMemHandleExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_COPY_RECT_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendMemBufferCopyRectExp({}) -> {};\n", args_str.str(), result); } @@ -9800,50 +9645,99 @@ __urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. - size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem) { - auto pfnAllocExportableMemoryExp = - getContext()->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferWriteRectExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteRectExp; - if (nullptr == pfnAllocExportableMemoryExp) + if (nullptr == pfnAppendMemBufferWriteRectExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_memory_export_alloc_exportable_memory_exp_params_t params = { - &hContext, &hDevice, &alignment, &size, &handleTypeToExport, &ppMem}; + ur_command_buffer_append_mem_buffer_write_rect_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &bufferOffset, + &hostOffset, + ®ion, + &bufferRowPitch, + &bufferSlicePitch, + &hostRowPitch, + &hostSlicePitch, + &pSrc, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, - "urMemoryExportAllocExportableMemoryExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_RECT_EXP, + "urCommandBufferAppendMemBufferWriteRectExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urMemoryExportAllocExportableMemoryExp\n"); + UR_LOG_L(logger, INFO, + " ---> urCommandBufferAppendMemBufferWriteRectExp\n"); - ur_result_t result = pfnAllocExportableMemoryExp( - hContext, hDevice, alignment, size, handleTypeToExport, ppMem); + ur_result_t result = pfnAppendMemBufferWriteRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); getContext()->notify_end( - UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, - "urMemoryExportAllocExportableMemoryExp", ¶ms, &result, instance); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_RECT_EXP, + "urCommandBufferAppendMemBufferWriteRectExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_MEMORY_EXPORT_ALLOC_EXPORTABLE_MEMORY_EXP, + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_WRITE_RECT_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urMemoryExportAllocExportableMemoryExp({}) -> {};\n", + " <--- urCommandBufferAppendMemBufferWriteRectExp({}) -> {};\n", args_str.str(), result); } @@ -9851,42 +9745,97 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem) { - auto pfnFreeExportableMemoryExp = - getContext()->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferReadRectExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadRectExp; - if (nullptr == pfnFreeExportableMemoryExp) + if (nullptr == pfnAppendMemBufferReadRectExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_memory_export_free_exportable_memory_exp_params_t params = { - &hContext, &hDevice, &pMem}; + ur_command_buffer_append_mem_buffer_read_rect_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &bufferOffset, + &hostOffset, + ®ion, + &bufferRowPitch, + &bufferSlicePitch, + &hostRowPitch, + &hostSlicePitch, + &pDst, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, - "urMemoryExportFreeExportableMemoryExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_RECT_EXP, + "urCommandBufferAppendMemBufferReadRectExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urMemoryExportFreeExportableMemoryExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferReadRectExp\n"); - ur_result_t result = pfnFreeExportableMemoryExp(hContext, hDevice, pMem); + ur_result_t result = pfnAppendMemBufferReadRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, - "urMemoryExportFreeExportableMemoryExp", ¶ms, - &result, instance); + getContext()->notify_end( + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_RECT_EXP, + "urCommandBufferAppendMemBufferReadRectExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_MEMORY_EXPORT_FREE_EXPORTABLE_MEMORY_EXP, + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_READ_RECT_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urMemoryExportFreeExportableMemoryExp({}) -> {};\n", + " <--- urCommandBufferAppendMemBufferReadRectExp({}) -> {};\n", args_str.str(), result); } @@ -9894,90 +9843,82 @@ __urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportExportMemoryHandleExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet) { - auto pfnExportMemoryHandleExp = - getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferFillExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferFillExp; - if (nullptr == pfnExportMemoryHandleExp) + if (nullptr == pfnAppendMemBufferFillExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_memory_export_export_memory_handle_exp_params_t params = { - &hContext, &hDevice, &handleTypeToExport, &pMem, &pMemHandleRet}; + ur_command_buffer_append_mem_buffer_fill_exp_params_t params = { + &hCommandBuffer, + &hBuffer, + &pPattern, + &patternSize, + &offset, + &size, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, - "urMemoryExportExportMemoryHandleExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_FILL_EXP, + "urCommandBufferAppendMemBufferFillExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urMemoryExportExportMemoryHandleExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendMemBufferFillExp\n"); - ur_result_t result = pfnExportMemoryHandleExp( - hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); + ur_result_t result = pfnAppendMemBufferFillExp( + hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, - "urMemoryExportExportMemoryHandleExp", ¶ms, - &result, instance); + getContext()->notify_end( + UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_FILL_EXP, + "urCommandBufferAppendMemBufferFillExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_MEMORY_EXPORT_EXPORT_MEMORY_HANDLE_EXP, ¶ms); + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_MEM_BUFFER_FILL_EXP, + ¶ms); UR_LOG_L(logger, INFO, - " <--- urMemoryExportExportMemoryHandleExp({}) -> {};\n", - args_str.str(), result); - } - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramBuildExp -__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { - auto pfnBuildExp = getContext()->urDdiTable.ProgramExp.pfnBuildExp; - - if (nullptr == pfnBuildExp) - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - - ur_program_build_exp_params_t params = {&hProgram, &numDevices, &phDevices, - &flags, &pOptions}; - uint64_t instance = getContext()->notify_begin(UR_FUNCTION_PROGRAM_BUILD_EXP, - "urProgramBuildExp", ¶ms); - - auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urProgramBuildExp\n"); - - ur_result_t result = - pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); - - getContext()->notify_end(UR_FUNCTION_PROGRAM_BUILD_EXP, "urProgramBuildExp", - ¶ms, &result, instance); - - if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { - std::ostringstream args_str; - ur::extras::printFunctionParams(args_str, UR_FUNCTION_PROGRAM_BUILD_EXP, - ¶ms); - UR_LOG_L(logger, INFO, " <--- urProgramBuildExp({}) -> {};\n", + " <--- urCommandBufferAppendMemBufferFillExp({}) -> {};\n", args_str.str(), result); } @@ -9985,42 +9926,75 @@ __urdlllocal ur_result_t UR_APICALL urProgramBuildExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramCompileExp -__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { - auto pfnCompileExp = getContext()->urDdiTable.ProgramExp.pfnCompileExp; +/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMPrefetchExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMPrefetchExp; - if (nullptr == pfnCompileExp) + if (nullptr == pfnAppendUSMPrefetchExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_program_compile_exp_params_t params = {&hProgram, &numDevices, &phDevices, - &flags, &pOptions}; + ur_command_buffer_append_usm_prefetch_exp_params_t params = { + &hCommandBuffer, + &pMemory, + &size, + &flags, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_PROGRAM_COMPILE_EXP, "urProgramCompileExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP, + "urCommandBufferAppendUSMPrefetchExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urProgramCompileExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMPrefetchExp\n"); - ur_result_t result = - pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); + ur_result_t result = pfnAppendUSMPrefetchExp( + hCommandBuffer, pMemory, size, flags, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_PROGRAM_COMPILE_EXP, - "urProgramCompileExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP, + "urCommandBufferAppendUSMPrefetchExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams(args_str, UR_FUNCTION_PROGRAM_COMPILE_EXP, - ¶ms); - UR_LOG_L(logger, INFO, " <--- urProgramCompileExp({}) -> {};\n", + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_PREFETCH_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendUSMPrefetchExp({}) -> {};\n", args_str.str(), result); } @@ -10028,52 +10002,75 @@ __urdlllocal ur_result_t UR_APICALL urProgramCompileExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram) { - if (nullptr != phProgram) { - *phProgram = nullptr; - } - auto pfnLinkExp = getContext()->urDdiTable.ProgramExp.pfnLinkExp; +/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMAdviseExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMAdviseExp; - if (nullptr == pfnLinkExp) + if (nullptr == pfnAppendUSMAdviseExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_program_link_exp_params_t params = {&hContext, &numDevices, &phDevices, - &flags, &count, &phPrograms, - &pOptions, &phProgram}; - uint64_t instance = getContext()->notify_begin(UR_FUNCTION_PROGRAM_LINK_EXP, - "urProgramLinkExp", ¶ms); + ur_command_buffer_append_usm_advise_exp_params_t params = { + &hCommandBuffer, + &pMemory, + &size, + &advice, + &numSyncPointsInWaitList, + &pSyncPointWaitList, + &numEventsInWaitList, + &phEventWaitList, + &pSyncPoint, + &phEvent, + &phCommand}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP, + "urCommandBufferAppendUSMAdviseExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urProgramLinkExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendUSMAdviseExp\n"); - ur_result_t result = pfnLinkExp(hContext, numDevices, phDevices, flags, count, - phPrograms, pOptions, phProgram); + ur_result_t result = pfnAppendUSMAdviseExp( + hCommandBuffer, pMemory, size, advice, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); - getContext()->notify_end(UR_FUNCTION_PROGRAM_LINK_EXP, "urProgramLinkExp", - ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP, + "urCommandBufferAppendUSMAdviseExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams(args_str, UR_FUNCTION_PROGRAM_LINK_EXP, - ¶ms); - UR_LOG_L(logger, INFO, " <--- urProgramLinkExp({}) -> {};\n", + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_USM_ADVISE_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendUSMAdviseExp({}) -> {};\n", args_str.str(), result); } @@ -10081,40 +10078,60 @@ __urdlllocal ur_result_t UR_APICALL urProgramLinkExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMContextMemcpyExp -__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size) { - auto pfnContextMemcpyExp = - getContext()->urDdiTable.USMExp.pfnContextMemcpyExp; +/// @brief Intercept function for urCommandBufferAppendNativeCommandExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint) { + auto pfnAppendNativeCommandExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendNativeCommandExp; - if (nullptr == pfnContextMemcpyExp) + if (nullptr == pfnAppendNativeCommandExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_usm_context_memcpy_exp_params_t params = {&hContext, &pDst, &pSrc, &size}; + ur_command_buffer_append_native_command_exp_params_t params = { + &hCommandBuffer, &pfnNativeCommand, &pData, + &hChildCommandBuffer, &numSyncPointsInWaitList, &pSyncPointWaitList, + &pSyncPoint}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP, "urUSMContextMemcpyExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_APPEND_NATIVE_COMMAND_EXP, + "urCommandBufferAppendNativeCommandExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urUSMContextMemcpyExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferAppendNativeCommandExp\n"); - ur_result_t result = pfnContextMemcpyExp(hContext, pDst, pSrc, size); + ur_result_t result = pfnAppendNativeCommandExp( + hCommandBuffer, pfnNativeCommand, pData, hChildCommandBuffer, + numSyncPointsInWaitList, pSyncPointWaitList, pSyncPoint); - getContext()->notify_end(UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP, - "urUSMContextMemcpyExp", ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_APPEND_NATIVE_COMMAND_EXP, + "urCommandBufferAppendNativeCommandExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_USM_CONTEXT_MEMCPY_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urUSMContextMemcpyExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_APPEND_NATIVE_COMMAND_EXP, + ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferAppendNativeCommandExp({}) -> {};\n", args_str.str(), result); } @@ -10122,36 +10139,52 @@ __urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMImportExp -__urdlllocal ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size) { - auto pfnImportExp = getContext()->urDdiTable.USMExp.pfnImportExp; +/// @brief Intercept function for urEnqueueCommandBufferExp +__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { + auto pfnCommandBufferExp = + getContext()->urDdiTable.EnqueueExp.pfnCommandBufferExp; - if (nullptr == pfnImportExp) + if (nullptr == pfnCommandBufferExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_usm_import_exp_params_t params = {&hContext, &pMem, &size}; - uint64_t instance = getContext()->notify_begin(UR_FUNCTION_USM_IMPORT_EXP, - "urUSMImportExp", ¶ms); + ur_enqueue_command_buffer_exp_params_t params = {&hQueue, &hCommandBuffer, + &numEventsInWaitList, + &phEventWaitList, &phEvent}; + uint64_t instance = + getContext()->notify_begin(UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP, + "urEnqueueCommandBufferExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urUSMImportExp\n"); + UR_LOG_L(logger, INFO, " ---> urEnqueueCommandBufferExp\n"); - ur_result_t result = pfnImportExp(hContext, pMem, size); + ur_result_t result = pfnCommandBufferExp( + hQueue, hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); - getContext()->notify_end(UR_FUNCTION_USM_IMPORT_EXP, "urUSMImportExp", - ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP, + "urEnqueueCommandBufferExp", ¶ms, &result, + instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams(args_str, UR_FUNCTION_USM_IMPORT_EXP, - ¶ms); - UR_LOG_L(logger, INFO, " <--- urUSMImportExp({}) -> {};\n", + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_ENQUEUE_COMMAND_BUFFER_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urEnqueueCommandBufferExp({}) -> {};\n", args_str.str(), result); } @@ -10159,34 +10192,44 @@ __urdlllocal ur_result_t UR_APICALL urUSMImportExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMReleaseExp -__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem) { - auto pfnReleaseExp = getContext()->urDdiTable.USMExp.pfnReleaseExp; +/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch) { + auto pfnUpdateKernelLaunchExp = + getContext()->urDdiTable.CommandBufferExp.pfnUpdateKernelLaunchExp; - if (nullptr == pfnReleaseExp) + if (nullptr == pfnUpdateKernelLaunchExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_usm_release_exp_params_t params = {&hContext, &pMem}; - uint64_t instance = getContext()->notify_begin(UR_FUNCTION_USM_RELEASE_EXP, - "urUSMReleaseExp", ¶ms); + ur_command_buffer_update_kernel_launch_exp_params_t params = { + &hCommandBuffer, &numKernelUpdates, &pUpdateKernelLaunch}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP, + "urCommandBufferUpdateKernelLaunchExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urUSMReleaseExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferUpdateKernelLaunchExp\n"); - ur_result_t result = pfnReleaseExp(hContext, pMem); + ur_result_t result = pfnUpdateKernelLaunchExp( + hCommandBuffer, numKernelUpdates, pUpdateKernelLaunch); - getContext()->notify_end(UR_FUNCTION_USM_RELEASE_EXP, "urUSMReleaseExp", - ¶ms, &result, instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP, + "urCommandBufferUpdateKernelLaunchExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; - ur::extras::printFunctionParams(args_str, UR_FUNCTION_USM_RELEASE_EXP, - ¶ms); - UR_LOG_L(logger, INFO, " <--- urUSMReleaseExp({}) -> {};\n", + ur::extras::printFunctionParams( + args_str, UR_FUNCTION_COMMAND_BUFFER_UPDATE_KERNEL_LAUNCH_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferUpdateKernelLaunchExp({}) -> {};\n", args_str.str(), result); } @@ -10194,38 +10237,39 @@ __urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PEnablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { - auto pfnEnablePeerAccessExp = - getContext()->urDdiTable.UsmP2PExp.pfnEnablePeerAccessExp; +/// @brief Intercept function for urCommandBufferUpdateSignalEventExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent) { + auto pfnUpdateSignalEventExp = + getContext()->urDdiTable.CommandBufferExp.pfnUpdateSignalEventExp; - if (nullptr == pfnEnablePeerAccessExp) + if (nullptr == pfnUpdateSignalEventExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_usm_p2p_enable_peer_access_exp_params_t params = {&commandDevice, - &peerDevice}; - uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP, - "urUsmP2PEnablePeerAccessExp", ¶ms); + ur_command_buffer_update_signal_event_exp_params_t params = {&hCommand, + &phSignalEvent}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP, + "urCommandBufferUpdateSignalEventExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urUsmP2PEnablePeerAccessExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferUpdateSignalEventExp\n"); - ur_result_t result = pfnEnablePeerAccessExp(commandDevice, peerDevice); + ur_result_t result = pfnUpdateSignalEventExp(hCommand, phSignalEvent); - getContext()->notify_end(UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP, - "urUsmP2PEnablePeerAccessExp", ¶ms, &result, - instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP, + "urCommandBufferUpdateSignalEventExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_USM_P2P_ENABLE_PEER_ACCESS_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urUsmP2PEnablePeerAccessExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_UPDATE_SIGNAL_EVENT_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferUpdateSignalEventExp({}) -> {};\n", args_str.str(), result); } @@ -10233,38 +10277,44 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PDisablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { - auto pfnDisablePeerAccessExp = - getContext()->urDdiTable.UsmP2PExp.pfnDisablePeerAccessExp; +/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList) { + auto pfnUpdateWaitEventsExp = + getContext()->urDdiTable.CommandBufferExp.pfnUpdateWaitEventsExp; - if (nullptr == pfnDisablePeerAccessExp) + if (nullptr == pfnUpdateWaitEventsExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_usm_p2p_disable_peer_access_exp_params_t params = {&commandDevice, - &peerDevice}; - uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_USM_P2P_DISABLE_PEER_ACCESS_EXP, - "urUsmP2PDisablePeerAccessExp", ¶ms); + ur_command_buffer_update_wait_events_exp_params_t params = { + &hCommand, &numEventsInWaitList, &phEventWaitList}; + uint64_t instance = getContext()->notify_begin( + UR_FUNCTION_COMMAND_BUFFER_UPDATE_WAIT_EVENTS_EXP, + "urCommandBufferUpdateWaitEventsExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urUsmP2PDisablePeerAccessExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferUpdateWaitEventsExp\n"); - ur_result_t result = pfnDisablePeerAccessExp(commandDevice, peerDevice); + ur_result_t result = + pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); - getContext()->notify_end(UR_FUNCTION_USM_P2P_DISABLE_PEER_ACCESS_EXP, - "urUsmP2PDisablePeerAccessExp", ¶ms, &result, - instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_UPDATE_WAIT_EVENTS_EXP, + "urCommandBufferUpdateWaitEventsExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_USM_P2P_DISABLE_PEER_ACCESS_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urUsmP2PDisablePeerAccessExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_UPDATE_WAIT_EVENTS_EXP, ¶ms); + UR_LOG_L(logger, INFO, + " <--- urCommandBufferUpdateWaitEventsExp({}) -> {};\n", args_str.str(), result); } @@ -10272,54 +10322,45 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// @brief Intercept function for urCommandBufferGetInfoExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet) { - auto pfnPeerAccessGetInfoExp = - getContext()->urDdiTable.UsmP2PExp.pfnPeerAccessGetInfoExp; + auto pfnGetInfoExp = getContext()->urDdiTable.CommandBufferExp.pfnGetInfoExp; - if (nullptr == pfnPeerAccessGetInfoExp) + if (nullptr == pfnGetInfoExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_usm_p2p_peer_access_get_info_exp_params_t params = { - &commandDevice, &peerDevice, &propName, - &propSize, &pPropValue, &pPropSizeRet}; + ur_command_buffer_get_info_exp_params_t params = { + &hCommandBuffer, &propName, &propSize, &pPropValue, &pPropSizeRet}; uint64_t instance = - getContext()->notify_begin(UR_FUNCTION_USM_P2P_PEER_ACCESS_GET_INFO_EXP, - "urUsmP2PPeerAccessGetInfoExp", ¶ms); + getContext()->notify_begin(UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP, + "urCommandBufferGetInfoExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urUsmP2PPeerAccessGetInfoExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferGetInfoExp\n"); - ur_result_t result = pfnPeerAccessGetInfoExp( - commandDevice, peerDevice, propName, propSize, pPropValue, pPropSizeRet); + ur_result_t result = pfnGetInfoExp(hCommandBuffer, propName, propSize, + pPropValue, pPropSizeRet); - getContext()->notify_end(UR_FUNCTION_USM_P2P_PEER_ACCESS_GET_INFO_EXP, - "urUsmP2PPeerAccessGetInfoExp", ¶ms, &result, + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP, + "urCommandBufferGetInfoExp", ¶ms, &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_USM_P2P_PEER_ACCESS_GET_INFO_EXP, ¶ms); - UR_LOG_L(logger, INFO, " <--- urUsmP2PPeerAccessGetInfoExp({}) -> {};\n", + args_str, UR_FUNCTION_COMMAND_BUFFER_GET_INFO_EXP, ¶ms); + UR_LOG_L(logger, INFO, " <--- urCommandBufferGetInfoExp({}) -> {};\n", args_str.str(), result); } @@ -10327,81 +10368,40 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp -__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { - auto pfnKernelLaunchWithArgsExp = - getContext()->urDdiTable.EnqueueExp.pfnKernelLaunchWithArgsExp; +/// @brief Intercept function for urCommandBufferGetNativeHandleExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer) { + auto pfnGetNativeHandleExp = + getContext()->urDdiTable.CommandBufferExp.pfnGetNativeHandleExp; - if (nullptr == pfnKernelLaunchWithArgsExp) + if (nullptr == pfnGetNativeHandleExp) return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - ur_enqueue_kernel_launch_with_args_exp_params_t params = { - &hQueue, &hKernel, - &workDim, &pGlobalWorkOffset, - &pGlobalWorkSize, &pLocalWorkSize, - &numArgs, &pArgs, - &launchPropList, &numEventsInWaitList, - &phEventWaitList, &phEvent}; + ur_command_buffer_get_native_handle_exp_params_t params = { + &hCommandBuffer, &phNativeCommandBuffer}; uint64_t instance = getContext()->notify_begin( - UR_FUNCTION_ENQUEUE_KERNEL_LAUNCH_WITH_ARGS_EXP, - "urEnqueueKernelLaunchWithArgsExp", ¶ms); + UR_FUNCTION_COMMAND_BUFFER_GET_NATIVE_HANDLE_EXP, + "urCommandBufferGetNativeHandleExp", ¶ms); auto &logger = getContext()->logger; - UR_LOG_L(logger, INFO, " ---> urEnqueueKernelLaunchWithArgsExp\n"); + UR_LOG_L(logger, INFO, " ---> urCommandBufferGetNativeHandleExp\n"); - ur_result_t result = pfnKernelLaunchWithArgsExp( - hQueue, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numArgs, pArgs, launchPropList, numEventsInWaitList, - phEventWaitList, phEvent); + ur_result_t result = + pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); - getContext()->notify_end(UR_FUNCTION_ENQUEUE_KERNEL_LAUNCH_WITH_ARGS_EXP, - "urEnqueueKernelLaunchWithArgsExp", ¶ms, &result, - instance); + getContext()->notify_end(UR_FUNCTION_COMMAND_BUFFER_GET_NATIVE_HANDLE_EXP, + "urCommandBufferGetNativeHandleExp", ¶ms, + &result, instance); if (logger.getLevel() <= UR_LOGGER_LEVEL_INFO) { std::ostringstream args_str; ur::extras::printFunctionParams( - args_str, UR_FUNCTION_ENQUEUE_KERNEL_LAUNCH_WITH_ARGS_EXP, ¶ms); + args_str, UR_FUNCTION_COMMAND_BUFFER_GET_NATIVE_HANDLE_EXP, ¶ms); UR_LOG_L(logger, INFO, - " <--- urEnqueueKernelLaunchWithArgsExp({}) -> {};\n", + " <--- urCommandBufferGetNativeHandleExp({}) -> {};\n", args_str.str(), result); } @@ -11510,13 +11510,13 @@ __urdlllocal ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( dditable.pfnUSMFreeExp = pDdiTable->pfnUSMFreeExp; pDdiTable->pfnUSMFreeExp = ur_tracing_layer::urEnqueueUSMFreeExp; - dditable.pfnCommandBufferExp = pDdiTable->pfnCommandBufferExp; - pDdiTable->pfnCommandBufferExp = ur_tracing_layer::urEnqueueCommandBufferExp; - dditable.pfnTimestampRecordingExp = pDdiTable->pfnTimestampRecordingExp; pDdiTable->pfnTimestampRecordingExp = ur_tracing_layer::urEnqueueTimestampRecordingExp; + dditable.pfnCommandBufferExp = pDdiTable->pfnCommandBufferExp; + pDdiTable->pfnCommandBufferExp = ur_tracing_layer::urEnqueueCommandBufferExp; + dditable.pfnHostTaskExp = pDdiTable->pfnHostTaskExp; pDdiTable->pfnHostTaskExp = ur_tracing_layer::urEnqueueHostTaskExp; diff --git a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp index 569545718f69e..a2f0b00f79a44 100644 --- a/unified-runtime/source/loader/layers/validation/ur_valddi.cpp +++ b/unified-runtime/source/loader/layers/validation/ur_valddi.cpp @@ -8801,39 +8801,22 @@ __urdlllocal ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferCreateExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. - ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer) { - auto pfnCreateExp = getContext()->urDdiTable.CommandBufferExp.pfnCreateExp; +/// @brief Intercept function for urDeviceWaitExp +__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice) { + auto pfnWaitExp = getContext()->urDdiTable.DeviceExp.pfnWaitExp; - if (nullptr == pfnCreateExp) { + if (nullptr == pfnWaitExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pCommandBufferDesc) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == phCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == hContext) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hDevice) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + if (hDevice == nullptr) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } if (getContext()->enableLifetimeValidation && @@ -8841,126 +8824,93 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( URLOG_CTX_INVALID_REFERENCE(hDevice); } - ur_result_t result = - pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); + ur_result_t result = pfnWaitExp(hDevice); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferRetainExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - auto pfnRetainExp = getContext()->urDdiTable.CommandBufferExp.pfnRetainExp; +/// @brief Intercept function for urProgramDynamicLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms) { + auto pfnDynamicLinkExp = + getContext()->urDdiTable.ProgramExp.pfnDynamicLinkExp; - if (nullptr == pfnRetainExp) { + if (nullptr == pfnDynamicLinkExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - ur_result_t result = pfnRetainExp(hCommandBuffer); - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferReleaseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - auto pfnReleaseExp = getContext()->urDdiTable.CommandBufferExp.pfnReleaseExp; - - if (nullptr == pfnReleaseExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } + if (NULL == phPrograms) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (getContext()->enableParameterValidation) { - if (NULL == hCommandBuffer) + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - ur_result_t result = pfnReleaseExp(hCommandBuffer); - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferFinalizeExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - auto pfnFinalizeExp = - getContext()->urDdiTable.CommandBufferExp.pfnFinalizeExp; - - if (nullptr == pfnFinalizeExp) { - return UR_RESULT_ERROR_UNINITIALIZED; + if (count == 0) + return UR_RESULT_ERROR_INVALID_SIZE; } - if (getContext()->enableParameterValidation) { - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } - ur_result_t result = pfnFinalizeExp(hCommandBuffer); + ur_result_t result = pfnDynamicLinkExp(hContext, count, phPrograms); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. +/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp +__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendKernelLaunchExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendKernelLaunchExp; + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { + auto pfnKernelLaunchWithArgsExp = + getContext()->urDdiTable.EnqueueExp.pfnKernelLaunchWithArgsExp; - if (nullptr == pfnAppendKernelLaunchExp) { + if (nullptr == pfnKernelLaunchWithArgsExp) { return UR_RESULT_ERROR_UNINITIALIZED; } @@ -8968,23 +8918,21 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( if (NULL == pGlobalWorkSize) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) + if (pArgs == NULL && numArgs > 0) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hQueue) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; if (NULL == hKernel) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (phKernelAlternatives == NULL && numKernelAlternatives > 0) - return UR_RESULT_ERROR_INVALID_VALUE; - - if (phKernelAlternatives != NULL && numKernelAlternatives == 0) - return UR_RESULT_ERROR_INVALID_VALUE; - - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (NULL != pArgs && UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type) + return UR_RESULT_ERROR_INVALID_ENUMERATION; - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (NULL != launchPropList && + UR_KERNEL_LAUNCH_FLAGS_MASK & launchPropList->flags) + return UR_RESULT_ERROR_INVALID_ENUMERATION; if (phEventWaitList == NULL && numEventsInWaitList > 0) return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; @@ -8992,6 +8940,14 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( if (phEventWaitList != NULL && numEventsInWaitList == 0) return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || + pGlobalWorkSize[2] == 0) + return UR_RESULT_ERROR_INVALID_WORK_DIMENSION; + + if (pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == 0 || + pLocalWorkSize[2] == 0)) + return UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE; + if (phEventWaitList != NULL && numEventsInWaitList > 0) { for (uint32_t i = 0; i < numEventsInWaitList; ++i) { if (phEventWaitList[i] == NULL) { @@ -9001,76 +8957,70 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( } } + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hQueue)) { + URLOG_CTX_INVALID_REFERENCE(hQueue); + } + if (getContext()->enableLifetimeValidation && !getContext()->refCountContext->isReferenceValid(hKernel)) { URLOG_CTX_INVALID_REFERENCE(hKernel); } - ur_result_t result = pfnAppendKernelLaunchExp( - hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnKernelLaunchWithArgsExp( + hQueue, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numArgs, pArgs, launchPropList, numEventsInWaitList, + phEventWaitList, phEvent); + + if (getContext()->enableLeakChecking && result == UR_RESULT_SUCCESS && + phEvent) { + getContext()->refCountContext->createRefCount(*phEvent); + } return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. +/// @brief Intercept function for urEnqueueTimestampRecordingExp +__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMMemcpyExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMMemcpyExp; + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent) { + auto pfnTimestampRecordingExp = + getContext()->urDdiTable.EnqueueExp.pfnTimestampRecordingExp; - if (nullptr == pfnAppendUSMMemcpyExp) { + if (nullptr == pfnTimestampRecordingExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pDst) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == pSrc) + if (NULL == phEvent) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) + if (NULL == hQueue) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (size == 0) - return UR_RESULT_ERROR_INVALID_SIZE; - - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (phEventWaitList == NULL && numEventsInWaitList > 0) return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; @@ -9086,601 +9036,484 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( } } - ur_result_t result = pfnAppendUSMMemcpyExp( - hCommandBuffer, pDst, pSrc, size, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hQueue)) { + URLOG_CTX_INVALID_REFERENCE(hQueue); + } + + ur_result_t result = pfnTimestampRecordingExp( + hQueue, blocking, numEventsInWaitList, phEventWaitList, phEvent); + + if (getContext()->enableLeakChecking && result == UR_RESULT_SUCCESS && + phEvent) { + getContext()->refCountContext->createRefCount(*phEvent); + } return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMFillExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMFillExp; +/// @brief Intercept function for urIPCGetMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet) { + auto pfnGetMemHandleExp = getContext()->urDdiTable.IPCExp.pfnGetMemHandleExp; - if (nullptr == pfnAppendUSMFillExp) { + if (nullptr == pfnGetMemHandleExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pMemory) + if (NULL == ppIPCMemHandleData) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == pPattern) + if (NULL == pIPCMemHandleDataSizeRet) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (patternSize == 0 || size == 0) - return UR_RESULT_ERROR_INVALID_SIZE; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } - if (patternSize > size) - return UR_RESULT_ERROR_INVALID_SIZE; + ur_result_t result = pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, + pIPCMemHandleDataSizeRet); - if (size % patternSize != 0) - return UR_RESULT_ERROR_INVALID_SIZE; + return result; +} - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urIPCPutMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData) { + auto pfnPutMemHandleExp = getContext()->urDdiTable.IPCExp.pfnPutMemHandleExp; - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (nullptr == pfnPutMemHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (getContext()->enableParameterValidation) { + if (NULL == pIPCMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } - ur_result_t result = pfnAppendUSMFillExp( - hCommandBuffer, pMemory, pPattern, patternSize, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnPutMemHandleExp(hContext, pIPCMemHandleData); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferCopyExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyExp; +/// @brief Intercept function for urIPCOpenMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem) { + auto pfnOpenMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnOpenMemHandleExp; - if (nullptr == pfnAppendMemBufferCopyExp) { + if (nullptr == pfnOpenMemHandleExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (NULL == pIPCMemHandleData) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hSrcMem) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (NULL == ppMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hDstMem) + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hSrcMem)) { - URLOG_CTX_INVALID_REFERENCE(hSrcMem); + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDstMem)) { - URLOG_CTX_INVALID_REFERENCE(hDstMem); + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); } - ur_result_t result = pfnAppendMemBufferCopyExp( - hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, + ipcMemHandleDataSize, ppMem); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferWriteExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteExp; +/// @brief Intercept function for urIPCCloseMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem) { + auto pfnCloseMemHandleExp = + getContext()->urDdiTable.IPCExp.pfnCloseMemHandleExp; - if (nullptr == pfnAppendMemBufferWriteExp) { + if (nullptr == pfnCloseMemHandleExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pSrc) + if (NULL == pMem) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (NULL == hBuffer) + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hBuffer)) { - URLOG_CTX_INVALID_REFERENCE(hBuffer); + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } - ur_result_t result = pfnAppendMemBufferWriteExp( - hCommandBuffer, hBuffer, offset, size, pSrc, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnCloseMemHandleExp(hContext, pMem); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferReadExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadExp; + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { + auto pfnAllocExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; - if (nullptr == pfnAppendMemBufferReadExp) { + if (nullptr == pfnAllocExportableMemoryExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pDst) + if (NULL == ppMem) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hBuffer) + if (NULL == hDevice) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < handleTypeToExport) + return UR_RESULT_ERROR_INVALID_ENUMERATION; - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (alignment != 0 && ((alignment & (alignment - 1)) != 0)) + return UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT; - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (size == 0) + return UR_RESULT_ERROR_INVALID_USM_SIZE; + } - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hBuffer)) { - URLOG_CTX_INVALID_REFERENCE(hBuffer); + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); } - ur_result_t result = pfnAppendMemBufferReadExp( - hCommandBuffer, hBuffer, offset, size, pDst, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnAllocExportableMemoryExp( + hContext, hDevice, alignment, size, handleTypeToExport, ppMem); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferCopyRectExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyRectExp; +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { + auto pfnFreeExportableMemoryExp = + getContext()->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; - if (nullptr == pfnAppendMemBufferCopyRectExp) { + if (nullptr == pfnFreeExportableMemoryExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (NULL == pMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hSrcMem) + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hDstMem) + if (NULL == hDevice) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); + } - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + ur_result_t result = pfnFreeExportableMemoryExp(hContext, hDevice, pMem); - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + return result; +} - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { + auto pfnExportMemoryHandleExp = + getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; + + if (nullptr == pfnExportMemoryHandleExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == pMemHandleRet || NULL == pMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < handleTypeToExport) + return UR_RESULT_ERROR_INVALID_ENUMERATION; } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hSrcMem)) { - URLOG_CTX_INVALID_REFERENCE(hSrcMem); + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDstMem)) { - URLOG_CTX_INVALID_REFERENCE(hDstMem); + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); } - ur_result_t result = pfnAppendMemBufferCopyRectExp( - hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, - srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnExportMemoryHandleExp( + hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferWriteRectExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteRectExp; +/// @brief Intercept function for urProgramBuildExp +__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { + auto pfnBuildExp = getContext()->urDdiTable.ProgramExp.pfnBuildExp; - if (nullptr == pfnAppendMemBufferWriteRectExp) { + if (nullptr == pfnBuildExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pSrc) + if (NULL == phDevices) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) + if (NULL == hProgram) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hBuffer) + if (UR_EXP_PROGRAM_FLAGS_MASK & flags) + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hProgram)) { + URLOG_CTX_INVALID_REFERENCE(hProgram); + } + + ur_result_t result = + pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urProgramCompileExp +__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { + auto pfnCompileExp = getContext()->urDdiTable.ProgramExp.pfnCompileExp; + + if (nullptr == pfnCompileExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == phDevices) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hProgram) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (UR_EXP_PROGRAM_FLAGS_MASK & flags) + return UR_RESULT_ERROR_INVALID_ENUMERATION; + } - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hProgram)) { + URLOG_CTX_INVALID_REFERENCE(hProgram); + } - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + ur_result_t result = + pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + return result; +} - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urProgramLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram) { + if (nullptr != phProgram) { + *phProgram = nullptr; + } + auto pfnLinkExp = getContext()->urDdiTable.ProgramExp.pfnLinkExp; + + if (nullptr == pfnLinkExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == phDevices) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == phPrograms) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == phProgram) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (UR_EXP_PROGRAM_FLAGS_MASK & flags) + return UR_RESULT_ERROR_INVALID_ENUMERATION; + + if (count == 0) + return UR_RESULT_ERROR_INVALID_SIZE; } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hBuffer)) { - URLOG_CTX_INVALID_REFERENCE(hBuffer); + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); } - ur_result_t result = pfnAppendMemBufferWriteRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnLinkExp(hContext, numDevices, phDevices, flags, count, + phPrograms, pOptions, phProgram); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. +/// @brief Intercept function for urUSMContextMemcpyExp +__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferReadRectExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadRectExp; + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size) { + auto pfnContextMemcpyExp = + getContext()->urDdiTable.USMExp.pfnContextMemcpyExp; - if (nullptr == pfnAppendMemBufferReadRectExp) { + if (nullptr == pfnContextMemcpyExp) { return UR_RESULT_ERROR_UNINITIALIZED; } @@ -9688,374 +9521,414 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( if (NULL == pDst) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) + if (NULL == pSrc) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hBuffer) + if (size == 0) + return UR_RESULT_ERROR_INVALID_SIZE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + ur_result_t result = pfnContextMemcpyExp(hContext, pDst, pSrc, size); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUSMImportExp +__urdlllocal ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size) { + auto pfnImportExp = getContext()->urDdiTable.USMExp.pfnImportExp; + + if (nullptr == pfnImportExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == pMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + ur_result_t result = pfnImportExp(hContext, pMem, size); - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + return result; +} - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUSMReleaseExp +__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem) { + auto pfnReleaseExp = getContext()->urDdiTable.USMExp.pfnReleaseExp; + + if (nullptr == pfnReleaseExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == pMem) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } + + ur_result_t result = pfnReleaseExp(hContext, pMem); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUsmP2PEnablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { + auto pfnEnablePeerAccessExp = + getContext()->urDdiTable.UsmP2PExp.pfnEnablePeerAccessExp; + + if (nullptr == pfnEnablePeerAccessExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == commandDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == peerDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(commandDevice)) { + URLOG_CTX_INVALID_REFERENCE(commandDevice); + } + + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(peerDevice)) { + URLOG_CTX_INVALID_REFERENCE(peerDevice); + } + + ur_result_t result = pfnEnablePeerAccessExp(commandDevice, peerDevice); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urUsmP2PDisablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { + auto pfnDisablePeerAccessExp = + getContext()->urDdiTable.UsmP2PExp.pfnDisablePeerAccessExp; + + if (nullptr == pfnDisablePeerAccessExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } + + if (getContext()->enableParameterValidation) { + if (NULL == commandDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == peerDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(commandDevice)) { + URLOG_CTX_INVALID_REFERENCE(commandDevice); } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hBuffer)) { - URLOG_CTX_INVALID_REFERENCE(hBuffer); + !getContext()->refCountContext->isReferenceValid(peerDevice)) { + URLOG_CTX_INVALID_REFERENCE(peerDevice); } - ur_result_t result = pfnAppendMemBufferReadRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnDisablePeerAccessExp(commandDevice, peerDevice); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendMemBufferFillExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferFillExp; +/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. + size_t propSize, + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. + void *pPropValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. + size_t *pPropSizeRet) { + auto pfnPeerAccessGetInfoExp = + getContext()->urDdiTable.UsmP2PExp.pfnPeerAccessGetInfoExp; - if (nullptr == pfnAppendMemBufferFillExp) { + if (nullptr == pfnPeerAccessGetInfoExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pPattern) + if (propSize != 0 && pPropValue == NULL) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (pPropValue == NULL && pPropSizeRet == NULL) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hBuffer) + if (NULL == commandDevice) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (NULL == peerDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName) + return UR_RESULT_ERROR_INVALID_ENUMERATION; - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (propSize == 0 && pPropValue != NULL) + return UR_RESULT_ERROR_INVALID_SIZE; + } - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(commandDevice)) { + URLOG_CTX_INVALID_REFERENCE(commandDevice); } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hBuffer)) { - URLOG_CTX_INVALID_REFERENCE(hBuffer); + !getContext()->refCountContext->isReferenceValid(peerDevice)) { + URLOG_CTX_INVALID_REFERENCE(peerDevice); } - ur_result_t result = pfnAppendMemBufferFillExp( - hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + ur_result_t result = pfnPeerAccessGetInfoExp( + commandDevice, peerDevice, propName, propSize, pPropValue, pPropSizeRet); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMPrefetchExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMPrefetchExp; +/// @brief Intercept function for urCommandBufferCreateExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer) { + auto pfnCreateExp = getContext()->urDdiTable.CommandBufferExp.pfnCreateExp; - if (nullptr == pfnAppendUSMPrefetchExp) { + if (nullptr == pfnCreateExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pMemory) + if (NULL == pCommandBufferDesc) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (UR_USM_MIGRATION_FLAGS_MASK & flags) - return UR_RESULT_ERROR_INVALID_ENUMERATION; - - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (NULL == phCommandBuffer) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (size == 0) - return UR_RESULT_ERROR_INVALID_SIZE; + if (NULL == hContext) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (NULL == hDevice) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hContext)) { + URLOG_CTX_INVALID_REFERENCE(hContext); + } - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDevice)) { + URLOG_CTX_INVALID_REFERENCE(hDevice); } - ur_result_t result = pfnAppendUSMPrefetchExp( - hCommandBuffer, pMemory, size, flags, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = + pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - auto pfnAppendUSMAdviseExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMAdviseExp; +/// @brief Intercept function for urCommandBufferRetainExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { + auto pfnRetainExp = getContext()->urDdiTable.CommandBufferExp.pfnRetainExp; - if (nullptr == pfnAppendUSMAdviseExp) { + if (nullptr == pfnRetainExp) { return UR_RESULT_ERROR_UNINITIALIZED; - } - - if (getContext()->enableParameterValidation) { - if (NULL == pMemory) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + if (getContext()->enableParameterValidation) { if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } - if (UR_USM_ADVICE_FLAGS_MASK & advice) - return UR_RESULT_ERROR_INVALID_ENUMERATION; - - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + ur_result_t result = pfnRetainExp(hCommandBuffer); - if (size == 0) - return UR_RESULT_ERROR_INVALID_SIZE; + return result; +} - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urCommandBufferReleaseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { + auto pfnReleaseExp = getContext()->urDdiTable.CommandBufferExp.pfnReleaseExp; - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + if (nullptr == pfnReleaseExp) { + return UR_RESULT_ERROR_UNINITIALIZED; + } - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } + if (getContext()->enableParameterValidation) { + if (NULL == hCommandBuffer) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } - ur_result_t result = pfnAppendUSMAdviseExp( - hCommandBuffer, pMemory, size, advice, numSyncPointsInWaitList, - pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + ur_result_t result = pfnReleaseExp(hCommandBuffer); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendNativeCommandExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( +/// @brief Intercept function for urCommandBufferFinalizeExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint) { - auto pfnAppendNativeCommandExp = - getContext()->urDdiTable.CommandBufferExp.pfnAppendNativeCommandExp; + ur_exp_command_buffer_handle_t hCommandBuffer) { + auto pfnFinalizeExp = + getContext()->urDdiTable.CommandBufferExp.pfnFinalizeExp; - if (nullptr == pfnAppendNativeCommandExp) { + if (nullptr == pfnFinalizeExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pfnNativeCommand) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - - if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; } - ur_result_t result = pfnAppendNativeCommandExp( - hCommandBuffer, pfnNativeCommand, pData, hChildCommandBuffer, - numSyncPointsInWaitList, pSyncPointWaitList, pSyncPoint); + ur_result_t result = pfnFinalizeExp(hCommandBuffer); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueCommandBufferExp -__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, +/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( /// [in] Handle of the command-buffer object. ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { - auto pfnCommandBufferExp = - getContext()->urDdiTable.EnqueueExp.pfnCommandBufferExp; + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendKernelLaunchExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendKernelLaunchExp; - if (nullptr == pfnCommandBufferExp) { + if (nullptr == pfnAppendKernelLaunchExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == hQueue) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (NULL == pGlobalWorkSize) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (NULL == hKernel) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (phKernelAlternatives == NULL && numKernelAlternatives > 0) + return UR_RESULT_ERROR_INVALID_VALUE; + + if (phKernelAlternatives != NULL && numKernelAlternatives == 0) + return UR_RESULT_ERROR_INVALID_VALUE; + + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (phEventWaitList == NULL && numEventsInWaitList > 0) return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; @@ -10072,108 +9945,75 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hQueue)) { - URLOG_CTX_INVALID_REFERENCE(hQueue); + !getContext()->refCountContext->isReferenceValid(hKernel)) { + URLOG_CTX_INVALID_REFERENCE(hKernel); } - ur_result_t result = pfnCommandBufferExp( - hQueue, hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); - - if (getContext()->enableLeakChecking && result == UR_RESULT_SUCCESS && - phEvent) { - getContext()->refCountContext->createRefCount(*phEvent); - } + ur_result_t result = pfnAppendKernelLaunchExp( + hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( +/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( /// [in] Handle of the command-buffer object. ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch) { - auto pfnUpdateKernelLaunchExp = - getContext()->urDdiTable.CommandBufferExp.pfnUpdateKernelLaunchExp; - - if (nullptr == pfnUpdateKernelLaunchExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } - - if (getContext()->enableParameterValidation) { - if (NULL == pUpdateKernelLaunch) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (NULL == pUpdateKernelLaunch->hCommand) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (numKernelUpdates == 0) - return UR_RESULT_ERROR_INVALID_SIZE; - } - - ur_result_t result = pfnUpdateKernelLaunchExp( - hCommandBuffer, numKernelUpdates, pUpdateKernelLaunch); - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateSignalEventExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent) { - auto pfnUpdateSignalEventExp = - getContext()->urDdiTable.CommandBufferExp.pfnUpdateSignalEventExp; - - if (nullptr == pfnUpdateSignalEventExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } - - if (getContext()->enableParameterValidation) { - if (NULL == phSignalEvent) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == hCommand) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - ur_result_t result = pfnUpdateSignalEventExp(hCommand, phSignalEvent); - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of /// events that must be complete before the command execution. If nullptr, /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList) { - auto pfnUpdateWaitEventsExp = - getContext()->urDdiTable.CommandBufferExp.pfnUpdateWaitEventsExp; + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMMemcpyExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMMemcpyExp; - if (nullptr == pfnUpdateWaitEventsExp) { + if (nullptr == pfnAppendUSMMemcpyExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == hCommand) + if (NULL == pDst) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == pSrc) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (size == 0) + return UR_RESULT_ERROR_INVALID_SIZE; + + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (phEventWaitList == NULL && numEventsInWaitList > 0) return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; @@ -10189,190 +10029,158 @@ __urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( } } - ur_result_t result = - pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); + ur_result_t result = pfnAppendUSMMemcpyExp( + hCommandBuffer, pDst, pSrc, size, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetInfoExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object +/// @brief Intercept function for urCommandBufferAppendUSMFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value - size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property - void *pPropValue, - /// [out][optional] bytes returned in command-buffer property - size_t *pPropSizeRet) { - auto pfnGetInfoExp = getContext()->urDdiTable.CommandBufferExp.pfnGetInfoExp; + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMFillExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMFillExp; - if (nullptr == pfnGetInfoExp) { + if (nullptr == pfnAppendUSMFillExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (propSize != 0 && pPropValue == NULL) + if (NULL == pMemory) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (pPropValue == NULL && pPropSizeRet == NULL) + if (NULL == pPattern) return UR_RESULT_ERROR_INVALID_NULL_POINTER; if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName) - return UR_RESULT_ERROR_INVALID_ENUMERATION; - - if (propSize == 0 && pPropValue != NULL) + if (patternSize == 0 || size == 0) return UR_RESULT_ERROR_INVALID_SIZE; - } - - ur_result_t result = pfnGetInfoExp(hCommandBuffer, propName, propSize, - pPropValue, pPropSizeRet); - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetNativeHandleExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer) { - auto pfnGetNativeHandleExp = - getContext()->urDdiTable.CommandBufferExp.pfnGetNativeHandleExp; - - if (nullptr == pfnGetNativeHandleExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } - - if (getContext()->enableParameterValidation) { - if (NULL == phNativeCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == hCommandBuffer) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - ur_result_t result = - pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); - - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urDeviceWaitExp -__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice) { - auto pfnWaitExp = getContext()->urDdiTable.DeviceExp.pfnWaitExp; - - if (nullptr == pfnWaitExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } - if (getContext()->enableParameterValidation) { - if (NULL == hDevice) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (hDevice == nullptr) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDevice)) { - URLOG_CTX_INVALID_REFERENCE(hDevice); - } - - ur_result_t result = pfnWaitExp(hDevice); - - return result; -} + if (patternSize > size) + return UR_RESULT_ERROR_INVALID_SIZE; -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramDynamicLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) { - auto pfnDynamicLinkExp = - getContext()->urDdiTable.ProgramExp.pfnDynamicLinkExp; + if (size % patternSize != 0) + return UR_RESULT_ERROR_INVALID_SIZE; - if (nullptr == pfnDynamicLinkExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (getContext()->enableParameterValidation) { - if (NULL == phPrograms) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (NULL == hContext) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (count == 0) - return UR_RESULT_ERROR_INVALID_SIZE; - } + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } - ur_result_t result = pfnDynamicLinkExp(hContext, count, phPrograms); + ur_result_t result = pfnAppendUSMFillExp( + hCommandBuffer, pMemory, pPattern, patternSize, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueTimestampRecordingExp -__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent) { - auto pfnTimestampRecordingExp = - getContext()->urDdiTable.EnqueueExp.pfnTimestampRecordingExp; + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferCopyExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyExp; - if (nullptr == pfnTimestampRecordingExp) { + if (nullptr == pfnAppendMemBufferCopyExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == phEvent) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + if (NULL == hCommandBuffer) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hQueue) + if (NULL == hSrcMem) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (NULL == hDstMem) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + if (phEventWaitList == NULL && numEventsInWaitList > 0) return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; @@ -10389,673 +10197,964 @@ __urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hQueue)) { - URLOG_CTX_INVALID_REFERENCE(hQueue); + !getContext()->refCountContext->isReferenceValid(hSrcMem)) { + URLOG_CTX_INVALID_REFERENCE(hSrcMem); } - ur_result_t result = pfnTimestampRecordingExp( - hQueue, blocking, numEventsInWaitList, phEventWaitList, phEvent); - - if (getContext()->enableLeakChecking && result == UR_RESULT_SUCCESS && - phEvent) { - getContext()->refCountContext->createRefCount(*phEvent); + if (getContext()->enableLifetimeValidation && + !getContext()->refCountContext->isReferenceValid(hDstMem)) { + URLOG_CTX_INVALID_REFERENCE(hDstMem); } + ur_result_t result = pfnAppendMemBufferCopyExp( + hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); + return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCGetMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet) { - auto pfnGetMemHandleExp = getContext()->urDdiTable.IPCExp.pfnGetMemHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferWriteExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteExp; - if (nullptr == pfnGetMemHandleExp) { + if (nullptr == pfnAppendMemBufferWriteExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == ppIPCMemHandleData) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == pIPCMemHandleDataSizeRet) + if (NULL == pSrc) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); - } - ur_result_t result = pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, - pIPCMemHandleDataSizeRet); + if (NULL == hBuffer) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - return result; -} + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCPutMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData) { - auto pfnPutMemHandleExp = getContext()->urDdiTable.IPCExp.pfnPutMemHandleExp; + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (nullptr == pfnPutMemHandleExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (getContext()->enableParameterValidation) { - if (NULL == pIPCMemHandleData) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (NULL == hContext) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + !getContext()->refCountContext->isReferenceValid(hBuffer)) { + URLOG_CTX_INVALID_REFERENCE(hBuffer); } - ur_result_t result = pfnPutMemHandleExp(hContext, pIPCMemHandleData); + ur_result_t result = pfnAppendMemBufferWriteExp( + hCommandBuffer, hBuffer, offset, size, pSrc, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCOpenMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem) { - auto pfnOpenMemHandleExp = - getContext()->urDdiTable.IPCExp.pfnOpenMemHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferReadExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadExp; - if (nullptr == pfnOpenMemHandleExp) { + if (nullptr == pfnAppendMemBufferReadExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pIPCMemHandleData) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == ppMem) + if (NULL == pDst) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hDevice) + if (NULL == hBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDevice)) { - URLOG_CTX_INVALID_REFERENCE(hDevice); - } - - ur_result_t result = pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, - ipcMemHandleDataSize, ppMem); - return result; -} + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCCloseMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem) { - auto pfnCloseMemHandleExp = - getContext()->urDdiTable.IPCExp.pfnCloseMemHandleExp; + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (nullptr == pfnCloseMemHandleExp) { - return UR_RESULT_ERROR_UNINITIALIZED; - } + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (getContext()->enableParameterValidation) { - if (NULL == pMem) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (NULL == hContext) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + !getContext()->refCountContext->isReferenceValid(hBuffer)) { + URLOG_CTX_INVALID_REFERENCE(hBuffer); } - ur_result_t result = pfnCloseMemHandleExp(hContext, pMem); + ur_result_t result = pfnAppendMemBufferReadExp( + hCommandBuffer, hBuffer, offset, size, pDst, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. - size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem) { - auto pfnAllocExportableMemoryExp = - getContext()->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferCopyRectExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyRectExp; - if (nullptr == pfnAllocExportableMemoryExp) { + if (nullptr == pfnAppendMemBufferCopyRectExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == ppMem) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + if (NULL == hCommandBuffer) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hContext) + if (NULL == hSrcMem) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hDevice) + if (NULL == hDstMem) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < handleTypeToExport) - return UR_RESULT_ERROR_INVALID_ENUMERATION; + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (alignment != 0 && ((alignment & (alignment - 1)) != 0)) - return UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT; + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (size == 0) - return UR_RESULT_ERROR_INVALID_USM_SIZE; + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + !getContext()->refCountContext->isReferenceValid(hSrcMem)) { + URLOG_CTX_INVALID_REFERENCE(hSrcMem); } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDevice)) { - URLOG_CTX_INVALID_REFERENCE(hDevice); + !getContext()->refCountContext->isReferenceValid(hDstMem)) { + URLOG_CTX_INVALID_REFERENCE(hDstMem); } - ur_result_t result = pfnAllocExportableMemoryExp( - hContext, hDevice, alignment, size, handleTypeToExport, ppMem); + ur_result_t result = pfnAppendMemBufferCopyRectExp( + hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, + srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem) { - auto pfnFreeExportableMemoryExp = - getContext()->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferWriteRectExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteRectExp; - if (nullptr == pfnFreeExportableMemoryExp) { + if (nullptr == pfnAppendMemBufferWriteRectExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pMem) + if (NULL == pSrc) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hDevice) + if (NULL == hBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDevice)) { - URLOG_CTX_INVALID_REFERENCE(hDevice); + !getContext()->refCountContext->isReferenceValid(hBuffer)) { + URLOG_CTX_INVALID_REFERENCE(hBuffer); } - ur_result_t result = pfnFreeExportableMemoryExp(hContext, hDevice, pMem); + ur_result_t result = pfnAppendMemBufferWriteRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportExportMemoryHandleExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet) { - auto pfnExportMemoryHandleExp = - getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferReadRectExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadRectExp; - if (nullptr == pfnExportMemoryHandleExp) { + if (nullptr == pfnAppendMemBufferReadRectExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pMemHandleRet || NULL == pMem) + if (NULL == pDst) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hDevice) + if (NULL == hBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < handleTypeToExport) - return UR_RESULT_ERROR_INVALID_ENUMERATION; - } + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hDevice)) { - URLOG_CTX_INVALID_REFERENCE(hDevice); + !getContext()->refCountContext->isReferenceValid(hBuffer)) { + URLOG_CTX_INVALID_REFERENCE(hBuffer); } - ur_result_t result = pfnExportMemoryHandleExp( - hContext, hDevice, handleTypeToExport, pMem, pMemHandleRet); + ur_result_t result = pfnAppendMemBufferReadRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramBuildExp -__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { - auto pfnBuildExp = getContext()->urDdiTable.ProgramExp.pfnBuildExp; +/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendMemBufferFillExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendMemBufferFillExp; - if (nullptr == pfnBuildExp) { + if (nullptr == pfnAppendMemBufferFillExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == phDevices) + if (NULL == pPattern) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hProgram) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_PROGRAM_FLAGS_MASK & flags) - return UR_RESULT_ERROR_INVALID_ENUMERATION; + if (NULL == hBuffer) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hProgram)) { - URLOG_CTX_INVALID_REFERENCE(hProgram); + !getContext()->refCountContext->isReferenceValid(hBuffer)) { + URLOG_CTX_INVALID_REFERENCE(hBuffer); } - ur_result_t result = - pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); + ur_result_t result = pfnAppendMemBufferFillExp( + hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramCompileExp -__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { - auto pfnCompileExp = getContext()->urDdiTable.ProgramExp.pfnCompileExp; +/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMPrefetchExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMPrefetchExp; - if (nullptr == pfnCompileExp) { + if (nullptr == pfnAppendUSMPrefetchExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == phDevices) + if (NULL == pMemory) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hProgram) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_PROGRAM_FLAGS_MASK & flags) + if (UR_USM_MIGRATION_FLAGS_MASK & flags) return UR_RESULT_ERROR_INVALID_ENUMERATION; - } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hProgram)) { - URLOG_CTX_INVALID_REFERENCE(hProgram); - } + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - ur_result_t result = - pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - return result; -} + if (size == 0) + return UR_RESULT_ERROR_INVALID_SIZE; -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram) { - if (nullptr != phProgram) { - *phProgram = nullptr; + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } - auto pfnLinkExp = getContext()->urDdiTable.ProgramExp.pfnLinkExp; - if (nullptr == pfnLinkExp) { + ur_result_t result = pfnAppendUSMPrefetchExp( + hCommandBuffer, pMemory, size, flags, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); + + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + auto pfnAppendUSMAdviseExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMAdviseExp; + + if (nullptr == pfnAppendUSMAdviseExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == phDevices) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == phPrograms) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == phProgram) + if (NULL == pMemory) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_PROGRAM_FLAGS_MASK & flags) + if (UR_USM_ADVICE_FLAGS_MASK & advice) return UR_RESULT_ERROR_INVALID_ENUMERATION; - if (count == 0) + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; + + if (size == 0) return UR_RESULT_ERROR_INVALID_SIZE; - } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } - ur_result_t result = pfnLinkExp(hContext, numDevices, phDevices, flags, count, - phPrograms, pOptions, phProgram); + ur_result_t result = pfnAppendUSMAdviseExp( + hCommandBuffer, pMemory, size, advice, numSyncPointsInWaitList, + pSyncPointWaitList, numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMContextMemcpyExp -__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size) { - auto pfnContextMemcpyExp = - getContext()->urDdiTable.USMExp.pfnContextMemcpyExp; +/// @brief Intercept function for urCommandBufferAppendNativeCommandExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint) { + auto pfnAppendNativeCommandExp = + getContext()->urDdiTable.CommandBufferExp.pfnAppendNativeCommandExp; - if (nullptr == pfnContextMemcpyExp) { + if (nullptr == pfnAppendNativeCommandExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pDst) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (NULL == pSrc) + if (NULL == pfnNativeCommand) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (size == 0) - return UR_RESULT_ERROR_INVALID_SIZE; - } + if (pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + if (pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP; } - ur_result_t result = pfnContextMemcpyExp(hContext, pDst, pSrc, size); + ur_result_t result = pfnAppendNativeCommandExp( + hCommandBuffer, pfnNativeCommand, pData, hChildCommandBuffer, + numSyncPointsInWaitList, pSyncPointWaitList, pSyncPoint); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMImportExp -__urdlllocal ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size) { - auto pfnImportExp = getContext()->urDdiTable.USMExp.pfnImportExp; +/// @brief Intercept function for urEnqueueCommandBufferExp +__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { + auto pfnCommandBufferExp = + getContext()->urDdiTable.EnqueueExp.pfnCommandBufferExp; - if (nullptr == pfnImportExp) { + if (nullptr == pfnCommandBufferExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pMem) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; + if (NULL == hQueue) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); + !getContext()->refCountContext->isReferenceValid(hQueue)) { + URLOG_CTX_INVALID_REFERENCE(hQueue); } - ur_result_t result = pfnImportExp(hContext, pMem, size); + ur_result_t result = pfnCommandBufferExp( + hQueue, hCommandBuffer, numEventsInWaitList, phEventWaitList, phEvent); + + if (getContext()->enableLeakChecking && result == UR_RESULT_SUCCESS && + phEvent) { + getContext()->refCountContext->createRefCount(*phEvent); + } return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMReleaseExp -__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem) { - auto pfnReleaseExp = getContext()->urDdiTable.USMExp.pfnReleaseExp; +/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch) { + auto pfnUpdateKernelLaunchExp = + getContext()->urDdiTable.CommandBufferExp.pfnUpdateKernelLaunchExp; - if (nullptr == pfnReleaseExp) { + if (nullptr == pfnUpdateKernelLaunchExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pMem) + if (NULL == pUpdateKernelLaunch) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hContext) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hContext)) { - URLOG_CTX_INVALID_REFERENCE(hContext); - } - ur_result_t result = pfnReleaseExp(hContext, pMem); + if (NULL == pUpdateKernelLaunch->hCommand) + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - return result; -} + if (numKernelUpdates == 0) + return UR_RESULT_ERROR_INVALID_SIZE; + } -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PEnablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { - auto pfnEnablePeerAccessExp = - getContext()->urDdiTable.UsmP2PExp.pfnEnablePeerAccessExp; + ur_result_t result = pfnUpdateKernelLaunchExp( + hCommandBuffer, numKernelUpdates, pUpdateKernelLaunch); - if (nullptr == pfnEnablePeerAccessExp) { + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urCommandBufferUpdateSignalEventExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent) { + auto pfnUpdateSignalEventExp = + getContext()->urDdiTable.CommandBufferExp.pfnUpdateSignalEventExp; + + if (nullptr == pfnUpdateSignalEventExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == commandDevice) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + if (NULL == phSignalEvent) + return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == peerDevice) + if (NULL == hCommand) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(commandDevice)) { - URLOG_CTX_INVALID_REFERENCE(commandDevice); - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(peerDevice)) { - URLOG_CTX_INVALID_REFERENCE(peerDevice); - } - - ur_result_t result = pfnEnablePeerAccessExp(commandDevice, peerDevice); + ur_result_t result = pfnUpdateSignalEventExp(hCommand, phSignalEvent); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PDisablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { - auto pfnDisablePeerAccessExp = - getContext()->urDdiTable.UsmP2PExp.pfnDisablePeerAccessExp; +/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList) { + auto pfnUpdateWaitEventsExp = + getContext()->urDdiTable.CommandBufferExp.pfnUpdateWaitEventsExp; - if (nullptr == pfnDisablePeerAccessExp) { + if (nullptr == pfnUpdateWaitEventsExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == commandDevice) + if (NULL == hCommand) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (NULL == peerDevice) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } + if (phEventWaitList == NULL && numEventsInWaitList > 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(commandDevice)) { - URLOG_CTX_INVALID_REFERENCE(commandDevice); - } + if (phEventWaitList != NULL && numEventsInWaitList == 0) + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(peerDevice)) { - URLOG_CTX_INVALID_REFERENCE(peerDevice); + if (phEventWaitList != NULL && numEventsInWaitList > 0) { + for (uint32_t i = 0; i < numEventsInWaitList; ++i) { + if (phEventWaitList[i] == NULL) { + return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; + } + } + } } - ur_result_t result = pfnDisablePeerAccessExp(commandDevice, peerDevice); + ur_result_t result = + pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// @brief Intercept function for urCommandBufferGetInfoExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet) { - auto pfnPeerAccessGetInfoExp = - getContext()->urDdiTable.UsmP2PExp.pfnPeerAccessGetInfoExp; + auto pfnGetInfoExp = getContext()->urDdiTable.CommandBufferExp.pfnGetInfoExp; - if (nullptr == pfnPeerAccessGetInfoExp) { + if (nullptr == pfnGetInfoExp) { return UR_RESULT_ERROR_UNINITIALIZED; } @@ -11066,145 +11165,46 @@ __urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( if (pPropValue == NULL && pPropSizeRet == NULL) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == commandDevice) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (NULL == peerDevice) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - if (UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName) + if (UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName) return UR_RESULT_ERROR_INVALID_ENUMERATION; if (propSize == 0 && pPropValue != NULL) return UR_RESULT_ERROR_INVALID_SIZE; } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(commandDevice)) { - URLOG_CTX_INVALID_REFERENCE(commandDevice); - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(peerDevice)) { - URLOG_CTX_INVALID_REFERENCE(peerDevice); - } - - ur_result_t result = pfnPeerAccessGetInfoExp( - commandDevice, peerDevice, propName, propSize, pPropValue, pPropSizeRet); + ur_result_t result = pfnGetInfoExp(hCommandBuffer, propName, propSize, + pPropValue, pPropSizeRet); return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp -__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { - auto pfnKernelLaunchWithArgsExp = - getContext()->urDdiTable.EnqueueExp.pfnKernelLaunchWithArgsExp; +/// @brief Intercept function for urCommandBufferGetNativeHandleExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer) { + auto pfnGetNativeHandleExp = + getContext()->urDdiTable.CommandBufferExp.pfnGetNativeHandleExp; - if (nullptr == pfnKernelLaunchWithArgsExp) { + if (nullptr == pfnGetNativeHandleExp) { return UR_RESULT_ERROR_UNINITIALIZED; } if (getContext()->enableParameterValidation) { - if (NULL == pGlobalWorkSize) - return UR_RESULT_ERROR_INVALID_NULL_POINTER; - - if (pArgs == NULL && numArgs > 0) + if (NULL == phNativeCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_POINTER; - if (NULL == hQueue) - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (NULL == hKernel) + if (NULL == hCommandBuffer) return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - - if (NULL != pArgs && UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type) - return UR_RESULT_ERROR_INVALID_ENUMERATION; - - if (NULL != launchPropList && - UR_KERNEL_LAUNCH_FLAGS_MASK & launchPropList->flags) - return UR_RESULT_ERROR_INVALID_ENUMERATION; - - if (phEventWaitList == NULL && numEventsInWaitList > 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - - if (phEventWaitList != NULL && numEventsInWaitList == 0) - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - - if (pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || - pGlobalWorkSize[2] == 0) - return UR_RESULT_ERROR_INVALID_WORK_DIMENSION; - - if (pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == 0 || - pLocalWorkSize[2] == 0)) - return UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE; - - if (phEventWaitList != NULL && numEventsInWaitList > 0) { - for (uint32_t i = 0; i < numEventsInWaitList; ++i) { - if (phEventWaitList[i] == NULL) { - return UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST; - } - } - } - } - - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hQueue)) { - URLOG_CTX_INVALID_REFERENCE(hQueue); } - if (getContext()->enableLifetimeValidation && - !getContext()->refCountContext->isReferenceValid(hKernel)) { - URLOG_CTX_INVALID_REFERENCE(hKernel); - } - - ur_result_t result = pfnKernelLaunchWithArgsExp( - hQueue, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numArgs, pArgs, launchPropList, numEventsInWaitList, - phEventWaitList, phEvent); - - if (getContext()->enableLeakChecking && result == UR_RESULT_SUCCESS && - phEvent) { - getContext()->refCountContext->createRefCount(*phEvent); - } + ur_result_t result = + pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); return result; } @@ -12276,14 +12276,14 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( dditable.pfnUSMFreeExp = pDdiTable->pfnUSMFreeExp; pDdiTable->pfnUSMFreeExp = ur_validation_layer::urEnqueueUSMFreeExp; - dditable.pfnCommandBufferExp = pDdiTable->pfnCommandBufferExp; - pDdiTable->pfnCommandBufferExp = - ur_validation_layer::urEnqueueCommandBufferExp; - dditable.pfnTimestampRecordingExp = pDdiTable->pfnTimestampRecordingExp; pDdiTable->pfnTimestampRecordingExp = ur_validation_layer::urEnqueueTimestampRecordingExp; + dditable.pfnCommandBufferExp = pDdiTable->pfnCommandBufferExp; + pDdiTable->pfnCommandBufferExp = + ur_validation_layer::urEnqueueCommandBufferExp; + dditable.pfnHostTaskExp = pDdiTable->pfnHostTaskExp; pDdiTable->pfnHostTaskExp = ur_validation_layer::urEnqueueHostTaskExp; diff --git a/unified-runtime/source/loader/ur_ldrddi.cpp b/unified-runtime/source/loader/ur_ldrddi.cpp index 090193594d687..3b98988aeb5b9 100644 --- a/unified-runtime/source/loader/ur_ldrddi.cpp +++ b/unified-runtime/source/loader/ur_ldrddi.cpp @@ -4553,1371 +4553,1371 @@ __urdlllocal ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferCreateExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. - ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer) { +/// @brief Intercept function for urDeviceWaitExp +__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hDevice); - auto *pfnCreateExp = dditable->CommandBufferExp.pfnCreateExp; - if (nullptr == pfnCreateExp) + auto *pfnWaitExp = dditable->DeviceExp.pfnWaitExp; + if (nullptr == pfnWaitExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); + return pfnWaitExp(hDevice); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferRetainExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { +/// @brief Intercept function for urProgramDynamicLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnRetainExp = dditable->CommandBufferExp.pfnRetainExp; - if (nullptr == pfnRetainExp) + auto *pfnDynamicLinkExp = dditable->ProgramExp.pfnDynamicLinkExp; + if (nullptr == pfnDynamicLinkExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnRetainExp(hCommandBuffer); + return pfnDynamicLinkExp(hContext, count, phPrograms); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferReleaseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { +/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp +__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object + ur_kernel_handle_t hKernel, + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items + uint32_t workDim, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item + const size_t *pGlobalWorkOffset, + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function + const size_t *pGlobalWorkSize, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. + const size_t *pLocalWorkSize, + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hQueue); - auto *pfnReleaseExp = dditable->CommandBufferExp.pfnReleaseExp; - if (nullptr == pfnReleaseExp) + auto *pfnKernelLaunchWithArgsExp = + dditable->EnqueueExp.pfnKernelLaunchWithArgsExp; + if (nullptr == pfnKernelLaunchWithArgsExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnReleaseExp(hCommandBuffer); + return pfnKernelLaunchWithArgsExp(hQueue, hKernel, workDim, pGlobalWorkOffset, + pGlobalWorkSize, pLocalWorkSize, numArgs, + pArgs, launchPropList, numEventsInWaitList, + phEventWaitList, phEvent); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferFinalizeExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { +/// @brief Intercept function for urEnqueueTimestampRecordingExp +__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hQueue); - auto *pfnFinalizeExp = dditable->CommandBufferExp.pfnFinalizeExp; - if (nullptr == pfnFinalizeExp) + auto *pfnTimestampRecordingExp = + dditable->EnqueueExp.pfnTimestampRecordingExp; + if (nullptr == pfnTimestampRecordingExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnFinalizeExp(hCommandBuffer); + return pfnTimestampRecordingExp(hQueue, blocking, numEventsInWaitList, + phEventWaitList, phEvent); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. - ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. - uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. - const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. - const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. - const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urIPCGetMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendKernelLaunchExp = - dditable->CommandBufferExp.pfnAppendKernelLaunchExp; - if (nullptr == pfnAppendKernelLaunchExp) + auto *pfnGetMemHandleExp = dditable->IPCExp.pfnGetMemHandleExp; + if (nullptr == pfnGetMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendKernelLaunchExp( - hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, + pIPCMemHandleDataSizeRet); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urIPCPutMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendUSMMemcpyExp = - dditable->CommandBufferExp.pfnAppendUSMMemcpyExp; - if (nullptr == pfnAppendUSMMemcpyExp) + auto *pfnPutMemHandleExp = dditable->IPCExp.pfnPutMemHandleExp; + if (nullptr == pfnPutMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendUSMMemcpyExp(hCommandBuffer, pDst, pSrc, size, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + return pfnPutMemHandleExp(hContext, pIPCMemHandleData); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urIPCOpenMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendUSMFillExp = dditable->CommandBufferExp.pfnAppendUSMFillExp; - if (nullptr == pfnAppendUSMFillExp) + auto *pfnOpenMemHandleExp = dditable->IPCExp.pfnOpenMemHandleExp; + if (nullptr == pfnOpenMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendUSMFillExp(hCommandBuffer, pMemory, pPattern, patternSize, - size, numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + return pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, + ipcMemHandleDataSize, ppMem); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urIPCCloseMemHandleExp +__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendMemBufferCopyExp = - dditable->CommandBufferExp.pfnAppendMemBufferCopyExp; - if (nullptr == pfnAppendMemBufferCopyExp) + auto *pfnCloseMemHandleExp = dditable->IPCExp.pfnCloseMemHandleExp; + if (nullptr == pfnCloseMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendMemBufferCopyExp( - hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnCloseMemHandleExp(hContext, pMem); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. +/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendMemBufferWriteExp = - dditable->CommandBufferExp.pfnAppendMemBufferWriteExp; - if (nullptr == pfnAppendMemBufferWriteExp) + auto *pfnAllocExportableMemoryExp = + dditable->MemoryExportExp.pfnAllocExportableMemoryExp; + if (nullptr == pfnAllocExportableMemoryExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendMemBufferWriteExp(hCommandBuffer, hBuffer, offset, size, pSrc, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, - pSyncPoint, phEvent, phCommand); + return pfnAllocExportableMemoryExp(hContext, hDevice, alignment, size, + handleTypeToExport, ppMem); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendMemBufferReadExp = - dditable->CommandBufferExp.pfnAppendMemBufferReadExp; - if (nullptr == pfnAppendMemBufferReadExp) + auto *pfnFreeExportableMemoryExp = + dditable->MemoryExportExp.pfnFreeExportableMemoryExp; + if (nullptr == pfnFreeExportableMemoryExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendMemBufferReadExp(hCommandBuffer, hBuffer, offset, size, pDst, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, - pSyncPoint, phEvent, phCommand); + return pfnFreeExportableMemoryExp(hContext, hDevice, pMem); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - - auto *dditable = *reinterpret_cast(hCommandBuffer); - - auto *pfnAppendMemBufferCopyRectExp = - dditable->CommandBufferExp.pfnAppendMemBufferCopyRectExp; - if (nullptr == pfnAppendMemBufferCopyRectExp) - return UR_RESULT_ERROR_UNINITIALIZED; - - // forward to device-platform - return pfnAppendMemBufferCopyRectExp( - hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, - srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urMemoryExportExportMemoryHandleExp +__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendMemBufferWriteRectExp = - dditable->CommandBufferExp.pfnAppendMemBufferWriteRectExp; - if (nullptr == pfnAppendMemBufferWriteRectExp) + auto *pfnExportMemoryHandleExp = + dditable->MemoryExportExp.pfnExportMemoryHandleExp; + if (nullptr == pfnExportMemoryHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendMemBufferWriteRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnExportMemoryHandleExp(hContext, hDevice, handleTypeToExport, pMem, + pMemHandleRet); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urProgramBuildExp +__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hProgram); - auto *pfnAppendMemBufferReadRectExp = - dditable->CommandBufferExp.pfnAppendMemBufferReadRectExp; - if (nullptr == pfnAppendMemBufferReadRectExp) + auto *pfnBuildExp = dditable->ProgramExp.pfnBuildExp; + if (nullptr == pfnBuildExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendMemBufferReadRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urProgramCompileExp +__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hProgram); - auto *pfnAppendMemBufferFillExp = - dditable->CommandBufferExp.pfnAppendMemBufferFillExp; - if (nullptr == pfnAppendMemBufferFillExp) + auto *pfnCompileExp = dditable->ProgramExp.pfnCompileExp; + if (nullptr == pfnCompileExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendMemBufferFillExp( - hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { - - auto *dditable = *reinterpret_cast(hCommandBuffer); +/// @brief Intercept function for urProgramLinkExp +__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram) { + if (nullptr != phProgram) { + *phProgram = nullptr; + } + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendUSMPrefetchExp = - dditable->CommandBufferExp.pfnAppendUSMPrefetchExp; - if (nullptr == pfnAppendUSMPrefetchExp) + auto *pfnLinkExp = dditable->ProgramExp.pfnLinkExp; + if (nullptr == pfnLinkExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendUSMPrefetchExp(hCommandBuffer, pMemory, size, flags, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, - pSyncPoint, phEvent, phCommand); + return pfnLinkExp(hContext, numDevices, phDevices, flags, count, phPrograms, + pOptions, phProgram); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// @brief Intercept function for urUSMContextMemcpyExp +__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. + void *pDst, + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendUSMAdviseExp = - dditable->CommandBufferExp.pfnAppendUSMAdviseExp; - if (nullptr == pfnAppendUSMAdviseExp) + auto *pfnContextMemcpyExp = dditable->USMExp.pfnContextMemcpyExp; + if (nullptr == pfnContextMemcpyExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendUSMAdviseExp(hCommandBuffer, pMemory, size, advice, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + return pfnContextMemcpyExp(hContext, pDst, pSrc, size); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferAppendNativeCommandExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint) { +/// @brief Intercept function for urUSMImportExp +__urdlllocal ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnAppendNativeCommandExp = - dditable->CommandBufferExp.pfnAppendNativeCommandExp; - if (nullptr == pfnAppendNativeCommandExp) + auto *pfnImportExp = dditable->USMExp.pfnImportExp; + if (nullptr == pfnImportExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAppendNativeCommandExp(hCommandBuffer, pfnNativeCommand, pData, - hChildCommandBuffer, numSyncPointsInWaitList, - pSyncPointWaitList, pSyncPoint); + return pfnImportExp(hContext, pMem, size); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueCommandBufferExp -__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { +/// @brief Intercept function for urUSMReleaseExp +__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem) { - auto *dditable = *reinterpret_cast(hQueue); + auto *dditable = *reinterpret_cast(hContext); - auto *pfnCommandBufferExp = dditable->EnqueueExp.pfnCommandBufferExp; - if (nullptr == pfnCommandBufferExp) + auto *pfnReleaseExp = dditable->USMExp.pfnReleaseExp; + if (nullptr == pfnReleaseExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnCommandBufferExp(hQueue, hCommandBuffer, numEventsInWaitList, - phEventWaitList, phEvent); + return pfnReleaseExp(hContext, pMem); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch) { +/// @brief Intercept function for urUsmP2PEnablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { - auto *dditable = *reinterpret_cast(hCommandBuffer); + auto *dditable = *reinterpret_cast(commandDevice); - auto *pfnUpdateKernelLaunchExp = - dditable->CommandBufferExp.pfnUpdateKernelLaunchExp; - if (nullptr == pfnUpdateKernelLaunchExp) + auto *pfnEnablePeerAccessExp = dditable->UsmP2PExp.pfnEnablePeerAccessExp; + if (nullptr == pfnEnablePeerAccessExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnUpdateKernelLaunchExp(hCommandBuffer, numKernelUpdates, - pUpdateKernelLaunch); + return pfnEnablePeerAccessExp(commandDevice, peerDevice); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateSignalEventExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent) { +/// @brief Intercept function for urUsmP2PDisablePeerAccessExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { - auto *dditable = *reinterpret_cast(hCommand); + auto *dditable = *reinterpret_cast(commandDevice); - auto *pfnUpdateSignalEventExp = - dditable->CommandBufferExp.pfnUpdateSignalEventExp; - if (nullptr == pfnUpdateSignalEventExp) + auto *pfnDisablePeerAccessExp = dditable->UsmP2PExp.pfnDisablePeerAccessExp; + if (nullptr == pfnDisablePeerAccessExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnUpdateSignalEventExp(hCommand, phSignalEvent); + return pfnDisablePeerAccessExp(commandDevice, peerDevice); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList) { +/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp +__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. + size_t propSize, + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. + void *pPropValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. + size_t *pPropSizeRet) { - auto *dditable = *reinterpret_cast(hCommand); + auto *dditable = *reinterpret_cast(commandDevice); - auto *pfnUpdateWaitEventsExp = - dditable->CommandBufferExp.pfnUpdateWaitEventsExp; - if (nullptr == pfnUpdateWaitEventsExp) + auto *pfnPeerAccessGetInfoExp = dditable->UsmP2PExp.pfnPeerAccessGetInfoExp; + if (nullptr == pfnPeerAccessGetInfoExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); + return pfnPeerAccessGetInfoExp(commandDevice, peerDevice, propName, propSize, + pPropValue, pPropSizeRet); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetInfoExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value - size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property - void *pPropValue, - /// [out][optional] bytes returned in command-buffer property - size_t *pPropSizeRet) { +/// @brief Intercept function for urCommandBufferCreateExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer) { + + auto *dditable = *reinterpret_cast(hContext); + + auto *pfnCreateExp = dditable->CommandBufferExp.pfnCreateExp; + if (nullptr == pfnCreateExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + // forward to device-platform + return pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Intercept function for urCommandBufferRetainExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnGetInfoExp = dditable->CommandBufferExp.pfnGetInfoExp; - if (nullptr == pfnGetInfoExp) + auto *pfnRetainExp = dditable->CommandBufferExp.pfnRetainExp; + if (nullptr == pfnRetainExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnGetInfoExp(hCommandBuffer, propName, propSize, pPropValue, - pPropSizeRet); + return pfnRetainExp(hCommandBuffer); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urCommandBufferGetNativeHandleExp -__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer) { +/// @brief Intercept function for urCommandBufferReleaseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnGetNativeHandleExp = - dditable->CommandBufferExp.pfnGetNativeHandleExp; - if (nullptr == pfnGetNativeHandleExp) + auto *pfnReleaseExp = dditable->CommandBufferExp.pfnReleaseExp; + if (nullptr == pfnReleaseExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); + return pfnReleaseExp(hCommandBuffer); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urDeviceWaitExp -__urdlllocal ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice) { +/// @brief Intercept function for urCommandBufferFinalizeExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferFinalizeExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { - auto *dditable = *reinterpret_cast(hDevice); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnWaitExp = dditable->DeviceExp.pfnWaitExp; - if (nullptr == pfnWaitExp) + auto *pfnFinalizeExp = dditable->CommandBufferExp.pfnFinalizeExp; + if (nullptr == pfnFinalizeExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnWaitExp(hDevice); + return pfnFinalizeExp(hCommandBuffer); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramDynamicLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) { +/// @brief Intercept function for urCommandBufferAppendKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnDynamicLinkExp = dditable->ProgramExp.pfnDynamicLinkExp; - if (nullptr == pfnDynamicLinkExp) + auto *pfnAppendKernelLaunchExp = + dditable->CommandBufferExp.pfnAppendKernelLaunchExp; + if (nullptr == pfnAppendKernelLaunchExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnDynamicLinkExp(hContext, count, phPrograms); + return pfnAppendKernelLaunchExp( + hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueTimestampRecordingExp -__urdlllocal ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// @brief Intercept function for urCommandBufferAppendUSMMemcpyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent) { + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hQueue); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnTimestampRecordingExp = - dditable->EnqueueExp.pfnTimestampRecordingExp; - if (nullptr == pfnTimestampRecordingExp) + auto *pfnAppendUSMMemcpyExp = + dditable->CommandBufferExp.pfnAppendUSMMemcpyExp; + if (nullptr == pfnAppendUSMMemcpyExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnTimestampRecordingExp(hQueue, blocking, numEventsInWaitList, - phEventWaitList, phEvent); + return pfnAppendUSMMemcpyExp(hCommandBuffer, pDst, pSrc, size, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCGetMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet) { +/// @brief Intercept function for urCommandBufferAppendUSMFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnGetMemHandleExp = dditable->IPCExp.pfnGetMemHandleExp; - if (nullptr == pfnGetMemHandleExp) + auto *pfnAppendUSMFillExp = dditable->CommandBufferExp.pfnAppendUSMFillExp; + if (nullptr == pfnAppendUSMFillExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, - pIPCMemHandleDataSizeRet); + return pfnAppendUSMFillExp(hCommandBuffer, pMemory, pPattern, patternSize, + size, numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCPutMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData) { +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnPutMemHandleExp = dditable->IPCExp.pfnPutMemHandleExp; - if (nullptr == pfnPutMemHandleExp) + auto *pfnAppendMemBufferCopyExp = + dditable->CommandBufferExp.pfnAppendMemBufferCopyExp; + if (nullptr == pfnAppendMemBufferCopyExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnPutMemHandleExp(hContext, pIPCMemHandleData); + return pfnAppendMemBufferCopyExp( + hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCOpenMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem) { +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnOpenMemHandleExp = dditable->IPCExp.pfnOpenMemHandleExp; - if (nullptr == pfnOpenMemHandleExp) + auto *pfnAppendMemBufferWriteExp = + dditable->CommandBufferExp.pfnAppendMemBufferWriteExp; + if (nullptr == pfnAppendMemBufferWriteExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, - ipcMemHandleDataSize, ppMem); + return pfnAppendMemBufferWriteExp(hCommandBuffer, hBuffer, offset, size, pSrc, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, + pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urIPCCloseMemHandleExp -__urdlllocal ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem) { +/// @brief Intercept function for urCommandBufferAppendMemBufferReadExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnCloseMemHandleExp = dditable->IPCExp.pfnCloseMemHandleExp; - if (nullptr == pfnCloseMemHandleExp) + auto *pfnAppendMemBufferReadExp = + dditable->CommandBufferExp.pfnAppendMemBufferReadExp; + if (nullptr == pfnAppendMemBufferReadExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnCloseMemHandleExp(hContext, pMem); + return pfnAppendMemBufferReadExp(hCommandBuffer, hBuffer, offset, size, pDst, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, + pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportAllocExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. - size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem) { +/// @brief Intercept function for urCommandBufferAppendMemBufferCopyRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnAllocExportableMemoryExp = - dditable->MemoryExportExp.pfnAllocExportableMemoryExp; - if (nullptr == pfnAllocExportableMemoryExp) + auto *pfnAppendMemBufferCopyRectExp = + dditable->CommandBufferExp.pfnAppendMemBufferCopyRectExp; + if (nullptr == pfnAppendMemBufferCopyRectExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnAllocExportableMemoryExp(hContext, hDevice, alignment, size, - handleTypeToExport, ppMem); + return pfnAppendMemBufferCopyRectExp( + hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, + srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportFreeExportableMemoryExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem) { +/// @brief Intercept function for urCommandBufferAppendMemBufferWriteRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnFreeExportableMemoryExp = - dditable->MemoryExportExp.pfnFreeExportableMemoryExp; - if (nullptr == pfnFreeExportableMemoryExp) + auto *pfnAppendMemBufferWriteRectExp = + dditable->CommandBufferExp.pfnAppendMemBufferWriteRectExp; + if (nullptr == pfnAppendMemBufferWriteRectExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnFreeExportableMemoryExp(hContext, hDevice, pMem); + return pfnAppendMemBufferWriteRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urMemoryExportExportMemoryHandleExp -__urdlllocal ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet) { +/// @brief Intercept function for urCommandBufferAppendMemBufferReadRectExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnExportMemoryHandleExp = - dditable->MemoryExportExp.pfnExportMemoryHandleExp; - if (nullptr == pfnExportMemoryHandleExp) + auto *pfnAppendMemBufferReadRectExp = + dditable->CommandBufferExp.pfnAppendMemBufferReadRectExp; + if (nullptr == pfnAppendMemBufferReadRectExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnExportMemoryHandleExp(hContext, hDevice, handleTypeToExport, pMem, - pMemHandleRet); + return pfnAppendMemBufferReadRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramBuildExp -__urdlllocal ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { +/// @brief Intercept function for urCommandBufferAppendMemBufferFillExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hProgram); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnBuildExp = dditable->ProgramExp.pfnBuildExp; - if (nullptr == pfnBuildExp) + auto *pfnAppendMemBufferFillExp = + dditable->CommandBufferExp.pfnAppendMemBufferFillExp; + if (nullptr == pfnAppendMemBufferFillExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); + return pfnAppendMemBufferFillExp( + hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramCompileExp -__urdlllocal ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { +/// @brief Intercept function for urCommandBufferAppendUSMPrefetchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *dditable = *reinterpret_cast(hProgram); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnCompileExp = dditable->ProgramExp.pfnCompileExp; - if (nullptr == pfnCompileExp) + auto *pfnAppendUSMPrefetchExp = + dditable->CommandBufferExp.pfnAppendUSMPrefetchExp; + if (nullptr == pfnAppendUSMPrefetchExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); + return pfnAppendUSMPrefetchExp(hCommandBuffer, pMemory, size, flags, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, + pSyncPoint, phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urProgramLinkExp -__urdlllocal ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram) { - if (nullptr != phProgram) { - *phProgram = nullptr; - } - auto *dditable = *reinterpret_cast(hContext); +/// @brief Intercept function for urCommandBufferAppendUSMAdviseExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { - auto *pfnLinkExp = dditable->ProgramExp.pfnLinkExp; - if (nullptr == pfnLinkExp) + auto *dditable = *reinterpret_cast(hCommandBuffer); + + auto *pfnAppendUSMAdviseExp = + dditable->CommandBufferExp.pfnAppendUSMAdviseExp; + if (nullptr == pfnAppendUSMAdviseExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnLinkExp(hContext, numDevices, phDevices, flags, count, phPrograms, - pOptions, phProgram); + return pfnAppendUSMAdviseExp(hCommandBuffer, pMemory, size, advice, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMContextMemcpyExp -__urdlllocal ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size) { +/// @brief Intercept function for urCommandBufferAppendNativeCommandExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnContextMemcpyExp = dditable->USMExp.pfnContextMemcpyExp; - if (nullptr == pfnContextMemcpyExp) + auto *pfnAppendNativeCommandExp = + dditable->CommandBufferExp.pfnAppendNativeCommandExp; + if (nullptr == pfnAppendNativeCommandExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnContextMemcpyExp(hContext, pDst, pSrc, size); + return pfnAppendNativeCommandExp(hCommandBuffer, pfnNativeCommand, pData, + hChildCommandBuffer, numSyncPointsInWaitList, + pSyncPointWaitList, pSyncPoint); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMImportExp -__urdlllocal ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size) { +/// @brief Intercept function for urEnqueueCommandBufferExp +__urdlllocal ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hQueue); - auto *pfnImportExp = dditable->USMExp.pfnImportExp; - if (nullptr == pfnImportExp) + auto *pfnCommandBufferExp = dditable->EnqueueExp.pfnCommandBufferExp; + if (nullptr == pfnCommandBufferExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnImportExp(hContext, pMem, size); + return pfnCommandBufferExp(hQueue, hCommandBuffer, numEventsInWaitList, + phEventWaitList, phEvent); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUSMReleaseExp -__urdlllocal ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem) { +/// @brief Intercept function for urCommandBufferUpdateKernelLaunchExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch) { - auto *dditable = *reinterpret_cast(hContext); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnReleaseExp = dditable->USMExp.pfnReleaseExp; - if (nullptr == pfnReleaseExp) + auto *pfnUpdateKernelLaunchExp = + dditable->CommandBufferExp.pfnUpdateKernelLaunchExp; + if (nullptr == pfnUpdateKernelLaunchExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnReleaseExp(hContext, pMem); + return pfnUpdateKernelLaunchExp(hCommandBuffer, numKernelUpdates, + pUpdateKernelLaunch); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PEnablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { +/// @brief Intercept function for urCommandBufferUpdateSignalEventExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent) { - auto *dditable = *reinterpret_cast(commandDevice); + auto *dditable = *reinterpret_cast(hCommand); - auto *pfnEnablePeerAccessExp = dditable->UsmP2PExp.pfnEnablePeerAccessExp; - if (nullptr == pfnEnablePeerAccessExp) + auto *pfnUpdateSignalEventExp = + dditable->CommandBufferExp.pfnUpdateSignalEventExp; + if (nullptr == pfnUpdateSignalEventExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnEnablePeerAccessExp(commandDevice, peerDevice); + return pfnUpdateSignalEventExp(hCommand, phSignalEvent); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PDisablePeerAccessExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { +/// @brief Intercept function for urCommandBufferUpdateWaitEventsExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList) { - auto *dditable = *reinterpret_cast(commandDevice); + auto *dditable = *reinterpret_cast(hCommand); - auto *pfnDisablePeerAccessExp = dditable->UsmP2PExp.pfnDisablePeerAccessExp; - if (nullptr == pfnDisablePeerAccessExp) + auto *pfnUpdateWaitEventsExp = + dditable->CommandBufferExp.pfnUpdateWaitEventsExp; + if (nullptr == pfnUpdateWaitEventsExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnDisablePeerAccessExp(commandDevice, peerDevice); + return pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urUsmP2PPeerAccessGetInfoExp -__urdlllocal ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// @brief Intercept function for urCommandBufferGetInfoExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet) { - auto *dditable = *reinterpret_cast(commandDevice); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnPeerAccessGetInfoExp = dditable->UsmP2PExp.pfnPeerAccessGetInfoExp; - if (nullptr == pfnPeerAccessGetInfoExp) + auto *pfnGetInfoExp = dditable->CommandBufferExp.pfnGetInfoExp; + if (nullptr == pfnGetInfoExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnPeerAccessGetInfoExp(commandDevice, peerDevice, propName, propSize, - pPropValue, pPropSizeRet); + return pfnGetInfoExp(hCommandBuffer, propName, propSize, pPropValue, + pPropSizeRet); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Intercept function for urEnqueueKernelLaunchWithArgsExp -__urdlllocal ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { +/// @brief Intercept function for urCommandBufferGetNativeHandleExp +__urdlllocal ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer) { - auto *dditable = *reinterpret_cast(hQueue); + auto *dditable = *reinterpret_cast(hCommandBuffer); - auto *pfnKernelLaunchWithArgsExp = - dditable->EnqueueExp.pfnKernelLaunchWithArgsExp; - if (nullptr == pfnKernelLaunchWithArgsExp) + auto *pfnGetNativeHandleExp = + dditable->CommandBufferExp.pfnGetNativeHandleExp; + if (nullptr == pfnGetNativeHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; // forward to device-platform - return pfnKernelLaunchWithArgsExp(hQueue, hKernel, workDim, pGlobalWorkOffset, - pGlobalWorkSize, pLocalWorkSize, numArgs, - pArgs, launchPropList, numEventsInWaitList, - phEventWaitList, phEvent); + return pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); } /////////////////////////////////////////////////////////////////////////////// @@ -6688,9 +6688,9 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetEnqueueExpProcAddrTable( pDdiTable->pfnUSMSharedAllocExp = ur_loader::urEnqueueUSMSharedAllocExp; pDdiTable->pfnUSMHostAllocExp = ur_loader::urEnqueueUSMHostAllocExp; pDdiTable->pfnUSMFreeExp = ur_loader::urEnqueueUSMFreeExp; - pDdiTable->pfnCommandBufferExp = ur_loader::urEnqueueCommandBufferExp; pDdiTable->pfnTimestampRecordingExp = ur_loader::urEnqueueTimestampRecordingExp; + pDdiTable->pfnCommandBufferExp = ur_loader::urEnqueueCommandBufferExp; pDdiTable->pfnHostTaskExp = ur_loader::urEnqueueHostTaskExp; pDdiTable->pfnNativeCommandExp = ur_loader::urEnqueueNativeCommandExp; pDdiTable->pfnGraphExp = ur_loader::urEnqueueGraphExp; diff --git a/unified-runtime/source/loader/ur_libapi.cpp b/unified-runtime/source/loader/ur_libapi.cpp index f93dd94ff4dba..1cae1d517729c 100644 --- a/unified-runtime/source/loader/ur_libapi.cpp +++ b/unified-runtime/source/loader/ur_libapi.cpp @@ -8492,10 +8492,38 @@ ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Create a Command-Buffer object +/// @brief Synchronizes with all queues on the device. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hDevice` +/// + `hDevice == nullptr` +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice) try { + auto pfnWaitExp = ur_lib::getContext()->urDdiTable.DeviceExp.pfnWaitExp; + if (nullptr == pfnWaitExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnWaitExp(hDevice); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Creates dynamic links between exported and imported symbols in one or +/// more programs. /// /// @details -/// - Create a command-buffer object. +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point the programs in +/// `phPrograms` will have all external symbols resolved and kernels +/// inside these programs would be ready for use. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8504,39 +8532,40 @@ ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hContext` -/// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pCommandBufferDesc` -/// + `NULL == phCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// + `NULL == phPrograms` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If one of the programs in `phPrograms` isn't a valid program +/// object. +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `count == 0` +/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE +/// + If an error occurred while linking `phPrograms`. /// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns -/// 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP -/// query. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. +/// + If the adapter has no means to support the operation. +ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer) try { - auto pfnCreateExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnCreateExp; - if (nullptr == pfnCreateExp) + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms) try { + auto pfnDynamicLinkExp = + ur_lib::getContext()->urDdiTable.ProgramExp.pfnDynamicLinkExp; + if (nullptr == pfnDynamicLinkExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); + return pfnDynamicLinkExp(hContext, count, phPrograms); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Increment the command-buffer object's reference count. +/// @brief Enqueue a command to execute a kernel +/// +/// @remarks +/// _Analogues_ +/// - **clEnqueueNDRangeKernel** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8544,29 +8573,91 @@ ur_result_t UR_APICALL urCommandBufferCreateExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// + `NULL == hQueue` +/// + `NULL == hKernel` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pGlobalWorkSize` +/// + `pArgs == NULL && numArgs > 0` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `NULL != pArgs && ::UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type` +/// + `NULL != launchPropList && ::UR_KERNEL_LAUNCH_FLAGS_MASK & +/// launchPropList->flags` +/// - ::UR_RESULT_ERROR_INVALID_QUEUE +/// - ::UR_RESULT_ERROR_INVALID_KERNEL +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_IN_EVENT_LIST_EXEC_STATUS +/// + An event in `phEventWaitList` has ::UR_EVENT_STATUS_ERROR. +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// + `pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || +/// pGlobalWorkSize[2] == 0` +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// + `pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == +/// 0 || pLocalWorkSize[2] == 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGS - "The kernel argument values +/// have not been specified." /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) try { - auto pfnRetainExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnRetainExp; - if (nullptr == pfnRetainExp) +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If any property in `launchPropList` isn't supported by the device. +ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object + ur_kernel_handle_t hKernel, + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items + uint32_t workDim, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item + const size_t *pGlobalWorkOffset, + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function + const size_t *pGlobalWorkSize, + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. + const size_t *pLocalWorkSize, + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) try { + auto pfnKernelLaunchWithArgsExp = + ur_lib::getContext()->urDdiTable.EnqueueExp.pfnKernelLaunchWithArgsExp; + if (nullptr == pfnKernelLaunchWithArgsExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnRetainExp(hCommandBuffer); + return pfnKernelLaunchWithArgsExp(hQueue, hKernel, workDim, pGlobalWorkOffset, + pGlobalWorkSize, pLocalWorkSize, numArgs, + pArgs, launchPropList, numEventsInWaitList, + phEventWaitList, phEvent); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Decrement the command-buffer object's reference count and delete the -/// command-buffer object if the reference count becomes zero. It is -/// legal to call the entry-point while `hCommandBuffer` is still -/// executing, which will block on completion if the reference count of -/// `hCommandBuffer` becomes zero. +/// @brief Enqueue a command for recording the device timestamp /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8574,26 +8665,51 @@ ur_result_t UR_APICALL urCommandBufferRetainExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) try { - auto pfnReleaseExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnReleaseExp; - if (nullptr == pfnReleaseExp) +/// + `NULL == hQueue` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phEvent` +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent) try { + auto pfnTimestampRecordingExp = + ur_lib::getContext()->urDdiTable.EnqueueExp.pfnTimestampRecordingExp; + if (nullptr == pfnTimestampRecordingExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnReleaseExp(hCommandBuffer); + return pfnTimestampRecordingExp(hQueue, blocking, numEventsInWaitList, + phEventWaitList, phEvent); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Stop recording on a command-buffer object such that no more commands -/// can be appended and make it ready to enqueue. +/// @brief Gets an inter-process memory handle for a pointer to device USM +/// memory /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8601,27 +8717,35 @@ ur_result_t UR_APICALL urCommandBufferReleaseExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "If `hCommandBuffer` has already -/// been finalized" +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppIPCMemHandleData` +/// + `NULL == pIPCMemHandleDataSizeRet` /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) try { - auto pfnFinalizeExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnFinalizeExp; - if (nullptr == pfnFinalizeExp) +ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet) try { + auto pfnGetMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnGetMemHandleExp; + if (nullptr == pfnGetMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnFinalizeExp(hCommandBuffer); + return pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, + pIPCMemHandleDataSizeRet); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a kernel execution command to a command-buffer object. +/// @brief Releases an inter-process memory handle /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8629,96 +8753,30 @@ ur_result_t UR_APICALL urCommandBufferFinalizeExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hKernel` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pGlobalWorkSize` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_KERNEL -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + `phKernelAlternatives == NULL && numKernelAlternatives > 0` -/// + `phKernelAlternatives != NULL && numKernelAlternatives == 0` -/// + If `phKernelAlternatives` contains `hKernel` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pIPCMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "phCommand is not NULL and -/// hCommandBuffer is not updatable." -ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. - ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. - uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. - const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. - const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. - const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendKernelLaunchExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendKernelLaunchExp; - if (nullptr == pfnAppendKernelLaunchExp) +ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData) try { + auto pfnPutMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnPutMemHandleExp; + if (nullptr == pfnPutMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendKernelLaunchExp( - hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, - pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnPutMemHandleExp(hContext, pIPCMemHandleData); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM memcpy command to a command-buffer object. +/// @brief Opens an inter-process memory handle to get the corresponding pointer +/// to device USM memory /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8726,73 +8784,42 @@ ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pSrc` or `pDst` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pIPCMemHandleData` +/// + `NULL == ppMem` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcMemHandleDataSize is not the same as the size of IPC memory +/// handle data /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendUSMMemcpyExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMMemcpyExp; - if (nullptr == pfnAppendUSMMemcpyExp) +ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem) try { + auto pfnOpenMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnOpenMemHandleExp; + if (nullptr == pfnOpenMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendUSMMemcpyExp(hCommandBuffer, pDst, pSrc, size, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + return pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, + ipcMemHandleDataSize, ppMem); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM fill command to a command-buffer object. +/// @brief Closes an inter-process memory handle /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8800,77 +8827,30 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` -/// + `NULL == pPattern` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `patternSize == 0 || size == 0` -/// + `patternSize > size` -/// + `size % patternSize != 0` -/// + If `size` is higher than the allocation size of `ptr` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMem` /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendUSMFillExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMFillExp; - if (nullptr == pfnAppendUSMFillExp) +ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem) try { + auto pfnCloseMemHandleExp = + ur_lib::getContext()->urDdiTable.IPCExp.pfnCloseMemHandleExp; + if (nullptr == pfnCloseMemHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendUSMFillExp(hCommandBuffer, pMemory, pPattern, patternSize, - size, numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + return pfnCloseMemHandleExp(hContext, pMem); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory copy command to a command-buffer object. +/// @brief Allocate an exportable memory region and return a pointer to that +/// allocation. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8878,74 +8858,55 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hSrcMem` -/// + `NULL == hDstMem` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppMem` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT +/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `alignment` exceeds largest supported data type by `hDevice` +/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE +/// + `size == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE +/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - +/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferCopyExp = + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) try { + auto pfnAllocExportableMemoryExp = ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyExp; - if (nullptr == pfnAppendMemBufferCopyExp) + ->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; + if (nullptr == pfnAllocExportableMemoryExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendMemBufferCopyExp( - hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnAllocExportableMemoryExp(hContext, hDevice, alignment, size, + handleTypeToExport, ppMem); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory write command to a command-buffer object. +/// @brief Free an exportable memory allocation. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8953,73 +8914,40 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferWriteExp = +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) try { + auto pfnFreeExportableMemoryExp = ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteExp; - if (nullptr == pfnAppendMemBufferWriteExp) + ->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; + if (nullptr == pfnFreeExportableMemoryExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendMemBufferWriteExp(hCommandBuffer, hBuffer, offset, size, pSrc, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, - pSyncPoint, phEvent, phCommand); + return pfnFreeExportableMemoryExp(hContext, hDevice, pMem); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory read command to a command-buffer object. +/// @brief Obtain an exportable handle to a memory allocated with +/// `AllocExportableMemoryExp`.The returned external memory type will be +/// that which was specified upon +/// allocation of the exportable memory (e.g. `opaque_fd` or +/// `win32_nt_handle`). /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9027,245 +8955,54 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMemHandleRet || NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferReadExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadExp; - if (nullptr == pfnAppendMemBufferReadExp) +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) try { + auto pfnExportMemoryHandleExp = + ur_lib::getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; + if (nullptr == pfnExportMemoryHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendMemBufferReadExp(hCommandBuffer, hBuffer, offset, size, pDst, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, - pSyncPoint, phEvent, phCommand); + return pfnExportMemoryHandleExp(hContext, hDevice, handleTypeToExport, pMem, + pMemHandleRet); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory copy command to a command-buffer object. +/// @brief Produces an executable program from one program, negates need for the +/// linking step. /// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hSrcMem` -/// + `NULL == hDstMem` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferCopyRectExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyRectExp; - if (nullptr == pfnAppendMemBufferCopyRectExp) - return UR_RESULT_ERROR_UNINITIALIZED; - - return pfnAppendMemBufferCopyRectExp( - hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, - srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); -} catch (...) { - return exceptionToResult(std::current_exception()); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory write command to a command-buffer object. +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point, the program passed +/// will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type +/// for each device in `phDevices`. /// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferWriteRectExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteRectExp; - if (nullptr == pfnAppendMemBufferWriteRectExp) - return UR_RESULT_ERROR_UNINITIALIZED; - - return pfnAppendMemBufferWriteRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); -} catch (...) { - return exceptionToResult(std::current_exception()); -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory read command to a command-buffer object. +/// @remarks +/// _Analogues_ +/// - **clBuildProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9273,86 +9010,47 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hProgram` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferReadRectExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadRectExp; - if (nullptr == pfnAppendMemBufferReadRectExp) +/// + `NULL == phDevices` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If `hProgram` isn't a valid program object. +/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE +/// + If an error occurred when building `hProgram`. +ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) try { + auto pfnBuildExp = ur_lib::getContext()->urDdiTable.ProgramExp.pfnBuildExp; + if (nullptr == pfnBuildExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendMemBufferReadRectExp( - hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, - bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory fill command to a command-buffer object. +/// @brief Produces an executable program from one or more programs. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point `hProgram` will +/// contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type +/// for each device in `phDevices`. +/// +/// @remarks +/// _Analogues_ +/// - **clCompileProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9360,82 +9058,54 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hProgram` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pPattern` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + If `offset + size` results in an out-of-bounds access. -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendMemBufferFillExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendMemBufferFillExp; - if (nullptr == pfnAppendMemBufferFillExp) +/// + `NULL == phDevices` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If `hProgram` isn't a valid program object. +/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE +/// + If an error occurred while compiling `hProgram`. +ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) try { + auto pfnCompileExp = + ur_lib::getContext()->urDdiTable.ProgramExp.pfnCompileExp; + if (nullptr == pfnCompileExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendMemBufferFillExp( - hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, - numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, - phEventWaitList, pSyncPoint, phEvent, phCommand); + return pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM Prefetch command to a command-buffer object. +/// @brief Produces an executable program from one or more programs. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point the program returned +/// in `phProgram` will contain a binary of the +/// ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in +/// `phDevices`. +/// - If a non-success code is returned, adapters may store a program in +/// `phProgram`. This program should only be used with +/// `::urProgramGetBuildInfo` to get the build log for the failure. +/// Adapters which do not do not support producing build logs must set +/// this value to `nullptr`. /// -/// @details -/// - Prefetching may not be supported for all devices or allocation types. -/// If memory prefetching is not supported, the prefetch hint will be -/// ignored. +/// @remarks +/// _Analogues_ +/// - **clLinkProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9443,79 +9113,53 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` +/// + `NULL == phDevices` +/// + `NULL == phPrograms` +/// + `NULL == phProgram` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_USM_MIGRATION_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If one of the programs in `phPrograms` isn't a valid program +/// object. /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pMemory` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendUSMPrefetchExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMPrefetchExp; - if (nullptr == pfnAppendUSMPrefetchExp) +/// + `count == 0` +/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE +/// + If an error occurred while linking `phPrograms`. +ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram) try { + if (nullptr != phProgram) { + *phProgram = nullptr; + } + auto pfnLinkExp = ur_lib::getContext()->urDdiTable.ProgramExp.pfnLinkExp; + if (nullptr == pfnLinkExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendUSMPrefetchExp(hCommandBuffer, pMemory, size, flags, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, - pSyncPoint, phEvent, phCommand); + return pfnLinkExp(hContext, numDevices, phDevices, flags, count, phPrograms, + pOptions, phProgram); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM Advise command to a command-buffer object. -/// -/// @details -/// - Not all memory advice hints may be supported for all devices or -/// allocation types. If a memory advice hint is not supported, it will be -/// ignored. +/// @brief Perform a synchronous, blocking memcpy operation between USM +/// allocations. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9523,74 +9167,40 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_USM_ADVICE_FLAGS_MASK & advice` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// + `NULL == pDst` +/// + `NULL == pSrc` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_SIZE /// + `size == 0` -/// + If `size` is higher than the allocation size of `pMemory` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) try { - auto pfnAppendUSMAdviseExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMAdviseExp; - if (nullptr == pfnAppendUSMAdviseExp) +/// + If `size` is higher than the allocation size of `pSrc` or `pDst` +ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. + void *pDst, + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size) try { + auto pfnContextMemcpyExp = + ur_lib::getContext()->urDdiTable.USMExp.pfnContextMemcpyExp; + if (nullptr == pfnContextMemcpyExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendUSMAdviseExp(hCommandBuffer, pMemory, size, advice, - numSyncPointsInWaitList, pSyncPointWaitList, - numEventsInWaitList, phEventWaitList, pSyncPoint, - phEvent, phCommand); + return pfnContextMemcpyExp(hContext, pDst, pSrc, size); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append nodes to the command-buffer through a native backend API +/// @brief Import memory into USM +/// +/// @details +/// - Import memory into USM /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9598,53 +9208,81 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pfnNativeCommand` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint) try { - auto pfnAppendNativeCommandExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnAppendNativeCommandExp; - if (nullptr == pfnAppendNativeCommandExp) +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size) try { + auto pfnImportExp = ur_lib::getContext()->urDdiTable.USMExp.pfnImportExp; + if (nullptr == pfnImportExp) + return UR_RESULT_ERROR_UNINITIALIZED; + + return pfnImportExp(hContext, pMem, size); +} catch (...) { + return exceptionToResult(std::current_exception()); +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Release memory from USM +/// +/// @details +/// - Release memory from USM +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem) try { + auto pfnReleaseExp = ur_lib::getContext()->urDdiTable.USMExp.pfnReleaseExp; + if (nullptr == pfnReleaseExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAppendNativeCommandExp(hCommandBuffer, pfnNativeCommand, pData, - hChildCommandBuffer, numSyncPointsInWaitList, - pSyncPointWaitList, pSyncPoint); + return pfnReleaseExp(hContext, pMem); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Submit a command-buffer for execution on a queue. +/// @brief Enable access to peer device memory +/// +/// @details +/// - Enables the command device to access and write device memory +/// allocations located on the peer device, provided that a P2P link +/// between the two devices is available. +/// - When Peer Access is successfully enabled, P2P memory accesses are +/// guaranteed to be allowed on the peer device until +/// ::urUsmP2PDisablePeerAccessExp is called. +/// - Note that the function operands may, but aren't guaranteed to, commute +/// for a given adapter: the peer device is not guaranteed to have access +/// to device memory allocations located on the command device. +/// - It is not guaranteed that the commutation relations of the function +/// arguments are identical for peer access and peer copies: For example, +/// for a given adapter the peer device may be able to copy data from the +/// command device, but not access and write the same data on the command +/// device. +/// - Consult the appropriate adapter driver documentation for details of +/// adapter specific behavior and native error codes that may be returned. +/// +/// @remarks +/// _Analogues_ +/// - **cuCtxEnablePeerAccess** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9652,52 +9290,50 @@ ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_QUEUE -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) try { - auto pfnCommandBufferExp = - ur_lib::getContext()->urDdiTable.EnqueueExp.pfnCommandBufferExp; - if (nullptr == pfnCommandBufferExp) +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) try { + auto pfnEnablePeerAccessExp = + ur_lib::getContext()->urDdiTable.UsmP2PExp.pfnEnablePeerAccessExp; + if (nullptr == pfnEnablePeerAccessExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnCommandBufferExp(hQueue, hCommandBuffer, numEventsInWaitList, - phEventWaitList, phEvent); + return pfnEnablePeerAccessExp(commandDevice, peerDevice); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Update a kernel launch command in a finalized command-buffer. +/// @brief Disable access to peer device memory /// /// @details -/// This entry-point is synchronous and may block if the command-buffer is -/// executing when the entry-point is called. On error, the state of the -/// command-buffer commands being updated is undefined. +/// - Disables the ability of the command device to access and write device +/// memory allocations located on the peer device, provided that a P2P +/// link between the two devices was enabled prior to the call. +/// - Note that the function operands may, but aren't guaranteed to, commute +/// for a given adapter. If, prior to the function call, the peer device +/// had access to device memory allocations on the command device, it is +/// not guaranteed to still have such access following the function +/// return. +/// - It is not guaranteed that the commutation relations of the function +/// arguments are identical for peer access and peer copies: For example +/// for a given adapter, if, prior to the call, the peer device had access +/// to device memory allocations on the command device, the peer device +/// may still, following the function call, be able to copy data from the +/// command device, but not access and write the same data on the command +/// device. +/// - Consult the appropriate adapter driver documentation for details of +/// adapter specific behavior and native error codes that may be returned. +/// +/// @remarks +/// _Analogues_ +/// - **cuCtxDisablePeerAccess** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9705,96 +9341,35 @@ ur_result_t UR_APICALL urEnqueueCommandBufferExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == pUpdateKernelLaunch->hCommand` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pUpdateKernelLaunch` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `numKernelUpdates == 0` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS -/// is not supported by the device, and for any of any element of -/// `pUpdateKernelLaunch` the `numNewMemObjArgs`, `numNewPointerArgs`, -/// or `numNewValueArgs` members are not zero. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is -/// not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is not nullptr. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is -/// not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is nullptr and -/// `pNewGlobalWorkSize` is not nullptr. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewGlobalWorkSize` member is not nullptr -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewGlobalWorkOffset` member is not -/// nullptr. -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `hNewKernel` member is not nullptr. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the `hCommandBuffer`. -/// + If `hCommandBuffer` has not been finalized. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member -/// is not a kernel execution command. -/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member -/// was not created from `hCommandBuffer`. -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` -/// member is less than 1 or greater than 3. -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + If for any element of `pUpdateKernelLaunch` the `hNewKernel` -/// member was not passed to the `hKernel` or `phKernelAlternatives` -/// parameters of ::urCommandBufferAppendKernelLaunchExp when the -/// command was created. -/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` -/// member is different from the current workDim in the `hCommand` -/// member, and `pNewGlobalWorkSize` or `pNewGlobalWorkOffset` are -/// nullptr. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch) try { - auto pfnUpdateKernelLaunchExp = - ur_lib::getContext() - ->urDdiTable.CommandBufferExp.pfnUpdateKernelLaunchExp; - if (nullptr == pfnUpdateKernelLaunchExp) +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) try { + auto pfnDisablePeerAccessExp = + ur_lib::getContext()->urDdiTable.UsmP2PExp.pfnDisablePeerAccessExp; + if (nullptr == pfnDisablePeerAccessExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnUpdateKernelLaunchExp(hCommandBuffer, numKernelUpdates, - pUpdateKernelLaunch); + return pfnDisablePeerAccessExp(commandDevice, peerDevice); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Get a new event that will be signaled the next time the command in -/// the -/// command-buffer executes. +/// @brief Disable access to peer device memory /// /// @details -/// It is the users responsibility to release the returned `phSignalEvent`. +/// - Queries the peer access capabilities from the command device to the +/// peer device according to the query `propName`. +/// +/// @remarks +/// _Analogues_ +/// - **cuDeviceGetP2PAttribute** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9802,41 +9377,56 @@ ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommand` +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION +/// + If `propName` is not supported by the adapter. +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `propSize == 0 && pPropValue != NULL` +/// + If `propSize` is less than the real number of bytes needed to +/// return the info. /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phSignalEvent` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not -/// supported by the device associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the command-buffer `hCommand` belongs to. -/// + If the command-buffer `hCommand` belongs to has not been -/// finalized. -/// + If no `phEvent` parameter was set on creation of the command -/// associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent) try { - auto pfnUpdateSignalEventExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnUpdateSignalEventExp; - if (nullptr == pfnUpdateSignalEventExp) +/// + `propSize != 0 && pPropValue == NULL` +/// + `pPropValue == NULL && pPropSizeRet == NULL` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. + size_t propSize, + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. + void *pPropValue, + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. + size_t *pPropSizeRet) try { + auto pfnPeerAccessGetInfoExp = + ur_lib::getContext()->urDdiTable.UsmP2PExp.pfnPeerAccessGetInfoExp; + if (nullptr == pfnPeerAccessGetInfoExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnUpdateSignalEventExp(hCommand, phSignalEvent); + return pfnPeerAccessGetInfoExp(commandDevice, peerDevice, propName, propSize, + pPropValue, pPropSizeRet); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Set the list of wait events for a command to depend on to a list of -/// new events. +/// @brief Create a Command-Buffer object +/// +/// @details +/// - Create a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9844,45 +9434,40 @@ ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommand` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pCommandBufferDesc` +/// + `NULL == phCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE /// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not -/// supported by the device associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the command-buffer `hCommand` belongs to. -/// + If the command-buffer `hCommand` belongs to has not been -/// finalized. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// + If `numEventsInWaitList` does not match the number of wait events -/// set when the command associated with `hCommand` was created. +/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns +/// 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP +/// query. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList) try { - auto pfnUpdateWaitEventsExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnUpdateWaitEventsExp; - if (nullptr == pfnUpdateWaitEventsExp) +ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer) try { + auto pfnCreateExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnCreateExp; + if (nullptr == pfnCreateExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); + return pfnCreateExp(hContext, hDevice, pCommandBufferDesc, phCommandBuffer); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Get command-buffer object information. +/// @brief Increment the command-buffer object's reference count. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9891,49 +9476,28 @@ ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION -/// + If `propName` is not supported by the adapter. -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `propSize == 0 && pPropValue != NULL` -/// + If `propSize` is less than the real number of bytes needed to -/// return the info. -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `propSize != 0 && pPropValue == NULL` -/// + `pPropValue == NULL && pPropSizeRet == NULL` /// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value - size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property - void *pPropValue, - /// [out][optional] bytes returned in command-buffer property - size_t *pPropSizeRet) try { - auto pfnGetInfoExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnGetInfoExp; - if (nullptr == pfnGetInfoExp) +ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) try { + auto pfnRetainExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnRetainExp; + if (nullptr == pfnRetainExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnGetInfoExp(hCommandBuffer, propName, propSize, pPropValue, - pPropSizeRet); + return pfnRetainExp(hCommandBuffer); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Return platform native command-buffer handle. -/// -/// @details -/// - Retrieved native handle can be used for direct interaction with the -/// native platform driver. +/// @brief Decrement the command-buffer object's reference count and delete the +/// command-buffer object if the reference count becomes zero. It is +/// legal to call the entry-point while `hCommandBuffer` is still +/// executing, which will block on completion if the reference count of +/// `hCommandBuffer` becomes zero. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9942,27 +9506,25 @@ ur_result_t UR_APICALL urCommandBufferGetInfoExp( /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE /// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phNativeCommandBuffer` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the adapter has no underlying equivalent handle. -ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer) try { - auto pfnGetNativeHandleExp = - ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnGetNativeHandleExp; - if (nullptr == pfnGetNativeHandleExp) +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) try { + auto pfnReleaseExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnReleaseExp; + if (nullptr == pfnReleaseExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); + return pfnReleaseExp(hCommandBuffer); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Synchronizes with all queues on the device. +/// @brief Stop recording on a command-buffer object such that no more commands +/// can be appended and make it ready to enqueue. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9970,30 +9532,27 @@ ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hDevice` -/// + `hDevice == nullptr` -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice) try { - auto pfnWaitExp = ur_lib::getContext()->urDdiTable.DeviceExp.pfnWaitExp; - if (nullptr == pfnWaitExp) +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "If `hCommandBuffer` has already +/// been finalized" +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferFinalizeExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) try { + auto pfnFinalizeExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnFinalizeExp; + if (nullptr == pfnFinalizeExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnWaitExp(hDevice); + return pfnFinalizeExp(hCommandBuffer); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Creates dynamic links between exported and imported symbols in one or -/// more programs. -/// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point the programs in -/// `phPrograms` will have all external symbols resolved and kernels -/// inside these programs would be ready for use. +/// @brief Append a kernel execution command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10001,37 +9560,96 @@ ur_result_t UR_APICALL urDeviceWaitExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` +/// + `NULL == hKernel` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phPrograms` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If one of the programs in `phPrograms` isn't a valid program -/// object. -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `count == 0` -/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE -/// + If an error occurred while linking `phPrograms`. +/// + `NULL == pGlobalWorkSize` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_KERNEL +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + `phKernelAlternatives == NULL && numKernelAlternatives > 0` +/// + `phKernelAlternatives != NULL && numKernelAlternatives == 0` +/// + If `phKernelAlternatives` contains `hKernel` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. /// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the adapter has no means to support the operation. -ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) try { - auto pfnDynamicLinkExp = - ur_lib::getContext()->urDdiTable.ProgramExp.pfnDynamicLinkExp; - if (nullptr == pfnDynamicLinkExp) +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "phCommand is not NULL and +/// hCommandBuffer is not updatable." +ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendKernelLaunchExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendKernelLaunchExp; + if (nullptr == pfnAppendKernelLaunchExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnDynamicLinkExp(hContext, count, phPrograms); + return pfnAppendKernelLaunchExp( + hCommandBuffer, hKernel, workDim, pGlobalWorkOffset, pGlobalWorkSize, + pLocalWorkSize, numKernelAlternatives, phKernelAlternatives, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Enqueue a command for recording the device timestamp +/// @brief Append a USM memcpy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10039,51 +9657,73 @@ ur_result_t UR_APICALL urProgramDynamicLinkExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phEvent` +/// + `NULL == pDst` +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pSrc` or `pDst` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT /// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST /// + `phEventWaitList == NULL && numEventsInWaitList > 0` /// + `phEventWaitList != NULL && numEventsInWaitList == 0` -ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent) try { - auto pfnTimestampRecordingExp = - ur_lib::getContext()->urDdiTable.EnqueueExp.pfnTimestampRecordingExp; - if (nullptr == pfnTimestampRecordingExp) + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendUSMMemcpyExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMMemcpyExp; + if (nullptr == pfnAppendUSMMemcpyExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnTimestampRecordingExp(hQueue, blocking, numEventsInWaitList, - phEventWaitList, phEvent); + return pfnAppendUSMMemcpyExp(hCommandBuffer, pDst, pSrc, size, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Gets an inter-process memory handle for a pointer to device USM -/// memory +/// @brief Append a USM fill command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10091,35 +9731,77 @@ ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == ppIPCMemHandleData` -/// + `NULL == pIPCMemHandleDataSizeRet` +/// + `NULL == pMemory` +/// + `NULL == pPattern` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `patternSize == 0 || size == 0` +/// + `patternSize > size` +/// + `size % patternSize != 0` +/// + If `size` is higher than the allocation size of `ptr` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet) try { - auto pfnGetMemHandleExp = - ur_lib::getContext()->urDdiTable.IPCExp.pfnGetMemHandleExp; - if (nullptr == pfnGetMemHandleExp) +ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendUSMFillExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMFillExp; + if (nullptr == pfnAppendUSMFillExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnGetMemHandleExp(hContext, pMem, ppIPCMemHandleData, - pIPCMemHandleDataSizeRet); + return pfnAppendUSMFillExp(hCommandBuffer, pMemory, pPattern, patternSize, + size, numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Releases an inter-process memory handle +/// @brief Append a memory copy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10127,30 +9809,74 @@ ur_result_t UR_APICALL urIPCGetMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pIPCMemHandleData` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` +/// + `NULL == hSrcMem` +/// + `NULL == hDstMem` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData) try { - auto pfnPutMemHandleExp = - ur_lib::getContext()->urDdiTable.IPCExp.pfnPutMemHandleExp; - if (nullptr == pfnPutMemHandleExp) +ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferCopyExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyExp; + if (nullptr == pfnAppendMemBufferCopyExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnPutMemHandleExp(hContext, pIPCMemHandleData); + return pfnAppendMemBufferCopyExp( + hCommandBuffer, hSrcMem, hDstMem, srcOffset, dstOffset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Opens an inter-process memory handle to get the corresponding pointer -/// to device USM memory +/// @brief Append a memory write command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10158,42 +9884,73 @@ ur_result_t UR_APICALL urIPCPutMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pIPCMemHandleData` -/// + `NULL == ppMem` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + ipcMemHandleDataSize is not the same as the size of IPC memory -/// handle data +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem) try { - auto pfnOpenMemHandleExp = - ur_lib::getContext()->urDdiTable.IPCExp.pfnOpenMemHandleExp; - if (nullptr == pfnOpenMemHandleExp) +ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferWriteExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteExp; + if (nullptr == pfnAppendMemBufferWriteExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnOpenMemHandleExp(hContext, hDevice, pIPCMemHandleData, - ipcMemHandleDataSize, ppMem); + return pfnAppendMemBufferWriteExp(hCommandBuffer, hBuffer, offset, size, pSrc, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, + pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Closes an inter-process memory handle +/// @brief Append a memory read command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10201,30 +9958,73 @@ ur_result_t UR_APICALL urIPCOpenMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` +/// + `NULL == pDst` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem) try { - auto pfnCloseMemHandleExp = - ur_lib::getContext()->urDdiTable.IPCExp.pfnCloseMemHandleExp; - if (nullptr == pfnCloseMemHandleExp) +ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferReadExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadExp; + if (nullptr == pfnAppendMemBufferReadExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnCloseMemHandleExp(hContext, pMem); + return pfnAppendMemBufferReadExp(hCommandBuffer, hBuffer, offset, size, pDst, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, + pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Allocate an exportable memory region and return a pointer to that -/// allocation. +/// @brief Append a rectangular memory copy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10232,55 +10032,83 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < -/// handleTypeToExport` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == ppMem` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT -/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + If `alignment` exceeds largest supported data type by `hDevice` -/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE -/// + `size == 0` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE -/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - -/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP +/// + `NULL == hCommandBuffer` +/// + `NULL == hSrcMem` +/// + `NULL == hDstMem` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. - size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem) try { - auto pfnAllocExportableMemoryExp = +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferCopyRectExp = ur_lib::getContext() - ->urDdiTable.MemoryExportExp.pfnAllocExportableMemoryExp; - if (nullptr == pfnAllocExportableMemoryExp) + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferCopyRectExp; + if (nullptr == pfnAppendMemBufferCopyRectExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnAllocExportableMemoryExp(hContext, hDevice, alignment, size, - handleTypeToExport, ppMem); + return pfnAppendMemBufferCopyRectExp( + hCommandBuffer, hSrcMem, hDstMem, srcOrigin, dstOrigin, region, + srcRowPitch, srcSlicePitch, dstRowPitch, dstSlicePitch, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Free an exportable memory allocation. +/// @brief Append a rectangular memory write command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10288,40 +10116,87 @@ ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem) try { - auto pfnFreeExportableMemoryExp = +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferWriteRectExp = ur_lib::getContext() - ->urDdiTable.MemoryExportExp.pfnFreeExportableMemoryExp; - if (nullptr == pfnFreeExportableMemoryExp) + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferWriteRectExp; + if (nullptr == pfnAppendMemBufferWriteRectExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnFreeExportableMemoryExp(hContext, hDevice, pMem); + return pfnAppendMemBufferWriteRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pSrc, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Obtain an exportable handle to a memory allocated with -/// `AllocExportableMemoryExp`.The returned external memory type will be -/// that which was specified upon -/// allocation of the exportable memory (e.g. `opaque_fd` or -/// `win32_nt_handle`). +/// @brief Append a rectangular memory read command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10329,54 +10204,86 @@ ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < -/// handleTypeToExport` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemHandleRet || NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// + `NULL == pDst` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet) try { - auto pfnExportMemoryHandleExp = - ur_lib::getContext()->urDdiTable.MemoryExportExp.pfnExportMemoryHandleExp; - if (nullptr == pfnExportMemoryHandleExp) +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferReadRectExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferReadRectExp; + if (nullptr == pfnAppendMemBufferReadRectExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnExportMemoryHandleExp(hContext, hDevice, handleTypeToExport, pMem, - pMemHandleRet); + return pfnAppendMemBufferReadRectExp( + hCommandBuffer, hBuffer, bufferOffset, hostOffset, region, bufferRowPitch, + bufferSlicePitch, hostRowPitch, hostSlicePitch, pDst, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one program, negates need for the -/// linking step. -/// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point, the program passed -/// will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type -/// for each device in `phDevices`. -/// -/// @remarks -/// _Analogues_ -/// - **clBuildProgram** +/// @brief Append a memory fill command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10384,47 +10291,82 @@ ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hProgram` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If `hProgram` isn't a valid program object. -/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE -/// + If an error occurred when building `hProgram`. -ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) try { - auto pfnBuildExp = ur_lib::getContext()->urDdiTable.ProgramExp.pfnBuildExp; - if (nullptr == pfnBuildExp) +/// + `NULL == pPattern` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + If `offset + size` results in an out-of-bounds access. +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendMemBufferFillExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendMemBufferFillExp; + if (nullptr == pfnAppendMemBufferFillExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnBuildExp(hProgram, numDevices, phDevices, flags, pOptions); + return pfnAppendMemBufferFillExp( + hCommandBuffer, hBuffer, pPattern, patternSize, offset, size, + numSyncPointsInWaitList, pSyncPointWaitList, numEventsInWaitList, + phEventWaitList, pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one or more programs. +/// @brief Append a USM Prefetch command to a command-buffer object. /// /// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point `hProgram` will -/// contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type -/// for each device in `phDevices`. -/// -/// @remarks -/// _Analogues_ -/// - **clCompileProgram** +/// - Prefetching may not be supported for all devices or allocation types. +/// If memory prefetching is not supported, the prefetch hint will be +/// ignored. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10432,54 +10374,79 @@ ur_result_t UR_APICALL urProgramBuildExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hProgram` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` +/// + `NULL == pMemory` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If `hProgram` isn't a valid program object. -/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE -/// + If an error occurred while compiling `hProgram`. -ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) try { - auto pfnCompileExp = - ur_lib::getContext()->urDdiTable.ProgramExp.pfnCompileExp; - if (nullptr == pfnCompileExp) +/// + `::UR_USM_MIGRATION_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pMemory` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendUSMPrefetchExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMPrefetchExp; + if (nullptr == pfnAppendUSMPrefetchExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnCompileExp(hProgram, numDevices, phDevices, flags, pOptions); + return pfnAppendUSMPrefetchExp(hCommandBuffer, pMemory, size, flags, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, + pSyncPoint, phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one or more programs. +/// @brief Append a USM Advise command to a command-buffer object. /// /// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point the program returned -/// in `phProgram` will contain a binary of the -/// ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in -/// `phDevices`. -/// - If a non-success code is returned, adapters may store a program in -/// `phProgram`. This program should only be used with -/// `::urProgramGetBuildInfo` to get the build log for the failure. -/// Adapters which do not do not support producing build logs must set -/// this value to `nullptr`. -/// -/// @remarks -/// _Analogues_ -/// - **clLinkProgram** +/// - Not all memory advice hints may be supported for all devices or +/// allocation types. If a memory advice hint is not supported, it will be +/// ignored. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10487,53 +10454,74 @@ ur_result_t UR_APICALL urProgramCompileExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` -/// + `NULL == phPrograms` -/// + `NULL == phProgram` +/// + `NULL == pMemory` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If one of the programs in `phPrograms` isn't a valid program -/// object. +/// + `::UR_USM_ADVICE_FLAGS_MASK & advice` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `count == 0` -/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE -/// + If an error occurred while linking `phPrograms`. -ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram) try { - if (nullptr != phProgram) { - *phProgram = nullptr; - } - auto pfnLinkExp = ur_lib::getContext()->urDdiTable.ProgramExp.pfnLinkExp; - if (nullptr == pfnLinkExp) +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pMemory` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) try { + auto pfnAppendUSMAdviseExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnAppendUSMAdviseExp; + if (nullptr == pfnAppendUSMAdviseExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnLinkExp(hContext, numDevices, phDevices, flags, count, phPrograms, - pOptions, phProgram); + return pfnAppendUSMAdviseExp(hCommandBuffer, pMemory, size, advice, + numSyncPointsInWaitList, pSyncPointWaitList, + numEventsInWaitList, phEventWaitList, pSyncPoint, + phEvent, phCommand); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Perform a synchronous, blocking memcpy operation between USM -/// allocations. +/// @brief Append nodes to the command-buffer through a native backend API /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10541,40 +10529,53 @@ ur_result_t UR_APICALL urProgramLinkExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// + `NULL == pSrc` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pSrc` or `pDst` -ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size) try { - auto pfnContextMemcpyExp = - ur_lib::getContext()->urDdiTable.USMExp.pfnContextMemcpyExp; - if (nullptr == pfnContextMemcpyExp) +/// + `NULL == pfnNativeCommand` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint) try { + auto pfnAppendNativeCommandExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnAppendNativeCommandExp; + if (nullptr == pfnAppendNativeCommandExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnContextMemcpyExp(hContext, pDst, pSrc, size); + return pfnAppendNativeCommandExp(hCommandBuffer, pfnNativeCommand, pData, + hChildCommandBuffer, numSyncPointsInWaitList, + pSyncPointWaitList, pSyncPoint); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Import memory into USM -/// -/// @details -/// - Import memory into USM +/// @brief Submit a command-buffer for execution on a queue. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10582,32 +10583,52 @@ ur_result_t UR_APICALL urUSMContextMemcpyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size) try { - auto pfnImportExp = ur_lib::getContext()->urDdiTable.USMExp.pfnImportExp; - if (nullptr == pfnImportExp) +/// + `NULL == hQueue` +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_QUEUE +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) try { + auto pfnCommandBufferExp = + ur_lib::getContext()->urDdiTable.EnqueueExp.pfnCommandBufferExp; + if (nullptr == pfnCommandBufferExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnImportExp(hContext, pMem, size); + return pfnCommandBufferExp(hQueue, hCommandBuffer, numEventsInWaitList, + phEventWaitList, phEvent); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Release memory from USM +/// @brief Update a kernel launch command in a finalized command-buffer. /// /// @details -/// - Release memory from USM +/// This entry-point is synchronous and may block if the command-buffer is +/// executing when the entry-point is called. On error, the state of the +/// command-buffer commands being updated is undefined. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10615,99 +10636,138 @@ ur_result_t UR_APICALL urUSMImportExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` +/// + `NULL == pUpdateKernelLaunch->hCommand` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem) try { - auto pfnReleaseExp = ur_lib::getContext()->urDdiTable.USMExp.pfnReleaseExp; - if (nullptr == pfnReleaseExp) +/// + `NULL == pUpdateKernelLaunch` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `numKernelUpdates == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS +/// is not supported by the device, and for any of any element of +/// `pUpdateKernelLaunch` the `numNewMemObjArgs`, `numNewPointerArgs`, +/// or `numNewValueArgs` members are not zero. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is +/// not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is not nullptr. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is +/// not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is nullptr and +/// `pNewGlobalWorkSize` is not nullptr. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewGlobalWorkSize` member is not nullptr +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewGlobalWorkOffset` member is not +/// nullptr. +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `hNewKernel` member is not nullptr. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the `hCommandBuffer`. +/// + If `hCommandBuffer` has not been finalized. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member +/// is not a kernel execution command. +/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member +/// was not created from `hCommandBuffer`. +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` +/// member is less than 1 or greater than 3. +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If for any element of `pUpdateKernelLaunch` the `hNewKernel` +/// member was not passed to the `hKernel` or `phKernelAlternatives` +/// parameters of ::urCommandBufferAppendKernelLaunchExp when the +/// command was created. +/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` +/// member is different from the current workDim in the `hCommand` +/// member, and `pNewGlobalWorkSize` or `pNewGlobalWorkOffset` are +/// nullptr. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch) try { + auto pfnUpdateKernelLaunchExp = + ur_lib::getContext() + ->urDdiTable.CommandBufferExp.pfnUpdateKernelLaunchExp; + if (nullptr == pfnUpdateKernelLaunchExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnReleaseExp(hContext, pMem); + return pfnUpdateKernelLaunchExp(hCommandBuffer, numKernelUpdates, + pUpdateKernelLaunch); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Enable access to peer device memory +/// @brief Get a new event that will be signaled the next time the command in +/// the +/// command-buffer executes. /// /// @details -/// - Enables the command device to access and write device memory -/// allocations located on the peer device, provided that a P2P link -/// between the two devices is available. -/// - When Peer Access is successfully enabled, P2P memory accesses are -/// guaranteed to be allowed on the peer device until -/// ::urUsmP2PDisablePeerAccessExp is called. -/// - Note that the function operands may, but aren't guaranteed to, commute -/// for a given adapter: the peer device is not guaranteed to have access -/// to device memory allocations located on the command device. -/// - It is not guaranteed that the commutation relations of the function -/// arguments are identical for peer access and peer copies: For example, -/// for a given adapter the peer device may be able to copy data from the -/// command device, but not access and write the same data on the command -/// device. -/// - Consult the appropriate adapter driver documentation for details of -/// adapter specific behavior and native error codes that may be returned. -/// -/// @remarks -/// _Analogues_ -/// - **cuCtxEnablePeerAccess** -/// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) try { - auto pfnEnablePeerAccessExp = - ur_lib::getContext()->urDdiTable.UsmP2PExp.pfnEnablePeerAccessExp; - if (nullptr == pfnEnablePeerAccessExp) +/// It is the users responsibility to release the returned `phSignalEvent`. +/// +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hCommand` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phSignalEvent` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not +/// supported by the device associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the command-buffer `hCommand` belongs to. +/// + If the command-buffer `hCommand` belongs to has not been +/// finalized. +/// + If no `phEvent` parameter was set on creation of the command +/// associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent) try { + auto pfnUpdateSignalEventExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnUpdateSignalEventExp; + if (nullptr == pfnUpdateSignalEventExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnEnablePeerAccessExp(commandDevice, peerDevice); + return pfnUpdateSignalEventExp(hCommand, phSignalEvent); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Disable access to peer device memory -/// -/// @details -/// - Disables the ability of the command device to access and write device -/// memory allocations located on the peer device, provided that a P2P -/// link between the two devices was enabled prior to the call. -/// - Note that the function operands may, but aren't guaranteed to, commute -/// for a given adapter. If, prior to the function call, the peer device -/// had access to device memory allocations on the command device, it is -/// not guaranteed to still have such access following the function -/// return. -/// - It is not guaranteed that the commutation relations of the function -/// arguments are identical for peer access and peer copies: For example -/// for a given adapter, if, prior to the call, the peer device had access -/// to device memory allocations on the command device, the peer device -/// may still, following the function call, be able to copy data from the -/// command device, but not access and write the same data on the command -/// device. -/// - Consult the appropriate adapter driver documentation for details of -/// adapter specific behavior and native error codes that may be returned. -/// -/// @remarks -/// _Analogues_ -/// - **cuCtxDisablePeerAccess** +/// @brief Set the list of wait events for a command to depend on to a list of +/// new events. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10715,35 +10775,45 @@ ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) try { - auto pfnDisablePeerAccessExp = - ur_lib::getContext()->urDdiTable.UsmP2PExp.pfnDisablePeerAccessExp; - if (nullptr == pfnDisablePeerAccessExp) +/// + `NULL == hCommand` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not +/// supported by the device associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the command-buffer `hCommand` belongs to. +/// + If the command-buffer `hCommand` belongs to has not been +/// finalized. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// + If `numEventsInWaitList` does not match the number of wait events +/// set when the command associated with `hCommand` was created. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList) try { + auto pfnUpdateWaitEventsExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnUpdateWaitEventsExp; + if (nullptr == pfnUpdateWaitEventsExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnDisablePeerAccessExp(commandDevice, peerDevice); + return pfnUpdateWaitEventsExp(hCommand, numEventsInWaitList, phEventWaitList); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Disable access to peer device memory -/// -/// @details -/// - Queries the peer access capabilities from the command device to the -/// peer device according to the query `propName`. -/// -/// @remarks -/// _Analogues_ -/// - **cuDeviceGetP2PAttribute** +/// @brief Get command-buffer object information. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10751,10 +10821,9 @@ ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName` +/// + `::UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -10764,44 +10833,38 @@ ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER /// + `propSize != 0 && pPropValue == NULL` /// + `pPropValue == NULL && pPropSizeRet == NULL` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet) try { - auto pfnPeerAccessGetInfoExp = - ur_lib::getContext()->urDdiTable.UsmP2PExp.pfnPeerAccessGetInfoExp; - if (nullptr == pfnPeerAccessGetInfoExp) + auto pfnGetInfoExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnGetInfoExp; + if (nullptr == pfnGetInfoExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnPeerAccessGetInfoExp(commandDevice, peerDevice, propName, propSize, - pPropValue, pPropSizeRet); + return pfnGetInfoExp(hCommandBuffer, propName, propSize, pPropValue, + pPropSizeRet); } catch (...) { return exceptionToResult(std::current_exception()); } /////////////////////////////////////////////////////////////////////////////// -/// @brief Enqueue a command to execute a kernel +/// @brief Return platform native command-buffer handle. /// -/// @remarks -/// _Analogues_ -/// - **clEnqueueNDRangeKernel** +/// @details +/// - Retrieved native handle can be used for direct interaction with the +/// native platform driver. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -10809,85 +10872,22 @@ ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` -/// + `NULL == hKernel` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pGlobalWorkSize` -/// + `pArgs == NULL && numArgs > 0` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `NULL != pArgs && ::UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type` -/// + `NULL != launchPropList && ::UR_KERNEL_LAUNCH_FLAGS_MASK & -/// launchPropList->flags` -/// - ::UR_RESULT_ERROR_INVALID_QUEUE -/// - ::UR_RESULT_ERROR_INVALID_KERNEL -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_IN_EVENT_LIST_EXEC_STATUS -/// + An event in `phEventWaitList` has ::UR_EVENT_STATUS_ERROR. -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// + `pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || -/// pGlobalWorkSize[2] == 0` -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// + `pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == -/// 0 || pLocalWorkSize[2] == 0)` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGS - "The kernel argument values -/// have not been specified." -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If any property in `launchPropList` isn't supported by the device. -ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) try { - auto pfnKernelLaunchWithArgsExp = - ur_lib::getContext()->urDdiTable.EnqueueExp.pfnKernelLaunchWithArgsExp; - if (nullptr == pfnKernelLaunchWithArgsExp) +/// + `NULL == phNativeCommandBuffer` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the adapter has no underlying equivalent handle. +ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer) try { + auto pfnGetNativeHandleExp = + ur_lib::getContext()->urDdiTable.CommandBufferExp.pfnGetNativeHandleExp; + if (nullptr == pfnGetNativeHandleExp) return UR_RESULT_ERROR_UNINITIALIZED; - return pfnKernelLaunchWithArgsExp(hQueue, hKernel, workDim, pGlobalWorkOffset, - pGlobalWorkSize, pLocalWorkSize, numArgs, - pArgs, launchPropList, numEventsInWaitList, - phEventWaitList, phEvent); + return pfnGetNativeHandleExp(hCommandBuffer, phNativeCommandBuffer); } catch (...) { return exceptionToResult(std::current_exception()); } diff --git a/unified-runtime/source/loader/ur_print.cpp b/unified-runtime/source/loader/ur_print.cpp index 6d20fcdd33669..e6230303cde57 100644 --- a/unified-runtime/source/loader/ur_print.cpp +++ b/unified-runtime/source/loader/ur_print.cpp @@ -1089,6 +1089,45 @@ urPrintExpImageCopyRegion(const struct ur_exp_image_copy_region_t params, return str_copy(&ss, buffer, buff_size, out_size); } +ur_result_t urPrintExpKernelArgType(enum ur_exp_kernel_arg_type_t value, + char *buffer, const size_t buff_size, + size_t *out_size) { + std::stringstream ss; + ss << value; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintExpKernelArgMemObjTuple( + const struct ur_exp_kernel_arg_mem_obj_tuple_t params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintExpKernelArgProperties( + const struct ur_exp_kernel_arg_properties_t params, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << params; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintExpProgramFlags(enum ur_exp_program_flag_t value, + char *buffer, const size_t buff_size, + size_t *out_size) { + std::stringstream ss; + ss << value; + return str_copy(&ss, buffer, buff_size, out_size); +} + +ur_result_t urPrintExpPeerInfo(enum ur_exp_peer_info_t value, char *buffer, + const size_t buff_size, size_t *out_size) { + std::stringstream ss; + ss << value; + return str_copy(&ss, buffer, buff_size, out_size); +} + ur_result_t urPrintDeviceCommandBufferUpdateCapabilityFlags( enum ur_device_command_buffer_update_capability_flag_t value, char *buffer, const size_t buff_size, size_t *out_size) { @@ -1154,45 +1193,6 @@ ur_result_t urPrintExpCommandBufferUpdateKernelLaunchDesc( return str_copy(&ss, buffer, buff_size, out_size); } -ur_result_t urPrintExpProgramFlags(enum ur_exp_program_flag_t value, - char *buffer, const size_t buff_size, - size_t *out_size) { - std::stringstream ss; - ss << value; - return str_copy(&ss, buffer, buff_size, out_size); -} - -ur_result_t urPrintExpPeerInfo(enum ur_exp_peer_info_t value, char *buffer, - const size_t buff_size, size_t *out_size) { - std::stringstream ss; - ss << value; - return str_copy(&ss, buffer, buff_size, out_size); -} - -ur_result_t urPrintExpKernelArgType(enum ur_exp_kernel_arg_type_t value, - char *buffer, const size_t buff_size, - size_t *out_size) { - std::stringstream ss; - ss << value; - return str_copy(&ss, buffer, buff_size, out_size); -} - -ur_result_t urPrintExpKernelArgMemObjTuple( - const struct ur_exp_kernel_arg_mem_obj_tuple_t params, char *buffer, - const size_t buff_size, size_t *out_size) { - std::stringstream ss; - ss << params; - return str_copy(&ss, buffer, buff_size, out_size); -} - -ur_result_t urPrintExpKernelArgProperties( - const struct ur_exp_kernel_arg_properties_t params, char *buffer, - const size_t buff_size, size_t *out_size) { - std::stringstream ss; - ss << params; - return str_copy(&ss, buffer, buff_size, out_size); -} - ur_result_t urPrintExpHostTaskFlags(enum ur_exp_host_task_flag_t value, char *buffer, const size_t buff_size, size_t *out_size) { @@ -1989,17 +1989,17 @@ ur_result_t urPrintEnqueueUsmFreeExpParams( return str_copy(&ss, buffer, buff_size, out_size); } -ur_result_t urPrintEnqueueCommandBufferExpParams( - const struct ur_enqueue_command_buffer_exp_params_t *params, char *buffer, - const size_t buff_size, size_t *out_size) { +ur_result_t urPrintEnqueueTimestampRecordingExpParams( + const struct ur_enqueue_timestamp_recording_exp_params_t *params, + char *buffer, const size_t buff_size, size_t *out_size) { std::stringstream ss; ss << params; return str_copy(&ss, buffer, buff_size, out_size); } -ur_result_t urPrintEnqueueTimestampRecordingExpParams( - const struct ur_enqueue_timestamp_recording_exp_params_t *params, - char *buffer, const size_t buff_size, size_t *out_size) { +ur_result_t urPrintEnqueueCommandBufferExpParams( + const struct ur_enqueue_command_buffer_exp_params_t *params, char *buffer, + const size_t buff_size, size_t *out_size) { std::stringstream ss; ss << params; return str_copy(&ss, buffer, buff_size, out_size); diff --git a/unified-runtime/source/ur_api.cpp b/unified-runtime/source/ur_api.cpp index d4c8260affdc5..acef3acd7a2b8 100644 --- a/unified-runtime/source/ur_api.cpp +++ b/unified-runtime/source/ur_api.cpp @@ -7400,10 +7400,7 @@ ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( } /////////////////////////////////////////////////////////////////////////////// -/// @brief Create a Command-Buffer object -/// -/// @details -/// - Create a command-buffer object. +/// @brief Synchronizes with all queues on the device. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7411,58 +7408,25 @@ ur_result_t UR_APICALL urBindlessImagesSignalExternalSemaphoreExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` /// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pCommandBufferDesc` -/// + `NULL == phCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `hDevice == nullptr` /// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns -/// 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP -/// query. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferCreateExp( - /// [in] Handle of the context object. - ur_context_handle_t hContext, - /// [in] Handle of the device object. - ur_device_handle_t hDevice, - /// [in] Command-buffer descriptor. - const ur_exp_command_buffer_desc_t *pCommandBufferDesc, - /// [out][alloc] Pointer to command-Buffer handle. - ur_exp_command_buffer_handle_t *phCommandBuffer) { +ur_result_t UR_APICALL urDeviceWaitExp( + /// [in] handle of the device instance. + ur_device_handle_t hDevice) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Increment the command-buffer object's reference count. +/// @brief Creates dynamic links between exported and imported symbols in one or +/// more programs. /// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -ur_result_t UR_APICALL urCommandBufferRetainExp( - /// [in][retain] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - ur_result_t result = UR_RESULT_SUCCESS; - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Decrement the command-buffer object's reference count and delete the -/// command-buffer object if the reference count becomes zero. It is -/// legal to call the entry-point while `hCommandBuffer` is still -/// executing, which will block on completion if the reference count of -/// `hCommandBuffer` becomes zero. +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point the programs in +/// `phPrograms` will have all external symbols resolved and kernels +/// inside these programs would be ready for use. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7470,42 +7434,35 @@ ur_result_t UR_APICALL urCommandBufferRetainExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -ur_result_t UR_APICALL urCommandBufferReleaseExp( - /// [in][release] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phPrograms` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If one of the programs in `phPrograms` isn't a valid program +/// object. +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `count == 0` +/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE +/// + If an error occurred while linking `phPrograms`. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the adapter has no means to support the operation. +ur_result_t UR_APICALL urProgramDynamicLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Stop recording on a command-buffer object such that no more commands -/// can be appended and make it ready to enqueue. +/// @brief Enqueue a command to execute a kernel /// -/// @returns -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_UNINITIALIZED -/// - ::UR_RESULT_ERROR_DEVICE_LOST -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "If `hCommandBuffer` has already -/// been finalized" -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferFinalizeExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer) { - ur_result_t result = UR_RESULT_SUCCESS; - return result; -} - -/////////////////////////////////////////////////////////////////////////////// -/// @brief Append a kernel execution command to a command-buffer object. +/// @remarks +/// _Analogues_ +/// - **clEnqueueNDRangeKernel** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7513,85 +7470,82 @@ ur_result_t UR_APICALL urCommandBufferFinalizeExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hQueue` /// + `NULL == hKernel` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER /// + `NULL == pGlobalWorkSize` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// + `pArgs == NULL && numArgs > 0` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `NULL != pArgs && ::UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type` +/// + `NULL != launchPropList && ::UR_KERNEL_LAUNCH_FLAGS_MASK & +/// launchPropList->flags` +/// - ::UR_RESULT_ERROR_INVALID_QUEUE /// - ::UR_RESULT_ERROR_INVALID_KERNEL -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + `phKernelAlternatives == NULL && numKernelAlternatives > 0` -/// + `phKernelAlternatives != NULL && numKernelAlternatives == 0` -/// + If `phKernelAlternatives` contains `hKernel` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` /// - ::UR_RESULT_ERROR_INVALID_EVENT /// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST /// + `phEventWaitList == NULL && numEventsInWaitList > 0` /// + `phEventWaitList != NULL && numEventsInWaitList == 0` /// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_IN_EVENT_LIST_EXEC_STATUS +/// + An event in `phEventWaitList` has ::UR_EVENT_STATUS_ERROR. +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// + `pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || +/// pGlobalWorkSize[2] == 0` +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// + `pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == +/// 0 || pLocalWorkSize[2] == 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGS - "The kernel argument values +/// have not been specified." /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "phCommand is not NULL and -/// hCommandBuffer is not updatable." -ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Kernel to append. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If any property in `launchPropList` isn't supported by the device. +ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] handle of the kernel object ur_kernel_handle_t hKernel, - /// [in] Dimension of the kernel execution. + /// [in] number of dimensions, from 1 to 3, to specify the global and + /// work-group work-items uint32_t workDim, - /// [in][optional] Offset to use when executing kernel. + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the offset used to calculate the global ID of a work-item const size_t *pGlobalWorkOffset, - /// [in] Global work size to use when executing kernel. + /// [in] pointer to an array of workDim unsigned values that specify the + /// number of global work-items in workDim that will execute the kernel + /// function const size_t *pGlobalWorkSize, - /// [in][optional] Local work size to use when executing kernel. If this - /// parameter is nullptr, then a local work size will be generated by the - /// implementation. + /// [in][optional] pointer to an array of workDim unsigned values that + /// specify the number of local work-items forming a work-group that will + /// execute the kernel function. + /// If nullptr, the runtime implementation will choose the work-group size. const size_t *pLocalWorkSize, - /// [in] The number of kernel alternatives provided in - /// phKernelAlternatives. - uint32_t numKernelAlternatives, - /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles - /// that might be used to update the kernel in this - /// command after the command-buffer is finalized. The default kernel - /// `hKernel` is implicitly marked as an alternative. It's - /// invalid to specify it as part of this list. - ur_kernel_handle_t *phKernelAlternatives, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. + /// [in] Number of entries in pArgs + uint32_t numArgs, + /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg + /// properties. + const ur_exp_kernel_arg_properties_t *pArgs, + /// [in][optional] pointer to a single linked list of launch properties + const ur_kernel_launch_ext_properties_t *launchPropList, + /// [in] size of the event wait list uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait + /// event. const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. Only available if the - /// command-buffer is updatable. - ur_exp_command_buffer_command_handle_t *phCommand) { + /// [out][optional][alloc] return an event object that identifies this + /// particular kernel execution instance. If phEventWaitList and phEvent + /// are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM memcpy command to a command-buffer object. +/// @brief Enqueue a command for recording the device timestamp /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7599,64 +7553,44 @@ ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hQueue` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pSrc` or `pDst` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// + `NULL == phEvent` /// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST /// + `phEventWaitList == NULL && numEventsInWaitList > 0` /// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Location the data will be copied to. - void *pDst, - /// [in] The data to be copied. - const void *pSrc, - /// [in] The number of bytes to copy. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. +ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( + /// [in] handle of the queue object + ur_queue_handle_t hQueue, + /// [in] indicates whether the call to this function should block until + /// until the device timestamp recording command has executed on the + /// device. + bool blocking, + /// [in] size of the event wait list uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. + /// events that must be complete before the kernel execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { + /// [in,out] return an event object that identifies this particular kernel + /// execution instance. Profiling information can be queried + /// from this event as if `hQueue` had profiling enabled. Querying + /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` + /// reports the timestamp at the time of the call to this function. + /// Querying `UR_PROFILING_INFO_COMMAND_START` or + /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the + /// command is executed on the device. If phEventWaitList and phEvent are + /// not NULL, phEvent must not refer to an element of the phEventWaitList + /// array. + ur_event_handle_t *phEvent) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM fill command to a command-buffer object. +/// @brief Gets an inter-process memory handle for a pointer to device USM +/// memory /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7664,68 +7598,28 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` -/// + `NULL == pPattern` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `patternSize == 0 || size == 0` -/// + `patternSize > size` -/// + `size % patternSize != 0` -/// + If `size` is higher than the allocation size of `ptr` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == ppIPCMemHandleData` +/// + `NULL == pIPCMemHandleDataSizeRet` /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to fill. - void *pMemory, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +ur_result_t UR_APICALL urIPCGetMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory + void *pMem, + /// [out][optional] a pointer to the IPC memory handle data + void **ppIPCMemHandleData, + /// [out][optional] size of the resulting IPC memory handle data + size_t *pIPCMemHandleDataSizeRet) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory copy command to a command-buffer object. +/// @brief Releases an inter-process memory handle /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7733,64 +7627,24 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hSrcMem` -/// + `NULL == hDstMem` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pIPCMemHandleData` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Offset into the source memory. - size_t srcOffset, - /// [in] Offset into the destination memory - size_t dstOffset, - /// [in] The number of bytes to be copied. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +ur_result_t UR_APICALL urIPCPutMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] a pointer to the IPC memory handle data + void *pIPCMemHandleData) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory write command to a command-buffer object. +/// @brief Opens an inter-process memory handle to get the corresponding pointer +/// to device USM memory /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7798,63 +7652,35 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pIPCMemHandleData` +/// + `NULL == ppMem` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + ipcMemHandleDataSize is not the same as the size of IPC memory +/// handle data /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written from. - const void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +ur_result_t UR_APICALL urIPCOpenMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] handle of the device object the corresponding USM device memory + /// was allocated on + ur_device_handle_t hDevice, + /// [in] the IPC memory handle data + void *pIPCMemHandleData, + /// [in] size of the IPC memory handle data + size_t ipcMemHandleDataSize, + /// [out] pointer to a pointer to device USM memory + void **ppMem) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory read command to a command-buffer object. +/// @brief Closes an inter-process memory handle /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7862,63 +7688,24 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMem` /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] Offset in bytes in the buffer object. - size_t offset, - /// [in] Size in bytes of data being written. - size_t size, - /// [in] Pointer to host memory where data is to be written to. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +ur_result_t UR_APICALL urIPCCloseMemHandleExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp + void *pMem) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory copy command to a command-buffer object. +/// @brief Allocate an exportable memory region and return a pointer to that +/// allocation. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7926,72 +7713,47 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hSrcMem` -/// + `NULL == hDstMem` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == ppMem` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT +/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If `alignment` exceeds largest supported data type by `hDevice` +/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE +/// + `size == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE +/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - +/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] The data to be copied. - ur_mem_handle_t hSrcMem, - /// [in] The location the data will be copied to. - ur_mem_handle_t hDstMem, - /// [in] Origin for the region of data to be copied from the source. - ur_rect_offset_t srcOrigin, - /// [in] Origin for the region of data to be copied to in the destination. - ur_rect_offset_t dstOrigin, - /// [in] The extents describing the region to be copied. - ur_rect_region_t region, - /// [in] Row pitch of the source memory. - size_t srcRowPitch, - /// [in] Slice pitch of the source memory. - size_t srcSlicePitch, - /// [in] Row pitch of the destination memory. - size_t dstRowPitch, - /// [in] Slice pitch of the destination memory. - size_t dstSlicePitch, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( + /// [in] Handle to context in which to allocate memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to allocate memory. + ur_device_handle_t hDevice, + /// [in] Requested alignment of the allocation. + size_t alignment, + /// [in] Requested size of the allocation. + size_t size, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [out][alloc] Pointer to allocated exportable memory. + void **ppMem) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory write command to a command-buffer object. +/// @brief Free an exportable memory allocation. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -7999,76 +7761,33 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pSrc` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being - /// written. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pSrc. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pSrc. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be written from. - void *pSrc, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( + /// [in] Handle to context in which to free memory. + ur_context_handle_t hContext, + /// [in] Handle to device on which to free memory. + ur_device_handle_t hDevice, + /// [in][release] Pointer to exportable memory to be deallocated. + void *pMem) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a rectangular memory read command to a command-buffer object. +/// @brief Obtain an exportable handle to a memory allocated with +/// `AllocExportableMemoryExp`.The returned external memory type will be +/// that which was specified upon +/// allocation of the exportable memory (e.g. `opaque_fd` or +/// `win32_nt_handle`). /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8076,75 +7795,47 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < +/// handleTypeToExport` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. +/// + `NULL == pMemHandleRet || NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE +/// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] 3D offset in the buffer. - ur_rect_offset_t bufferOffset, - /// [in] 3D offset in the host region. - ur_rect_offset_t hostOffset, - /// [in] 3D rectangular region descriptor: width, height, depth. - ur_rect_region_t region, - /// [in] Length of each row in bytes in the buffer object. - size_t bufferRowPitch, - /// [in] Length of each 2D slice in bytes in the buffer object being read. - size_t bufferSlicePitch, - /// [in] Length of each row in bytes in the host memory region pointed to - /// by pDst. - size_t hostRowPitch, - /// [in] Length of each 2D slice in bytes in the host memory region - /// pointed to by pDst. - size_t hostSlicePitch, - /// [in] Pointer to host memory where data is to be read into. - void *pDst, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional] return an event object that will be signaled by the - /// completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY +ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( + /// [in] Handle to context in which the exportable memory was allocated. + ur_context_handle_t hContext, + /// [in] Handle to device on which the exportable memory was allocated. + ur_device_handle_t hDevice, + /// [in] Type of the memory handle to be exported (e.g. file descriptor, + /// or win32 NT handle). + ur_exp_external_mem_type_t handleTypeToExport, + /// [in] Pointer to exportable memory handle. + void *pMem, + /// [out] Returned exportable handle to memory allocated in `pMem` + void *pMemHandleRet) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a memory fill command to a command-buffer object. +/// @brief Produces an executable program from one program, negates need for the +/// linking step. +/// +/// @details +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point, the program passed +/// will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type +/// for each device in `phDevices`. +/// +/// @remarks +/// _Analogues_ +/// - **clBuildProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8152,72 +7843,42 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == hBuffer` +/// + `NULL == hProgram` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pPattern` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + If `offset + size` results in an out-of-bounds access. -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] handle of the buffer object. - ur_mem_handle_t hBuffer, - /// [in] pointer to the fill pattern. - const void *pPattern, - /// [in] size in bytes of the pattern. - size_t patternSize, - /// [in] offset into the buffer. - size_t offset, - /// [in] fill size in bytes, must be a multiple of patternSize. - size_t size, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// + `NULL == phDevices` +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If `hProgram` isn't a valid program object. +/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE +/// + If an error occurred when building `hProgram`. +ur_result_t UR_APICALL urProgramBuildExp( + /// [in] Handle of the program to build. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM Prefetch command to a command-buffer object. +/// @brief Produces an executable program from one or more programs. /// /// @details -/// - Prefetching may not be supported for all devices or allocation types. -/// If memory prefetching is not supported, the prefetch hint will be -/// ignored. +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point `hProgram` will +/// contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type +/// for each device in `phDevices`. +/// +/// @remarks +/// _Analogues_ +/// - **clCompileProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8225,70 +7886,48 @@ ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hProgram` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` +/// + `NULL == phDevices` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_USM_MIGRATION_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pMemory` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to USM allocated memory to prefetch. - const void *pMemory, - /// [in] size in bytes to be fetched. - size_t size, - /// [in] USM migration flags - ur_usm_migration_flags_t flags, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If `hProgram` isn't a valid program object. +/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE +/// + If an error occurred while compiling `hProgram`. +ur_result_t UR_APICALL urProgramCompileExp( + /// [in][out] handle of the program to compile. + ur_program_handle_t hProgram, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in][optional] pointer to build options null-terminated string. + const char *pOptions) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append a USM Advise command to a command-buffer object. +/// @brief Produces an executable program from one or more programs. /// /// @details -/// - Not all memory advice hints may be supported for all devices or -/// allocation types. If a memory advice hint is not supported, it will be -/// ignored. +/// - The application may call this function from simultaneous threads. +/// - Following a successful call to this entry point the program returned +/// in `phProgram` will contain a binary of the +/// ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in +/// `phDevices`. +/// - If a non-success code is returned, adapters may store a program in +/// `phProgram`. This program should only be used with +/// `::urProgramGetBuildInfo` to get the build log for the failure. +/// Adapters which do not do not support producing build logs must set +/// this value to `nullptr`. +/// +/// @remarks +/// _Analogues_ +/// - **clLinkProgram** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8296,65 +7935,44 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemory` +/// + `NULL == phDevices` +/// + `NULL == phPrograms` +/// + `NULL == phProgram` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_USM_ADVICE_FLAGS_MASK & advice` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_PROGRAM +/// + If one of the programs in `phPrograms` isn't a valid program +/// object. /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pMemory` -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the device associated with `hCommandBuffer` does not support -/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` -/// or `phEventWaitList` are not NULL. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( - /// [in] handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] pointer to the USM memory object. - const void *pMemory, - /// [in] size in bytes to be advised. - size_t size, - /// [in] USM memory advice - ur_usm_advice_flags_t advice, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional] sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint, - /// [out][optional][alloc] return an event object that will be signaled by - /// the completion of this command in the next execution of the - /// command-buffer. - ur_event_handle_t *phEvent, - /// [out][optional][alloc] Handle to this command. - ur_exp_command_buffer_command_handle_t *phCommand) { +/// + `count == 0` +/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE +/// + If an error occurred while linking `phPrograms`. +ur_result_t UR_APICALL urProgramLinkExp( + /// [in] handle of the context instance. + ur_context_handle_t hContext, + /// [in] number of devices + uint32_t numDevices, + /// [in][range(0, numDevices)] pointer to array of device handles + ur_device_handle_t *phDevices, + /// [in] program information flags + ur_exp_program_flags_t flags, + /// [in] number of program handles in `phPrograms`. + uint32_t count, + /// [in][range(0, count)] pointer to array of program handles. + const ur_program_handle_t *phPrograms, + /// [in][optional] pointer to linker options null-terminated string. + const char *pOptions, + /// [out][alloc] pointer to handle of program object created. + ur_program_handle_t *phProgram) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Append nodes to the command-buffer through a native backend API +/// @brief Perform a synchronous, blocking memcpy operation between USM +/// allocations. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8362,44 +7980,34 @@ ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pfnNativeCommand` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP -/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` -/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Function calling the native underlying API, to be executed - /// immediately. - ur_exp_command_buffer_native_command_function_t pfnNativeCommand, - /// [in][optional] Data used by pfnNativeCommand - void *pData, - /// [in][optional] A command-buffer object which will be added to - /// hCommandBuffer as a child graph node containing the native commands. - /// Required for CUDA and HIP adapters and will be ignored by other - /// adapters, who use alternative backend mechanisms to add the native - /// nodes to hCommandBuffer. - ur_exp_command_buffer_handle_t hChildCommandBuffer, - /// [in] The number of sync points in the provided dependency list. - uint32_t numSyncPointsInWaitList, - /// [in][optional] A list of sync points that this command depends on. - /// Will be ignored if command-buffer is in-order. - const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, - /// [out][optional] Sync point associated with this command. - ur_exp_command_buffer_sync_point_t *pSyncPoint) { +/// + `NULL == pDst` +/// + `NULL == pSrc` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pSrc` or `pDst` +ur_result_t UR_APICALL urUSMContextMemcpyExp( + /// [in] Context associated with the device(s) that own the allocations + /// `pSrc` and `pDst`. + ur_context_handle_t hContext, + /// [in] Destination pointer to copy to. + void *pDst, + /// [in] Source pointer to copy from. + const void *pSrc, + /// [in] Size in bytes to be copied. + size_t size) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Submit a command-buffer for execution on a queue. +/// @brief Import memory into USM +/// +/// @details +/// - Import memory into USM /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8407,45 +8015,27 @@ ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` -/// + `NULL == hCommandBuffer` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_QUEUE -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urEnqueueCommandBufferExp( - /// [in] The queue to submit this command-buffer for execution. - ur_queue_handle_t hQueue, - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command-buffer execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular command-buffer execution instance. If phEventWaitList and - /// phEvent are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { +/// + `NULL == hContext` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +ur_result_t UR_APICALL urUSMImportExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem, + /// [in] size in bytes of the host memory object to be imported + size_t size) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Update a kernel launch command in a finalized command-buffer. +/// @brief Release memory from USM /// /// @details -/// This entry-point is synchronous and may block if the command-buffer is -/// executing when the entry-point is called. On error, the state of the -/// command-buffer commands being updated is undefined. +/// - Release memory from USM /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8453,124 +8043,88 @@ ur_result_t UR_APICALL urEnqueueCommandBufferExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` -/// + `NULL == pUpdateKernelLaunch->hCommand` +/// + `NULL == hContext` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pUpdateKernelLaunch` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `numKernelUpdates == 0` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS -/// is not supported by the device, and for any of any element of -/// `pUpdateKernelLaunch` the `numNewMemObjArgs`, `numNewPointerArgs`, -/// or `numNewValueArgs` members are not zero. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is -/// not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is not nullptr. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is -/// not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is nullptr and -/// `pNewGlobalWorkSize` is not nullptr. -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewGlobalWorkSize` member is not nullptr -/// + If -/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `pNewGlobalWorkOffset` member is not -/// nullptr. -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE -/// is not supported by the device, and for any element of -/// `pUpdateKernelLaunch` the `hNewKernel` member is not nullptr. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the `hCommandBuffer`. -/// + If `hCommandBuffer` has not been finalized. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member -/// is not a kernel execution command. -/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member -/// was not created from `hCommandBuffer`. -/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` -/// member is less than 1 or greater than 3. -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + If for any element of `pUpdateKernelLaunch` the `hNewKernel` -/// member was not passed to the `hKernel` or `phKernelAlternatives` -/// parameters of ::urCommandBufferAppendKernelLaunchExp when the -/// command was created. -/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` -/// member is different from the current workDim in the `hCommand` -/// member, and `pNewGlobalWorkSize` or `pNewGlobalWorkOffset` are -/// nullptr. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( - /// [in] Handle of the command-buffer object. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] Length of pUpdateKernelLaunch. - uint32_t numKernelUpdates, - /// [in][range(0, numKernelUpdates)] List of structs defining how a - /// kernel commands are to be updated. - const ur_exp_command_buffer_update_kernel_launch_desc_t - *pUpdateKernelLaunch) { +/// + `NULL == pMem` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +ur_result_t UR_APICALL urUSMReleaseExp( + /// [in] handle of the context object + ur_context_handle_t hContext, + /// [in] pointer to host memory object + void *pMem) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Get a new event that will be signaled the next time the command in -/// the -/// command-buffer executes. +/// @brief Enable access to peer device memory /// /// @details -/// It is the users responsibility to release the returned `phSignalEvent`. +/// - Enables the command device to access and write device memory +/// allocations located on the peer device, provided that a P2P link +/// between the two devices is available. +/// - When Peer Access is successfully enabled, P2P memory accesses are +/// guaranteed to be allowed on the peer device until +/// ::urUsmP2PDisablePeerAccessExp is called. +/// - Note that the function operands may, but aren't guaranteed to, commute +/// for a given adapter: the peer device is not guaranteed to have access +/// to device memory allocations located on the command device. +/// - It is not guaranteed that the commutation relations of the function +/// arguments are identical for peer access and peer copies: For example, +/// for a given adapter the peer device may be able to copy data from the +/// command device, but not access and write the same data on the command +/// device. +/// - Consult the appropriate adapter driver documentation for details of +/// adapter specific behavior and native error codes that may be returned. /// -/// @returns +/// @remarks +/// _Analogues_ +/// - **cuCtxEnablePeerAccess** +/// +/// @returns /// - ::UR_RESULT_SUCCESS /// - ::UR_RESULT_ERROR_UNINITIALIZED /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommand` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phSignalEvent` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not -/// supported by the device associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the command-buffer `hCommand` belongs to. -/// + If the command-buffer `hCommand` belongs to has not been -/// finalized. -/// + If no `phEvent` parameter was set on creation of the command -/// associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [out][alloc] Event to be signaled. - ur_event_handle_t *phSignalEvent) { +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Set the list of wait events for a command to depend on to a list of -/// new events. +/// @brief Disable access to peer device memory +/// +/// @details +/// - Disables the ability of the command device to access and write device +/// memory allocations located on the peer device, provided that a P2P +/// link between the two devices was enabled prior to the call. +/// - Note that the function operands may, but aren't guaranteed to, commute +/// for a given adapter. If, prior to the function call, the peer device +/// had access to device memory allocations on the command device, it is +/// not guaranteed to still have such access following the function +/// return. +/// - It is not guaranteed that the commutation relations of the function +/// arguments are identical for peer access and peer copies: For example +/// for a given adapter, if, prior to the call, the peer device had access +/// to device memory allocations on the command device, the peer device +/// may still, following the function call, be able to copy data from the +/// command device, but not access and write the same data on the command +/// device. +/// - Consult the appropriate adapter driver documentation for details of +/// adapter specific behavior and native error codes that may be returned. +/// +/// @remarks +/// _Analogues_ +/// - **cuCtxDisablePeerAccess** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8578,39 +8132,29 @@ ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommand` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not -/// supported by the device associated with `hCommand`. -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true -/// on creation of the command-buffer `hCommand` belongs to. -/// + If the command-buffer `hCommand` belongs to has not been -/// finalized. -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// + If `numEventsInWaitList` does not match the number of wait events -/// set when the command associated with `hCommand` was created. -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( - /// [in] Handle of the command-buffer command to update. - ur_exp_command_buffer_command_handle_t hCommand, - /// [in] Size of the event wait list. - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the command execution. If nullptr, - /// the numEventsInWaitList must be 0, indicating no wait events. - const ur_event_handle_t *phEventWaitList) { +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Get command-buffer object information. +/// @brief Disable access to peer device memory +/// +/// @details +/// - Queries the peer access capabilities from the command device to the +/// peer device according to the query `propName`. +/// +/// @remarks +/// _Analogues_ +/// - **cuDeviceGetP2PAttribute** /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8618,9 +8162,10 @@ ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == commandDevice` +/// + `NULL == peerDevice` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName` +/// + `::UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -8630,31 +8175,36 @@ ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER /// + `propSize != 0 && pPropValue == NULL` /// + `pPropValue == NULL && pPropSizeRet == NULL` -/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -ur_result_t UR_APICALL urCommandBufferGetInfoExp( - /// [in] handle of the command-buffer object - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [in] the name of the command-buffer property to query - ur_exp_command_buffer_info_t propName, - /// [in] size in bytes of the command-buffer property value +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( + /// [in] handle of the command device object + ur_device_handle_t commandDevice, + /// [in] handle of the peer device object + ur_device_handle_t peerDevice, + /// [in] type of the info to retrieve + ur_exp_peer_info_t propName, + /// [in] the number of bytes pointed to by pPropValue. size_t propSize, - /// [out][optional][typename(propName, propSize)] value of the - /// command-buffer property + /// [out][optional][typename(propName, propSize)] array of bytes holding + /// the info. + /// If propSize is not equal to or greater than the real number of bytes + /// needed to return the info + /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and + /// pPropValue is not used. void *pPropValue, - /// [out][optional] bytes returned in command-buffer property + /// [out][optional] pointer to the actual size in bytes of the queried + /// propName. size_t *pPropSizeRet) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Return platform native command-buffer handle. +/// @brief Create a Command-Buffer object /// /// @details -/// - Retrieved native handle can be used for direct interaction with the -/// native platform driver. +/// - Create a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8662,22 +8212,34 @@ ur_result_t UR_APICALL urCommandBufferGetInfoExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hCommandBuffer` +/// + `NULL == hContext` +/// + `NULL == hDevice` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phNativeCommandBuffer` +/// + `NULL == pCommandBufferDesc` +/// + `NULL == phCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// - ::UR_RESULT_ERROR_INVALID_DEVICE /// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the adapter has no underlying equivalent handle. -ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( - /// [in] Handle of the command-buffer. - ur_exp_command_buffer_handle_t hCommandBuffer, - /// [out] A pointer to the native handle of the command-buffer. - ur_native_handle_t *phNativeCommandBuffer) { +/// + If `pCommandBufferDesc->isUpdatable` is true and `hDevice` returns +/// 0 for the ::UR_DEVICE_INFO_COMMAND_BUFFER_UPDATE_CAPABILITIES_EXP +/// query. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferCreateExp( + /// [in] Handle of the context object. + ur_context_handle_t hContext, + /// [in] Handle of the device object. + ur_device_handle_t hDevice, + /// [in] Command-buffer descriptor. + const ur_exp_command_buffer_desc_t *pCommandBufferDesc, + /// [out][alloc] Pointer to command-Buffer handle. + ur_exp_command_buffer_handle_t *phCommandBuffer) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Synchronizes with all queues on the device. +/// @brief Increment the command-buffer object's reference count. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8685,25 +8247,44 @@ ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hDevice` -/// + `hDevice == nullptr` -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -ur_result_t UR_APICALL urDeviceWaitExp( - /// [in] handle of the device instance. - ur_device_handle_t hDevice) { +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +ur_result_t UR_APICALL urCommandBufferRetainExp( + /// [in][retain] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Creates dynamic links between exported and imported symbols in one or -/// more programs. +/// @brief Decrement the command-buffer object's reference count and delete the +/// command-buffer object if the reference count becomes zero. It is +/// legal to call the entry-point while `hCommandBuffer` is still +/// executing, which will block on completion if the reference count of +/// `hCommandBuffer` becomes zero. /// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point the programs in -/// `phPrograms` will have all external symbols resolved and kernels -/// inside these programs would be ready for use. +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +ur_result_t UR_APICALL urCommandBufferReleaseExp( + /// [in][release] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Stop recording on a command-buffer object such that no more commands +/// can be appended and make it ready to enqueue. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8711,31 +8292,21 @@ ur_result_t UR_APICALL urDeviceWaitExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phPrograms` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If one of the programs in `phPrograms` isn't a valid program -/// object. -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `count == 0` -/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE -/// + If an error occurred while linking `phPrograms`. -/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE -/// + If the adapter has no means to support the operation. -ur_result_t UR_APICALL urProgramDynamicLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms) { +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "If `hCommandBuffer` has already +/// been finalized" +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferFinalizeExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Enqueue a command for recording the device timestamp +/// @brief Append a kernel execution command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8743,44 +8314,85 @@ ur_result_t UR_APICALL urProgramDynamicLinkExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` +/// + `NULL == hCommandBuffer` +/// + `NULL == hKernel` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phEvent` +/// + `NULL == pGlobalWorkSize` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_KERNEL +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + `phKernelAlternatives == NULL && numKernelAlternatives > 0` +/// + `phKernelAlternatives != NULL && numKernelAlternatives == 0` +/// + If `phKernelAlternatives` contains `hKernel` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT /// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST /// + `phEventWaitList == NULL && numEventsInWaitList > 0` /// + `phEventWaitList != NULL && numEventsInWaitList == 0` -ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] indicates whether the call to this function should block until - /// until the device timestamp recording command has executed on the - /// device. - bool blocking, - /// [in] size of the event wait list +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_INVALID_OPERATION - "phCommand is not NULL and +/// hCommandBuffer is not updatable." +ur_result_t UR_APICALL urCommandBufferAppendKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Kernel to append. + ur_kernel_handle_t hKernel, + /// [in] Dimension of the kernel execution. + uint32_t workDim, + /// [in][optional] Offset to use when executing kernel. + const size_t *pGlobalWorkOffset, + /// [in] Global work size to use when executing kernel. + const size_t *pGlobalWorkSize, + /// [in][optional] Local work size to use when executing kernel. If this + /// parameter is nullptr, then a local work size will be generated by the + /// implementation. + const size_t *pLocalWorkSize, + /// [in] The number of kernel alternatives provided in + /// phKernelAlternatives. + uint32_t numKernelAlternatives, + /// [in][optional][range(0, numKernelAlternatives)] List of kernel handles + /// that might be used to update the kernel in this + /// command after the command-buffer is finalized. The default kernel + /// `hKernel` is implicitly marked as an alternative. It's + /// invalid to specify it as part of this list. + ur_kernel_handle_t *phKernelAlternatives, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. uint32_t numEventsInWaitList, /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating no wait - /// events. + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. const ur_event_handle_t *phEventWaitList, - /// [in,out] return an event object that identifies this particular kernel - /// execution instance. Profiling information can be queried - /// from this event as if `hQueue` had profiling enabled. Querying - /// `UR_PROFILING_INFO_COMMAND_QUEUED` or `UR_PROFILING_INFO_COMMAND_SUBMIT` - /// reports the timestamp at the time of the call to this function. - /// Querying `UR_PROFILING_INFO_COMMAND_START` or - /// `UR_PROFILING_INFO_COMMAND_END` reports the timestamp recorded when the - /// command is executed on the device. If phEventWaitList and phEvent are - /// not NULL, phEvent must not refer to an element of the phEventWaitList - /// array. - ur_event_handle_t *phEvent) { + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. Only available if the + /// command-buffer is updatable. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Gets an inter-process memory handle for a pointer to device USM -/// memory +/// @brief Append a USM memcpy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8788,28 +8400,64 @@ ur_result_t UR_APICALL urEnqueueTimestampRecordingExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == ppIPCMemHandleData` -/// + `NULL == pIPCMemHandleDataSizeRet` +/// + `NULL == pDst` +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pSrc` or `pDst` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCGetMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory - void *pMem, - /// [out][optional] a pointer to the IPC memory handle data - void **ppIPCMemHandleData, - /// [out][optional] size of the resulting IPC memory handle data - size_t *pIPCMemHandleDataSizeRet) { +ur_result_t UR_APICALL urCommandBufferAppendUSMMemcpyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Location the data will be copied to. + void *pDst, + /// [in] The data to be copied. + const void *pSrc, + /// [in] The number of bytes to copy. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Releases an inter-process memory handle +/// @brief Append a USM fill command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8817,24 +8465,68 @@ ur_result_t UR_APICALL urIPCGetMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pIPCMemHandleData` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == pMemory` +/// + `NULL == pPattern` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `patternSize == 0 || size == 0` +/// + `patternSize > size` +/// + `size % patternSize != 0` +/// + If `size` is higher than the allocation size of `ptr` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCPutMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] a pointer to the IPC memory handle data - void *pIPCMemHandleData) { +ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to fill. + void *pMemory, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Opens an inter-process memory handle to get the corresponding pointer -/// to device USM memory +/// @brief Append a memory copy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8842,35 +8534,64 @@ ur_result_t UR_APICALL urIPCPutMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pIPCMemHandleData` -/// + `NULL == ppMem` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + ipcMemHandleDataSize is not the same as the size of IPC memory -/// handle data +/// + `NULL == hCommandBuffer` +/// + `NULL == hSrcMem` +/// + `NULL == hDstMem` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCOpenMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] handle of the device object the corresponding USM device memory - /// was allocated on - ur_device_handle_t hDevice, - /// [in] the IPC memory handle data - void *pIPCMemHandleData, - /// [in] size of the IPC memory handle data - size_t ipcMemHandleDataSize, - /// [out] pointer to a pointer to device USM memory - void **ppMem) { +ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Offset into the source memory. + size_t srcOffset, + /// [in] Offset into the destination memory + size_t dstOffset, + /// [in] The number of bytes to be copied. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Closes an inter-process memory handle +/// @brief Append a memory write command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8878,24 +8599,63 @@ ur_result_t UR_APICALL urIPCOpenMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY /// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -ur_result_t UR_APICALL urIPCCloseMemHandleExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to device USM memory opened through urIPCOpenMemHandleExp - void *pMem) { +ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. + size_t size, + /// [in] Pointer to host memory where data is to be written from. + const void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Allocate an exportable memory region and return a pointer to that -/// allocation. +/// @brief Append a memory read command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8903,47 +8663,63 @@ ur_result_t UR_APICALL urIPCCloseMemHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < -/// handleTypeToExport` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == ppMem` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT -/// + `alignment != 0 && ((alignment & (alignment-1)) != 0)` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// + If `alignment` exceeds largest supported data type by `hDevice` -/// - ::UR_RESULT_ERROR_INVALID_USM_SIZE -/// + `size == 0` -/// - ::UR_RESULT_ERROR_UNSUPPORTED_SIZE -/// + `size` is greater than ::UR_DEVICE_INFO_MAX_MEM_ALLOC_SIZE. -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - -/// ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE:DEVICE_INFO_MEMORY_EXPORT_LINEAR_MEMORY_EXPORT_SUPPORT_EXP +/// + `NULL == pDst` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( - /// [in] Handle to context in which to allocate memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to allocate memory. - ur_device_handle_t hDevice, - /// [in] Requested alignment of the allocation. - size_t alignment, - /// [in] Requested size of the allocation. +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] Offset in bytes in the buffer object. + size_t offset, + /// [in] Size in bytes of data being written. size_t size, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [out][alloc] Pointer to allocated exportable memory. - void **ppMem) { + /// [in] Pointer to host memory where data is to be written to. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Free an exportable memory allocation. +/// @brief Append a rectangular memory copy command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8951,33 +8727,72 @@ ur_result_t UR_APICALL urMemoryExportAllocExportableMemoryExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// + `NULL == hCommandBuffer` +/// + `NULL == hSrcMem` +/// + `NULL == hDstMem` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( - /// [in] Handle to context in which to free memory. - ur_context_handle_t hContext, - /// [in] Handle to device on which to free memory. - ur_device_handle_t hDevice, - /// [in][release] Pointer to exportable memory to be deallocated. - void *pMem) { +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferCopyRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] The data to be copied. + ur_mem_handle_t hSrcMem, + /// [in] The location the data will be copied to. + ur_mem_handle_t hDstMem, + /// [in] Origin for the region of data to be copied from the source. + ur_rect_offset_t srcOrigin, + /// [in] Origin for the region of data to be copied to in the destination. + ur_rect_offset_t dstOrigin, + /// [in] The extents describing the region to be copied. + ur_rect_region_t region, + /// [in] Row pitch of the source memory. + size_t srcRowPitch, + /// [in] Slice pitch of the source memory. + size_t srcSlicePitch, + /// [in] Row pitch of the destination memory. + size_t dstRowPitch, + /// [in] Slice pitch of the destination memory. + size_t dstSlicePitch, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Obtain an exportable handle to a memory allocated with -/// `AllocExportableMemoryExp`.The returned external memory type will be -/// that which was specified upon -/// allocation of the exportable memory (e.g. `opaque_fd` or -/// `win32_nt_handle`). +/// @brief Append a rectangular memory write command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -8985,47 +8800,152 @@ ur_result_t UR_APICALL urMemoryExportFreeExportableMemoryExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// + `NULL == hDevice` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX11_RESOURCE < -/// handleTypeToExport` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMemHandleRet || NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_DEVICE -/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// + `NULL == pSrc` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. /// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_DEVICE_MEMORY -ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( - /// [in] Handle to context in which the exportable memory was allocated. - ur_context_handle_t hContext, - /// [in] Handle to device on which the exportable memory was allocated. - ur_device_handle_t hDevice, - /// [in] Type of the memory handle to be exported (e.g. file descriptor, - /// or win32 NT handle). - ur_exp_external_mem_type_t handleTypeToExport, - /// [in] Pointer to exportable memory handle. - void *pMem, - /// [out] Returned exportable handle to memory allocated in `pMem` - void *pMemHandleRet) { +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferWriteRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being + /// written. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pSrc. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pSrc. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be written from. + void *pSrc, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one program, negates need for the -/// linking step. -/// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point, the program passed -/// will contain a binary of the ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type -/// for each device in `phDevices`. +/// @brief Append a rectangular memory read command to a command-buffer object. /// -/// @remarks -/// _Analogues_ -/// - **clBuildProgram** +/// @returns +/// - ::UR_RESULT_SUCCESS +/// - ::UR_RESULT_ERROR_UNINITIALIZED +/// - ::UR_RESULT_ERROR_DEVICE_LOST +/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC +/// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == pDst` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferReadRectExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] 3D offset in the buffer. + ur_rect_offset_t bufferOffset, + /// [in] 3D offset in the host region. + ur_rect_offset_t hostOffset, + /// [in] 3D rectangular region descriptor: width, height, depth. + ur_rect_region_t region, + /// [in] Length of each row in bytes in the buffer object. + size_t bufferRowPitch, + /// [in] Length of each 2D slice in bytes in the buffer object being read. + size_t bufferSlicePitch, + /// [in] Length of each row in bytes in the host memory region pointed to + /// by pDst. + size_t hostRowPitch, + /// [in] Length of each 2D slice in bytes in the host memory region + /// pointed to by pDst. + size_t hostSlicePitch, + /// [in] Pointer to host memory where data is to be read into. + void *pDst, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional] return an event object that will be signaled by the + /// completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { + ur_result_t result = UR_RESULT_SUCCESS; + return result; +} + +/////////////////////////////////////////////////////////////////////////////// +/// @brief Append a memory fill command to a command-buffer object. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9033,42 +8953,72 @@ ur_result_t UR_APICALL urMemoryExportExportMemoryHandleExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hProgram` +/// + `NULL == hCommandBuffer` +/// + `NULL == hBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If `hProgram` isn't a valid program object. -/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE -/// + If an error occurred when building `hProgram`. -ur_result_t UR_APICALL urProgramBuildExp( - /// [in] Handle of the program to build. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { +/// + `NULL == pPattern` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + If `offset + size` results in an out-of-bounds access. +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendMemBufferFillExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] handle of the buffer object. + ur_mem_handle_t hBuffer, + /// [in] pointer to the fill pattern. + const void *pPattern, + /// [in] size in bytes of the pattern. + size_t patternSize, + /// [in] offset into the buffer. + size_t offset, + /// [in] fill size in bytes, must be a multiple of patternSize. + size_t size, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one or more programs. +/// @brief Append a USM Prefetch command to a command-buffer object. /// /// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point `hProgram` will -/// contain a binary of the ::UR_PROGRAM_BINARY_TYPE_COMPILED_OBJECT type -/// for each device in `phDevices`. -/// -/// @remarks -/// _Analogues_ -/// - **clCompileProgram** +/// - Prefetching may not be supported for all devices or allocation types. +/// If memory prefetching is not supported, the prefetch hint will be +/// ignored. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9076,48 +9026,70 @@ ur_result_t UR_APICALL urProgramBuildExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hProgram` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` +/// + `NULL == pMemory` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If `hProgram` isn't a valid program object. -/// - ::UR_RESULT_ERROR_PROGRAM_BUILD_FAILURE -/// + If an error occurred while compiling `hProgram`. -ur_result_t UR_APICALL urProgramCompileExp( - /// [in][out] handle of the program to compile. - ur_program_handle_t hProgram, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in][optional] pointer to build options null-terminated string. - const char *pOptions) { +/// + `::UR_USM_MIGRATION_FLAGS_MASK & flags` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pMemory` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendUSMPrefetchExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to USM allocated memory to prefetch. + const void *pMemory, + /// [in] size in bytes to be fetched. + size_t size, + /// [in] USM migration flags + ur_usm_migration_flags_t flags, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Produces an executable program from one or more programs. -/// -/// @details -/// - The application may call this function from simultaneous threads. -/// - Following a successful call to this entry point the program returned -/// in `phProgram` will contain a binary of the -/// ::UR_PROGRAM_BINARY_TYPE_EXECUTABLE type for each device in -/// `phDevices`. -/// - If a non-success code is returned, adapters may store a program in -/// `phProgram`. This program should only be used with -/// `::urProgramGetBuildInfo` to get the build log for the failure. -/// Adapters which do not do not support producing build logs must set -/// this value to `nullptr`. +/// @brief Append a USM Advise command to a command-buffer object. /// -/// @remarks -/// _Analogues_ -/// - **clLinkProgram** +/// @details +/// - Not all memory advice hints may be supported for all devices or +/// allocation types. If a memory advice hint is not supported, it will be +/// ignored. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9125,44 +9097,65 @@ ur_result_t UR_APICALL urProgramCompileExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == phDevices` -/// + `NULL == phPrograms` -/// + `NULL == phProgram` +/// + `NULL == pMemory` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PROGRAM_FLAGS_MASK & flags` -/// - ::UR_RESULT_ERROR_INVALID_PROGRAM -/// + If one of the programs in `phPrograms` isn't a valid program -/// object. +/// + `::UR_USM_ADVICE_FLAGS_MASK & advice` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT /// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `count == 0` -/// - ::UR_RESULT_ERROR_PROGRAM_LINK_FAILURE -/// + If an error occurred while linking `phPrograms`. -ur_result_t UR_APICALL urProgramLinkExp( - /// [in] handle of the context instance. - ur_context_handle_t hContext, - /// [in] number of devices - uint32_t numDevices, - /// [in][range(0, numDevices)] pointer to array of device handles - ur_device_handle_t *phDevices, - /// [in] program information flags - ur_exp_program_flags_t flags, - /// [in] number of program handles in `phPrograms`. - uint32_t count, - /// [in][range(0, count)] pointer to array of program handles. - const ur_program_handle_t *phPrograms, - /// [in][optional] pointer to linker options null-terminated string. - const char *pOptions, - /// [out][alloc] pointer to handle of program object created. - ur_program_handle_t *phProgram) { +/// + `size == 0` +/// + If `size` is higher than the allocation size of `pMemory` +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the device associated with `hCommandBuffer` does not support +/// UR_DEVICE_INFO_COMMAND_BUFFER_EVENT_SUPPORT_EXP and either `phEvent` +/// or `phEventWaitList` are not NULL. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendUSMAdviseExp( + /// [in] handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] pointer to the USM memory object. + const void *pMemory, + /// [in] size in bytes to be advised. + size_t size, + /// [in] USM memory advice + ur_usm_advice_flags_t advice, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional] sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint, + /// [out][optional][alloc] return an event object that will be signaled by + /// the completion of this command in the next execution of the + /// command-buffer. + ur_event_handle_t *phEvent, + /// [out][optional][alloc] Handle to this command. + ur_exp_command_buffer_command_handle_t *phCommand) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Perform a synchronous, blocking memcpy operation between USM -/// allocations. +/// @brief Append nodes to the command-buffer through a native backend API /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9170,34 +9163,44 @@ ur_result_t UR_APICALL urProgramLinkExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pDst` -/// + `NULL == pSrc` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -/// - ::UR_RESULT_ERROR_INVALID_SIZE -/// + `size == 0` -/// + If `size` is higher than the allocation size of `pSrc` or `pDst` -ur_result_t UR_APICALL urUSMContextMemcpyExp( - /// [in] Context associated with the device(s) that own the allocations - /// `pSrc` and `pDst`. - ur_context_handle_t hContext, - /// [in] Destination pointer to copy to. - void *pDst, - /// [in] Source pointer to copy from. - const void *pSrc, - /// [in] Size in bytes to be copied. - size_t size) { +/// + `NULL == pfnNativeCommand` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_EXP +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_SYNC_POINT_WAIT_LIST_EXP +/// + `pSyncPointWaitList == NULL && numSyncPointsInWaitList > 0` +/// + `pSyncPointWaitList != NULL && numSyncPointsInWaitList == 0` +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferAppendNativeCommandExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Function calling the native underlying API, to be executed + /// immediately. + ur_exp_command_buffer_native_command_function_t pfnNativeCommand, + /// [in][optional] Data used by pfnNativeCommand + void *pData, + /// [in][optional] A command-buffer object which will be added to + /// hCommandBuffer as a child graph node containing the native commands. + /// Required for CUDA and HIP adapters and will be ignored by other + /// adapters, who use alternative backend mechanisms to add the native + /// nodes to hCommandBuffer. + ur_exp_command_buffer_handle_t hChildCommandBuffer, + /// [in] The number of sync points in the provided dependency list. + uint32_t numSyncPointsInWaitList, + /// [in][optional] A list of sync points that this command depends on. + /// Will be ignored if command-buffer is in-order. + const ur_exp_command_buffer_sync_point_t *pSyncPointWaitList, + /// [out][optional] Sync point associated with this command. + ur_exp_command_buffer_sync_point_t *pSyncPoint) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Import memory into USM -/// -/// @details -/// - Import memory into USM +/// @brief Submit a command-buffer for execution on a queue. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9205,27 +9208,45 @@ ur_result_t UR_APICALL urUSMContextMemcpyExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` -/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -/// - ::UR_RESULT_ERROR_INVALID_SIZE -ur_result_t UR_APICALL urUSMImportExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem, - /// [in] size in bytes of the host memory object to be imported - size_t size) { +/// + `NULL == hQueue` +/// + `NULL == hCommandBuffer` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_QUEUE +/// - ::UR_RESULT_ERROR_INVALID_EVENT +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urEnqueueCommandBufferExp( + /// [in] The queue to submit this command-buffer for execution. + ur_queue_handle_t hQueue, + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command-buffer execution. + /// If nullptr, the numEventsInWaitList must be 0, indicating no wait + /// events. + const ur_event_handle_t *phEventWaitList, + /// [out][optional][alloc] return an event object that identifies this + /// particular command-buffer execution instance. If phEventWaitList and + /// phEvent are not NULL, phEvent must not refer to an element of the + /// phEventWaitList array. + ur_event_handle_t *phEvent) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Release memory from USM +/// @brief Update a kernel launch command in a finalized command-buffer. /// /// @details -/// - Release memory from USM +/// This entry-point is synchronous and may block if the command-buffer is +/// executing when the entry-point is called. On error, the state of the +/// command-buffer commands being updated is undefined. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9233,43 +9254,88 @@ ur_result_t UR_APICALL urUSMImportExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hContext` +/// + `NULL == hCommandBuffer` +/// + `NULL == pUpdateKernelLaunch->hCommand` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pMem` -/// - ::UR_RESULT_ERROR_INVALID_CONTEXT -ur_result_t UR_APICALL urUSMReleaseExp( - /// [in] handle of the context object - ur_context_handle_t hContext, - /// [in] pointer to host memory object - void *pMem) { +/// + `NULL == pUpdateKernelLaunch` +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_INVALID_SIZE +/// + `numKernelUpdates == 0` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_ARGUMENTS +/// is not supported by the device, and for any of any element of +/// `pUpdateKernelLaunch` the `numNewMemObjArgs`, `numNewPointerArgs`, +/// or `numNewValueArgs` members are not zero. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is +/// not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is not nullptr. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_LOCAL_WORK_SIZE is +/// not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewLocalWorkSize` member is nullptr and +/// `pNewGlobalWorkSize` is not nullptr. +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_SIZE +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewGlobalWorkSize` member is not nullptr +/// + If +/// ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_GLOBAL_WORK_OFFSET +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `pNewGlobalWorkOffset` member is not +/// nullptr. +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_KERNEL_HANDLE +/// is not supported by the device, and for any element of +/// `pUpdateKernelLaunch` the `hNewKernel` member is not nullptr. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the `hCommandBuffer`. +/// + If `hCommandBuffer` has not been finalized. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member +/// is not a kernel execution command. +/// + If for any element of `pUpdateKernelLaunch` the `hCommand` member +/// was not created from `hCommandBuffer`. +/// - ::UR_RESULT_ERROR_INVALID_MEM_OBJECT +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_INDEX +/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE +/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION +/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION +/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` +/// member is less than 1 or greater than 3. +/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// + If for any element of `pUpdateKernelLaunch` the `hNewKernel` +/// member was not passed to the `hKernel` or `phKernelAlternatives` +/// parameters of ::urCommandBufferAppendKernelLaunchExp when the +/// command was created. +/// + If for any element of `pUpdateKernelLaunch` the `newWorkDim` +/// member is different from the current workDim in the `hCommand` +/// member, and `pNewGlobalWorkSize` or `pNewGlobalWorkOffset` are +/// nullptr. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferUpdateKernelLaunchExp( + /// [in] Handle of the command-buffer object. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] Length of pUpdateKernelLaunch. + uint32_t numKernelUpdates, + /// [in][range(0, numKernelUpdates)] List of structs defining how a + /// kernel commands are to be updated. + const ur_exp_command_buffer_update_kernel_launch_desc_t + *pUpdateKernelLaunch) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Enable access to peer device memory +/// @brief Get a new event that will be signaled the next time the command in +/// the +/// command-buffer executes. /// /// @details -/// - Enables the command device to access and write device memory -/// allocations located on the peer device, provided that a P2P link -/// between the two devices is available. -/// - When Peer Access is successfully enabled, P2P memory accesses are -/// guaranteed to be allowed on the peer device until -/// ::urUsmP2PDisablePeerAccessExp is called. -/// - Note that the function operands may, but aren't guaranteed to, commute -/// for a given adapter: the peer device is not guaranteed to have access -/// to device memory allocations located on the command device. -/// - It is not guaranteed that the commutation relations of the function -/// arguments are identical for peer access and peer copies: For example, -/// for a given adapter the peer device may be able to copy data from the -/// command device, but not access and write the same data on the command -/// device. -/// - Consult the appropriate adapter driver documentation for details of -/// adapter specific behavior and native error codes that may be returned. -/// -/// @remarks -/// _Analogues_ -/// - **cuCtxEnablePeerAccess** +/// It is the users responsibility to release the returned `phSignalEvent`. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9277,44 +9343,35 @@ ur_result_t UR_APICALL urUSMReleaseExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { +/// + `NULL == hCommand` +/// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER +/// + `NULL == phSignalEvent` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not +/// supported by the device associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the command-buffer `hCommand` belongs to. +/// + If the command-buffer `hCommand` belongs to has not been +/// finalized. +/// + If no `phEvent` parameter was set on creation of the command +/// associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// - ::UR_RESULT_ERROR_INVALID_VALUE +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferUpdateSignalEventExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [out][alloc] Event to be signaled. + ur_event_handle_t *phSignalEvent) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Disable access to peer device memory -/// -/// @details -/// - Disables the ability of the command device to access and write device -/// memory allocations located on the peer device, provided that a P2P -/// link between the two devices was enabled prior to the call. -/// - Note that the function operands may, but aren't guaranteed to, commute -/// for a given adapter. If, prior to the function call, the peer device -/// had access to device memory allocations on the command device, it is -/// not guaranteed to still have such access following the function -/// return. -/// - It is not guaranteed that the commutation relations of the function -/// arguments are identical for peer access and peer copies: For example -/// for a given adapter, if, prior to the call, the peer device had access -/// to device memory allocations on the command device, the peer device -/// may still, following the function call, be able to copy data from the -/// command device, but not access and write the same data on the command -/// device. -/// - Consult the appropriate adapter driver documentation for details of -/// adapter specific behavior and native error codes that may be returned. -/// -/// @remarks -/// _Analogues_ -/// - **cuCtxDisablePeerAccess** +/// @brief Set the list of wait events for a command to depend on to a list of +/// new events. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9322,29 +9379,39 @@ ur_result_t UR_APICALL urUsmP2PEnablePeerAccessExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice) { +/// + `NULL == hCommand` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If ::UR_DEVICE_COMMAND_BUFFER_UPDATE_CAPABILITY_FLAG_EVENTS is not +/// supported by the device associated with `hCommand`. +/// - ::UR_RESULT_ERROR_INVALID_OPERATION +/// + If ::ur_exp_command_buffer_desc_t::isUpdatable was not set to true +/// on creation of the command-buffer `hCommand` belongs to. +/// + If the command-buffer `hCommand` belongs to has not been +/// finalized. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_COMMAND_HANDLE_EXP +/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST +/// + `phEventWaitList == NULL && numEventsInWaitList > 0` +/// + `phEventWaitList != NULL && numEventsInWaitList == 0` +/// + If event objects in phEventWaitList are not valid events. +/// + If `numEventsInWaitList` does not match the number of wait events +/// set when the command associated with `hCommand` was created. +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +ur_result_t UR_APICALL urCommandBufferUpdateWaitEventsExp( + /// [in] Handle of the command-buffer command to update. + ur_exp_command_buffer_command_handle_t hCommand, + /// [in] Size of the event wait list. + uint32_t numEventsInWaitList, + /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of + /// events that must be complete before the command execution. If nullptr, + /// the numEventsInWaitList must be 0, indicating no wait events. + const ur_event_handle_t *phEventWaitList) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Disable access to peer device memory -/// -/// @details -/// - Queries the peer access capabilities from the command device to the -/// peer device according to the query `propName`. -/// -/// @remarks -/// _Analogues_ -/// - **cuDeviceGetP2PAttribute** +/// @brief Get command-buffer object information. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9352,10 +9419,9 @@ ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == commandDevice` -/// + `NULL == peerDevice` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `::UR_EXP_PEER_INFO_UR_PEER_ATOMICS_SUPPORT < propName` +/// + `::UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR < propName` /// - ::UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION /// + If `propName` is not supported by the adapter. /// - ::UR_RESULT_ERROR_INVALID_SIZE @@ -9365,37 +9431,31 @@ ur_result_t UR_APICALL urUsmP2PDisablePeerAccessExp( /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER /// + `propSize != 0 && pPropValue == NULL` /// + `pPropValue == NULL && pPropSizeRet == NULL` -/// - ::UR_RESULT_SUCCESS -/// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC -ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( - /// [in] handle of the command device object - ur_device_handle_t commandDevice, - /// [in] handle of the peer device object - ur_device_handle_t peerDevice, - /// [in] type of the info to retrieve - ur_exp_peer_info_t propName, - /// [in] the number of bytes pointed to by pPropValue. +/// - ::UR_RESULT_ERROR_INVALID_COMMAND_BUFFER_EXP +/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES +/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY +ur_result_t UR_APICALL urCommandBufferGetInfoExp( + /// [in] handle of the command-buffer object + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [in] the name of the command-buffer property to query + ur_exp_command_buffer_info_t propName, + /// [in] size in bytes of the command-buffer property value size_t propSize, - /// [out][optional][typename(propName, propSize)] array of bytes holding - /// the info. - /// If propSize is not equal to or greater than the real number of bytes - /// needed to return the info - /// then the ::UR_RESULT_ERROR_INVALID_SIZE error is returned and - /// pPropValue is not used. + /// [out][optional][typename(propName, propSize)] value of the + /// command-buffer property void *pPropValue, - /// [out][optional] pointer to the actual size in bytes of the queried - /// propName. + /// [out][optional] bytes returned in command-buffer property size_t *pPropSizeRet) { ur_result_t result = UR_RESULT_SUCCESS; return result; } /////////////////////////////////////////////////////////////////////////////// -/// @brief Enqueue a command to execute a kernel +/// @brief Return platform native command-buffer handle. /// -/// @remarks -/// _Analogues_ -/// - **clEnqueueNDRangeKernel** +/// @details +/// - Retrieved native handle can be used for direct interaction with the +/// native platform driver. /// /// @returns /// - ::UR_RESULT_SUCCESS @@ -9403,76 +9463,16 @@ ur_result_t UR_APICALL urUsmP2PPeerAccessGetInfoExp( /// - ::UR_RESULT_ERROR_DEVICE_LOST /// - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC /// - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE -/// + `NULL == hQueue` -/// + `NULL == hKernel` +/// + `NULL == hCommandBuffer` /// - ::UR_RESULT_ERROR_INVALID_NULL_POINTER -/// + `NULL == pGlobalWorkSize` -/// + `pArgs == NULL && numArgs > 0` -/// - ::UR_RESULT_ERROR_INVALID_ENUMERATION -/// + `NULL != pArgs && ::UR_EXP_KERNEL_ARG_TYPE_SAMPLER < pArgs->type` -/// + `NULL != launchPropList && ::UR_KERNEL_LAUNCH_FLAGS_MASK & -/// launchPropList->flags` -/// - ::UR_RESULT_ERROR_INVALID_QUEUE -/// - ::UR_RESULT_ERROR_INVALID_KERNEL -/// - ::UR_RESULT_ERROR_INVALID_EVENT -/// - ::UR_RESULT_ERROR_INVALID_EVENT_WAIT_LIST -/// + `phEventWaitList == NULL && numEventsInWaitList > 0` -/// + `phEventWaitList != NULL && numEventsInWaitList == 0` -/// + If event objects in phEventWaitList are not valid events. -/// - ::UR_RESULT_ERROR_IN_EVENT_LIST_EXEC_STATUS -/// + An event in `phEventWaitList` has ::UR_EVENT_STATUS_ERROR. -/// - ::UR_RESULT_ERROR_INVALID_WORK_DIMENSION -/// + `pGlobalWorkSize[0] == 0 || pGlobalWorkSize[1] == 0 || -/// pGlobalWorkSize[2] == 0` -/// - ::UR_RESULT_ERROR_INVALID_WORK_GROUP_SIZE -/// + `pLocalWorkSize && (pLocalWorkSize[0] == 0 || pLocalWorkSize[1] == -/// 0 || pLocalWorkSize[2] == 0)` -/// - ::UR_RESULT_ERROR_INVALID_VALUE -/// - ::UR_RESULT_ERROR_INVALID_KERNEL_ARGS - "The kernel argument values -/// have not been specified." -/// - ::UR_RESULT_ERROR_OUT_OF_HOST_MEMORY -/// - ::UR_RESULT_ERROR_OUT_OF_RESOURCES -/// - ::UR_RESULT_ERROR_INVALID_OPERATION -/// + If any property in `launchPropList` isn't supported by the device. -ur_result_t UR_APICALL urEnqueueKernelLaunchWithArgsExp( - /// [in] handle of the queue object - ur_queue_handle_t hQueue, - /// [in] handle of the kernel object - ur_kernel_handle_t hKernel, - /// [in] number of dimensions, from 1 to 3, to specify the global and - /// work-group work-items - uint32_t workDim, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the offset used to calculate the global ID of a work-item - const size_t *pGlobalWorkOffset, - /// [in] pointer to an array of workDim unsigned values that specify the - /// number of global work-items in workDim that will execute the kernel - /// function - const size_t *pGlobalWorkSize, - /// [in][optional] pointer to an array of workDim unsigned values that - /// specify the number of local work-items forming a work-group that will - /// execute the kernel function. - /// If nullptr, the runtime implementation will choose the work-group size. - const size_t *pLocalWorkSize, - /// [in] Number of entries in pArgs - uint32_t numArgs, - /// [in][optional][range(0, numArgs)] pointer to a list of kernel arg - /// properties. - const ur_exp_kernel_arg_properties_t *pArgs, - /// [in][optional] pointer to a single linked list of launch properties - const ur_kernel_launch_ext_properties_t *launchPropList, - /// [in] size of the event wait list - uint32_t numEventsInWaitList, - /// [in][optional][range(0, numEventsInWaitList)] pointer to a list of - /// events that must be complete before the kernel execution. - /// If nullptr, the numEventsInWaitList must be 0, indicating that no wait - /// event. - const ur_event_handle_t *phEventWaitList, - /// [out][optional][alloc] return an event object that identifies this - /// particular kernel execution instance. If phEventWaitList and phEvent - /// are not NULL, phEvent must not refer to an element of the - /// phEventWaitList array. - ur_event_handle_t *phEvent) { +/// + `NULL == phNativeCommandBuffer` +/// - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE +/// + If the adapter has no underlying equivalent handle. +ur_result_t UR_APICALL urCommandBufferGetNativeHandleExp( + /// [in] Handle of the command-buffer. + ur_exp_command_buffer_handle_t hCommandBuffer, + /// [out] A pointer to the native handle of the command-buffer. + ur_native_handle_t *phNativeCommandBuffer) { ur_result_t result = UR_RESULT_SUCCESS; return result; }