Skip to content

Commit

Permalink
SQUASHME: various fixes
Browse files Browse the repository at this point in the history 
to get the one test we have from intel to work

Signed-off-by: Howard Pritchard <[email protected]>
  • Loading branch information
@hppritcha
hppritcha committed Aug 28, 2023
1 parent 8f61618 commit f277d21
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 42 deletions.
2 changes: 1 addition & 1 deletion opal/mca/accelerator/ze/accelerator_ze.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ OPAL_DECLSPEC extern ze_device_handle_t *opal_accelerator_ze_devices_handle;
OPAL_DECLSPEC extern ze_driver_handle_t opal_accelerator_ze_driver_handle;
OPAL_DECLSPEC extern ze_context_handle_t opal_accelerator_ze_context;
OPAL_DECLSPEC extern ze_event_pool_handle_t opal_accelerator_ze_event_pool;
OPAL_DECLSPEC extern opal_accelerator_ze_stream_t **opal_accelerator_ze_MemcpyStream;
OPAL_DECLSPEC extern opal_accelerator_stream_t **opal_accelerator_ze_MemcpyStream;

OPAL_DECLSPEC extern int opal_accelerator_ze_memcpy_async;
OPAL_DECLSPEC extern int opal_accelerator_ze_verbose;
Expand Down
24 changes: 15 additions & 9 deletions opal/mca/accelerator/ze/accelerator_ze_component.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ ze_device_handle_t *opal_accelerator_ze_devices_handle = NULL;
ze_driver_handle_t opal_accelerator_ze_driver_handle = NULL;
ze_context_handle_t opal_accelerator_ze_context = NULL;
ze_event_pool_handle_t opal_accelerator_ze_event_pool = NULL;
opal_accelerator_ze_stream_t **opal_accelerator_ze_MemcpyStream = NULL;
opal_accelerator_stream_t **opal_accelerator_ze_MemcpyStream = NULL;

/* Initialization lock for lazy ze initialization */
static opal_mutex_t accelerator_ze_init_lock;
Expand Down Expand Up @@ -144,15 +144,18 @@ int opal_accelerator_ze_lazy_init(void)

/* If already initialized, just exit */
if (true == accelerator_ze_init_complete) {
goto out;
goto fn_fail;
}

opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
"ZE: starting lazy init");

zret = zeDriverGet(&driver_count, NULL);
if (ZE_RESULT_SUCCESS != zret) {
opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
"ZE: zeDriverGet returned %d\n", zret);
err = OPAL_ERR_NOT_INITIALIZED;
goto out;
goto fn_fail;
}

/*
Expand All @@ -166,13 +169,13 @@ int opal_accelerator_ze_lazy_init(void)
all_drivers = (ze_driver_handle_t *)malloc(driver_count * sizeof(ze_driver_handle_t));
if (all_drivers == NULL) {
err = OPAL_ERR_OUT_OF_RESOURCE;
goto out;
goto fn_fail;
}

zret = zeDriverGet(&driver_count, all_drivers);
if (ZE_RESULT_SUCCESS != zret) {
err = OPAL_ERR_NOT_FOUND;
goto out;
goto fn_fail;
}

/*
Expand All @@ -195,7 +198,7 @@ int opal_accelerator_ze_lazy_init(void)
malloc(opal_accelerator_ze_device_count * sizeof(ze_device_handle_t));
if (NULL == opal_accelerator_ze_devices_handle) {
err = OPAL_ERR_OUT_OF_RESOURCE;
goto out;
goto fn_fail;
}
zret = zeDeviceGet(all_drivers[i], &opal_accelerator_ze_device_count, opal_accelerator_ze_devices_handle);
if (ZE_RESULT_SUCCESS != zret) {
Expand Down Expand Up @@ -248,8 +251,8 @@ int opal_accelerator_ze_lazy_init(void)
* allocate synchronous memcpy stream handles, but delay creating streams till needed
*/

opal_accelerator_ze_MemcpyStream = (opal_accelerator_ze_stream_t **)malloc(opal_accelerator_ze_device_count *
sizeof(opal_accelerator_ze_stream_t *));
opal_accelerator_ze_MemcpyStream = (opal_accelerator_stream_t **)calloc((size_t)opal_accelerator_ze_device_count,
sizeof(opal_accelerator_stream_t *));
if (NULL == opal_accelerator_ze_MemcpyStream) {
err = OPAL_ERR_OUT_OF_RESOURCE;
goto fn_fail;
Expand Down Expand Up @@ -282,9 +285,12 @@ int opal_accelerator_ze_lazy_init(void)
}

opal_atomic_wmb();
opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
"ZE: found %d devices", opal_accelerator_ze_device_count);
accelerator_ze_init_complete = true;

out:
return OPAL_SUCCESS;

fn_fail:
if (NULL != opal_accelerator_ze_MemcpyStream) {
free(opal_accelerator_ze_MemcpyStream);
Expand Down
61 changes: 29 additions & 32 deletions opal/mca/accelerator/ze/accelerator_ze_module.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
* $HEADER$
*/

#include <string.h>

#include "opal_config.h"

#include "accelerator_ze.h"
Expand Down Expand Up @@ -90,7 +92,7 @@ static int accelerator_ze_dev_handle_to_dev_id(ze_device_handle_t hDevice)
static int mca_accelerator_ze_check_addr (const void *addr, int *dev_id, uint64_t *flags)
{
ze_result_t zret;
int ret;
int ret = 0;
ze_memory_allocation_properties_t attr;
ze_device_handle_t hDevice;

Expand All @@ -106,6 +108,8 @@ static int mca_accelerator_ze_check_addr (const void *addr, int *dev_id, uint64_
return ret;
}

memset(&attr, 0, sizeof(ze_memory_allocation_properties_t));

zret = zeMemGetAllocProperties(opal_accelerator_ze_context,
addr,
&attr,
Expand All @@ -120,12 +124,7 @@ static int mca_accelerator_ze_check_addr (const void *addr, int *dev_id, uint64_
case ZE_MEMORY_TYPE_HOST:
break;
case ZE_MEMORY_TYPE_DEVICE:
/*
* Assume that although shared is accessible from host, access may be slow so
* like device memory for now.
*/
case ZE_MEMORY_TYPE_SHARED:
*flags |= MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY;
ret = 1;
*dev_id = accelerator_ze_dev_handle_to_dev_id(hDevice);
break;
Expand All @@ -140,6 +139,7 @@ static int mca_accelerator_ze_check_addr (const void *addr, int *dev_id, uint64_

static int mca_accelerator_ze_create_stream(int dev_id, opal_accelerator_stream_t **stream)
{
int ret;
ze_result_t zret;
ze_device_handle_t hDevice;
opal_accelerator_ze_stream_t *ze_stream;
Expand All @@ -158,6 +158,16 @@ static int mca_accelerator_ze_create_stream(int dev_id, opal_accelerator_stream_
return OPAL_ERR_BAD_PARAM;
}

ret = opal_accelerator_ze_lazy_init();
if (OPAL_SUCCESS != ret) {
return ret;
}

*stream = (opal_accelerator_stream_t*)OBJ_NEW(opal_accelerator_ze_stream_t);
if (NULL == *stream) {
return OPAL_ERR_OUT_OF_RESOURCE;
}

ze_stream = (opal_accelerator_ze_stream_t *)malloc(sizeof(opal_accelerator_ze_stream_t));
if (NULL == ze_stream) {
OBJ_RELEASE(*stream);
Expand All @@ -181,26 +191,6 @@ static int mca_accelerator_ze_create_stream(int dev_id, opal_accelerator_stream_
return OPAL_ERROR;
}

#if 0
/*
* set up an event pool
*/

ze_event_pool_desc_t eventPoolDesc = {
.stype = ZE_STRUCTURE_TYPE_EVENT_POOL_DESC,
.pNext = NULL,
.flags = 0,
.count = 1000, /* TODO: fix this! */
};

zret = zeEventPoolCreate(opal_accelerator_ze_context,
&eventPoolDesc,
1,
opal_accelerator_ze_devices_handle,
&ze_stream->hEventPool);
assert(zret == ZE_RESULT_SUCCESS);
#endif

/*
* create a command list
*/
Expand Down Expand Up @@ -422,23 +412,30 @@ static int mca_accelerator_ze_memcpy(int dest_dev_id, int src_dev_id, void *dest
const void *src, size_t size,
opal_accelerator_transfer_type_t type)
{
int ret, zret;
int ret, dev_id;
ze_result_t zret;

opal_accelerator_ze_stream_t *ze_stream = NULL;

if (NULL == src || NULL == dest || size <=0) {
return OPAL_ERR_BAD_PARAM;
}

if (NULL == opal_accelerator_ze_MemcpyStream[src_dev_id]) {
ret = mca_accelerator_ze_create_stream(src_dev_id,
(opal_accelerator_stream_t **)&opal_accelerator_ze_MemcpyStream[src_dev_id]);
if (MCA_ACCELERATOR_NO_DEVICE_ID == src_dev_id) {
dev_id = 0;
} else {
dev_id = src_dev_id;
}

if (NULL == opal_accelerator_ze_MemcpyStream[dev_id]) {
ret = mca_accelerator_ze_create_stream(dev_id,
(opal_accelerator_stream_t **)&opal_accelerator_ze_MemcpyStream[dev_id]);
if (OPAL_SUCCESS != ret) {
return ret;
}
}

ze_stream = opal_accelerator_ze_MemcpyStream[src_dev_id];

ze_stream = (opal_accelerator_ze_stream_t *)opal_accelerator_ze_MemcpyStream[dev_id]->stream;
zret = zeCommandListAppendMemoryCopy(ze_stream->hCommandList,
dest,
src,
Expand Down

0 comments on commit f277d21

Please sign in to comment.