accel/ze: initial pass at intel zero-level

api support for Intel PV and other accelerators

related to issue #11763

Signed-off-by: Howard Pritchard <[email protected]>

config/opal_check_ze.m4

@@ -0,0 +1,78 @@
+dnl
+dnl Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+dnl Copyright (c) 2023     Triad National Security, LLC. All rights reserved.
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+
+# OPAL_CHECK_ZE(prefix, [action-if-found], [action-if-not-found])
+# --------------------------------------------------------
+# check if Intel ZE support can be found.  sets prefix_{CPPFLAGS,
+# LDFLAGS, LIBS} as needed and runs action-if-found if there is
+# support, otherwise executes action-if-not-found
+
+
+#
+# Check for ZE  support
+#
+AC_DEFUN([OPAL_CHECK_ZE],[
+
+     OPAL_VAR_SCOPE_PUSH([opal_check_ze_happy ze_save_CPPFLAGS ze_save_LDFLAGS ze_save_LIBS ze_CPPFLAGS ze_LDFLAGS ze_LIBS])
+
+     ze_save_CPPFLAGS="$CPPFLAGS"
+     ze_save_LDFLAGS="$LDFLAGS"
+     ze_save_LIBS="$LIBS"
+
+     # Get some configuration information
+     AC_ARG_WITH([ze],
+        [AS_HELP_STRING([--with-ze(=DIR)],
+        [Build Intel ZE support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
+
+
+     AS_IF([ test -n "$with_ze" && test "$with_ze" = "yes" ],
+           [ with_ze="/opt/ze"] )
+
+     ze_CPPFLAGS="-D__HIP_PLATFORM_AMD__"
+     ze_LDFLAGS="-L${with_ze}/lib/hip"
+
+     AS_IF([ test -n "$with_ze" && test "$with_ze" != "no" ],
+           [ OPAL_APPEND([CPPFLAGS], [$ze_CPPFLAGS])
+	     OPAL_APPEND([LDFLAGS], [$ze_LDFLAGS]) ])
+
+     m4_define([ze_pkgconfig_module], [level-zero])
+     OAC_CHECK_PACKAGE([ze],
+                       [$1],
+                       [level_zero/ze_api.h],
+                       [ze_loader],
+		       [zeInit],
+                       [opal_check_ze_happy="yes"],
+                       [opal_check_ze_happy="no"])
+
+     LDFLAGS="$ze_save_LDFLAGS"
+     LIBS="$ze_save_LIBS"
+     OPAL_APPEND([CPPFLAGS], [${$1_CPPFLAGS}] )
+     OPAL_APPEND([LDFLAGS], [${$1_LDFLAGS}] )
+     OPAL_APPEND([LIBS], [${$1_LIBS}] )
+
+     AS_IF([ test "$opal_check_ze_happy" = "no" ],
+           [ CPPFLAGS="$ze_save_CPPFLAGS"])
+
+     AS_IF([ test "$opal_check_ze_happy" = "yes" ],
+           [ AC_DEFINE_UNQUOTED([OPAL_ZE_SUPPORT], [1], [Enable Intel ZE support])
+             ZE_SUPPORT=1 ],
+           [ AC_DEFINE_UNQUOTED([OPAL_ZE_SUPPORT], [0], [Disable Intel ZE support])
+             ZE_SUPPORT=0 ])
+
+     AS_IF([ test "$opal_check_ze_happy" = "yes" ],
+            [$2],
+            [AS_IF([test -n "$with_ze" && test "$with_ze" != "no"],
+                   [AC_MSG_ERROR([Intel ZE support requested but not found.  Aborting])])
+            $3])
+
+     AM_CONDITIONAL([OPAL_ze_support], [test "$opal_check_ze_happy" = "yes"])
+     OPAL_VAR_SCOPE_POP
+])

opal/mca/accelerator/ze/Makefile.am

@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+AM_CPPFLAGS = $(common_ze_CPPFLAGS)
+
+sources = \
+        accelerator_ze.h \
+        accelerator_ze_component.c \
+        accelerator_ze_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_accelerator_ze_DSO
+component_noinst =
+component_install = mca_accelerator_ze.la
+else
+component_noinst = libmca_accelerator_ze.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_accelerator_ze_la_SOURCES = $(sources)
+mca_accelerator_ze_la_LDFLAGS = -module -avoid-version $(opal_ze_LDFLAGS)
+mca_accelerator_ze_la_LIBADD = $(top_builddir)/opal/lib@[email protected] \
+	$(opal_ze_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_accelerator_ze_la_SOURCES =$(sources)
+libmca_accelerator_ze_la_LDFLAGS = -module -avoid-version $(opal_ze_LDFLAGS)
+libmca_accelerator_ze_la_LIBADD = $(opal_ze_LIBS)

opal/mca/accelerator/ze/accelerator_ze.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2022-2023  Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2023       Triad National Security, LLC. All rights reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OPAL_ACCELERATOR_ZE_H
+#define OPAL_ACCELERATOR_ZE_H
+
+#include "opal_config.h"
+
+#include "level_zero/ze_api.h"
+
+#include "opal/mca/accelerator/accelerator.h"
+#include "opal/mca/threads/mutex.h"
+
+typedef struct {
+    opal_accelerator_base_component_t super;
+} opal_accelerator_ze_component_t;
+
+OPAL_DECLSPEC extern opal_accelerator_ze_component_t mca_accelerator_ze_component;
+OPAL_DECLSPEC extern opal_accelerator_base_module_t opal_accelerator_ze_module;
+
+struct opal_accelerator_ze_stream_t {
+    opal_accelerator_stream_t base;
+    ze_command_queue_handle_t hCommandQueue;
+    ze_command_list_handle_t hCommandList;
+#if 0
+    ze_event_pool_handle_t hEventPool;
+#endif
+    int dev_id;
+};
+typedef struct opal_accelerator_ze_stream_t opal_accelerator_ze_stream_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ze_stream_t);
+
+struct opal_accelerator_ze_event_t {
+    opal_accelerator_event_t base;
+};
+typedef struct opal_accelerator_ze_event_t opal_accelerator_ze_event_t;
+OBJ_CLASS_DECLARATION(opal_accelerator_ze_event_t);
+
+OPAL_DECLSPEC extern uint32_t opal_accelerator_ze_device_count;
+OPAL_DECLSPEC extern ze_device_handle_t *opal_accelerator_ze_devices_handle;
+OPAL_DECLSPEC extern ze_driver_handle_t opal_accelerator_ze_driver_handle;
+OPAL_DECLSPEC extern ze_context_handle_t opal_accelerator_ze_context;
+OPAL_DECLSPEC extern ze_event_pool_handle_t opal_accelerator_ze_event_pool;
+OPAL_DECLSPEC extern opal_accelerator_ze_stream_t **opal_accelerator_ze_MemcpyStream;
+
+OPAL_DECLSPEC extern int opal_accelerator_ze_memcpy_async;
+OPAL_DECLSPEC extern int opal_accelerator_ze_verbose;
+
+OPAL_DECLSPEC extern int opal_accelerator_ze_lazy_init(void);
+
+#endif