Merge pull request #11875 from qkoziol/ompi_issue_10657

Bump required versions of OpenPMIX and PRTTE
open-mpi · Oct 16, 2023 · fee1d70 · fee1d70
2 parents 49a64f7 + fea96e1
commit fee1d70
Show file tree

Hide file tree

Showing 4 changed files with 178 additions and 79 deletions.
diff --git a/VERSION b/VERSION
@@ -25,8 +25,8 @@ mpi_standard_subversion=1
 
 # OMPI required dependency versions.
 # List in x.y.z format.
-pmix_min_version=4.1.2
-prte_min_version=2.0.2
+pmix_min_version=4.2.0
+prte_min_version=3.0.0
 hwloc_min_version=1.11.0
 event_min_version=2.0.21
 automake_min_version=1.13.4

diff --git a/docs/Makefile.am b/docs/Makefile.am
@@ -1031,8 +1031,8 @@ $(ALL_MAN_BUILT):
 	    filename=`basename $$file`; \
 	    cp -pf $(OMPI_PRRTE_RST_CONTENT_DIR)/$$filename "$(builddir)/prrte-rst-content"; \
 	done
-	$(OMPI_V_SPHINX_HTML) OMPI_VERSION_FILE=$(top_srcdir)/VERSION $(SPHINX_BUILD) -M html "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
-	$(OMPI_V_SPHINX_MAN) OMPI_VERSION_FILE=$(top_srcdir)/VERSION $(SPHINX_BUILD) -M man "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
+	$(OMPI_V_SPHINX_HTML) OMPI_TOP_SRCDIR=$(top_srcdir) $(SPHINX_BUILD) -M html "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
+	$(OMPI_V_SPHINX_MAN) OMPI_TOP_SRCDIR=$(top_srcdir) $(SPHINX_BUILD) -M man "$(builddir)" "$(OUTDIR)" $(SPHINX_OPTS)
 
 # A useful rule to invoke manually to ensure that all of the external
 # HTML links we have are valid.  Running this rule requires

diff --git a/docs/conf.py b/docs/conf.py
@@ -14,46 +14,85 @@
 
 # -- Project information -----------------------------------------------------
 
+import os
+import re
 import datetime
+
 year = datetime.datetime.now().year
 
 project = 'Open MPI'
 copyright = f'2003-{year}, The Open MPI Community'
 author = 'The Open MPI Community'
 
-# The full version, including alpha/beta/rc tags
-# Read the Open MPI version from the VERSION file in the source tree
-# The docs/Makefile.am will set the env var OMPI_VERSION_FILE, because
-# we might be doing a VPATH build.
-filename = None
-if 'OMPI_VERSION_FILE' in os.environ:
-    filename = os.environ['OMPI_VERSION_FILE']
-elif os.path.exists("../VERSION"):
-    filename = '../VERSION'
-
-if filename is None:
-    print("ERROR: Could not find Open MPI source tree VERSION file")
-    exit(1)
-
-with open(filename) as fp:
-    ompi_lines = fp.readlines()
-
-ompi_data = dict()
-for ompi_line in ompi_lines:
-    if '#' in ompi_line:
-        parts = ompi_line.split("#")
-        ompi_line = parts[0]
-    ompi_line = ompi_line.strip()
-
-    if '=' not in ompi_line:
-        continue
+# ---------------------------
 
-    ompi_key, ompi_val = ompi_line.split("=")
-    ompi_data[ompi_key.strip()] = ompi_val.strip()
+# The docs/Makefile.am will set the env var OMPI_TOP_SRCDIR, because
+# we might be doing a VPATH build.
+ompi_top_srcdir = '..'
+if 'OMPI_TOP_SRCDIR' in os.environ:
+    ompi_top_srcdir = os.environ['OMPI_TOP_SRCDIR']
+
+# Read an Open MPI-style VERSION file
+def read_version_file(path):
+    if not os.path.exists(path):
+        print(f"ERROR: Unable to find file {path}")
+        exit(1)
+
+    with open(path) as fp:
+        version_lines = fp.readlines()
+
+    data = dict()
+    for line in version_lines:
+        if '#' in line:
+            parts = line.split("#")
+            line = parts[0]
+        line = line.strip()
+
+        if '=' not in line:
+            continue
+
+        key, val = line.split("=")
+        data[key.strip()] = val.strip()
+
+    return data
+
+# Look for a version string via a regular expresion of a filename in a
+# given directory
+def get_tarball_version(path, expr):
+    if not os.path.exists(path):
+        print(f"ERROR: Unable to find path {path}")
+        exit(1)
+
+    for file in os.listdir(path):
+        m = re.match(expr, file)
+        if not m:
+            continue
+        return m.group(1)
+
+    return ""
+
+# Read all the various versions from the source tree
+
+ompi_data = read_version_file(f"{ompi_top_srcdir}/VERSION")
+pmix_data = read_version_file(f"{ompi_top_srcdir}/3rd-party/openpmix/VERSION")
+prte_data = read_version_file(f"{ompi_top_srcdir}/3rd-party/prrte/VERSION")
+
+hwloc_embedded_version = get_tarball_version(f"{ompi_top_srcdir}/3rd-party/",
+                                             r"hwloc-(.*).tar")
+event_embedded_version = get_tarball_version(f"{ompi_top_srcdir}/3rd-party/",
+                                             r"libevent-(.*)-stable.tar")
+
+# ---------------------------
+
+# Assemble several different combinations of version strings
 
 ompi_series = f"v{ompi_data['major']}.{ompi_data['minor']}.x"
 ompi_ver = f"v{ompi_data['major']}.{ompi_data['minor']}.{ompi_data['release']}{ompi_data['greek']}"
 
+pmix_embedded_version = f"v{pmix_data['major']}.{pmix_data['minor']}.{pmix_data['release']}{pmix_data['greek']}"
+prte_embedded_version = f"v{prte_data['major']}.{prte_data['minor']}.{prte_data['release']}{prte_data['greek']}"
+prte_embedded_series = f"v{prte_data['major']}.{prte_data['minor']}"
+
 pmix_min_version = f"{ompi_data['pmix_min_version']}"
 prte_min_version = f"{ompi_data['prte_min_version']}"
 hwloc_min_version = f"{ompi_data['hwloc_min_version']}"
@@ -86,7 +125,6 @@
 # If we're building in an RTD environment for a tag or external (i.e.,
 # PR), use the RTD version -- not what we just read from the VERSIONS
 # file.
-import os
 key = 'READTHEDOCS'
 if key in os.environ and os.environ[key] == 'True':
     print("OMPI: found ReadTheDocs build environment")
@@ -172,9 +210,6 @@
 
 # -- Options for MAN output -------------------------------------------------
 
-import os
-import re
-
 # Dynamically find all the man pages and build the appropriate list of
 # tuples so that we don't have to manually maintain it.
 
@@ -222,9 +257,14 @@ def _doit(topdir):
 .. |ompi_ver| replace:: {ompi_ver}
 .. |ompi_series| replace:: {ompi_series}
 .. |pmix_min_version| replace:: {pmix_min_version}
+.. |pmix_embedded_version| replace:: {pmix_embedded_version}
 .. |prte_min_version| replace:: {prte_min_version}
+.. |prte_embedded_version| replace:: {prte_embedded_version}
+.. |prte_embedded_series| replace:: {prte_embedded_series}
 .. |hwloc_min_version| replace:: {hwloc_min_version}
+.. |hwloc_embedded_version| replace:: {hwloc_embedded_version}
 .. |event_min_version| replace:: {event_min_version}
+.. |event_embedded_version| replace:: {event_embedded_version}
 .. |automake_min_version| replace:: {automake_min_version}
 .. |autoconf_min_version| replace:: {autoconf_min_version}
 .. |libtool_min_version| replace:: {libtool_min_version}
@@ -234,6 +274,10 @@ def _doit(topdir):
 .. |mpi_standard_minor_version| replace:: {mpi_standard_minor_version}
 .. |deprecated_favor| replace:: this routine is deprecated in favor of
 
+.. |br| raw:: html
+
+   <br />
+
 """
 
 # The sphinx_rtd_theme does not properly handle wrapping long lines in

diff --git a/docs/installing-open-mpi/required-support-libraries.rst b/docs/installing-open-mpi/required-support-libraries.rst
@@ -3,50 +3,28 @@
 Required support libraries
 ==========================
 
-Open MPI requires the following support libraries with the minimum listed versions:
-
-.. list-table::
-   :header-rows: 1
-
-   * - Library
-     - Minimum version
-     - Notes
-   * - `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
-     - |hwloc_min_version|
-     - This library is required; Open MPI will not build without it.
-   * - `Libevent <https://libevent.org/>`_
-     - |event_min_version|
-     - This library is required; Open MPI will not build without it.
-   * - `PMIx <https://pmix.org/>`_
-     - |pmix_min_version|
-     - This library is required; Open MPI will not build without it.
-   * - `PRRTE <https://github.com/openpmix/prrte>`_
-     - |prte_min_version|
-     - This library is optional in some environments. PRRTE provides
-       Open MPI's full-featured ``mpirun`` / ``mpiexec`` MPI
-       application launchers (the two are identical; they are symbolic
-       links to the same executable).
-
-       * If your environment uses another MPI application launcher
-         (e.g., Slurm users can use the ``srun`` launcher to "direct
-         launch" Open MPI applications), then the use of PRRTE is
-         optional.
-       * If your environment has no other MPI application launcher, then
-         you need to install PRRTE and build Open MPI with PRRTE
-         support.
-       * Open MPI can use the copy of PRRTE embedded in its source
-         code tree, or compile/link against an external PRRTE
-         installation.  :ref:`See this section for details about how
-         to specify each method
-         <label-building-ompi-cli-options-required-support-libraries>`.
-
-Since these support libraries are fundamental to Open MPI's operation
-and not universally available in all environments, they are directly
+
+While Open MPI can be built with support for a wide variety of
+systems, a small set of support libraries are *required* in order to
+build Open MPI in *any* environment.  Several of these packages are
+both fundamental to Open MPI's operation and not universally available
+in all environments.  As such, these "fundamental" packages are both
+embedded in Open MPI's distribution tarballs and also directly
 incorporated into Open MPI's configure, build, and installation
-process.  More on this below.
+process.
+
+:ref:`See below
+<required-support-libraries-configure-discovery-label>` for a
+description of how Open MPI chooses whether to use the embedded
+versions of these packages or versions already installed on your
+system.
+
+* `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
 
-  .. note:: The versions listed in this table are the *minimum* versions needed.  In general, the Open MPI community recommends using more recent versions of both the :ref:`required support libraries <label-install-required-support-libraries>` and any other optional support libraries.  This is because more recent versions typically tend to include bug fixes, sometimes affecting Open MPI functionality.  As a specific example, there is a known issue with `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_ releases older than v2.8.0 on systems with Intel Ponte Vecchio accelerators.  If you run Open MPI on such systems, you need to use Hwloc v2.8.0 or newer, or you will experience undefined behavior.
-   This effect is not unique to the Hardware Locality library; this is why the Open MPI community recommends using as recent as possible versions of all support libraries.
+  * This library is required; Open MPI will not build without it.
+  * **Minimum version required:** |hwloc_min_version|
+  * **Version embedded in Open MPI distribution:**
+    |hwloc_embedded_version|
 
   .. danger:: As of |ompi_ver|, Open MPI does not yet support the
               Hwloc v3.x series (which may not even be available at
@@ -77,6 +55,81 @@ process.  More on this below.
               uses Hwloc, it uses the *same* Hwloc with which Open MPI
               was compiled.
 
+* `Libevent <https://libevent.org/>`_
+
+  * This library is required; Open MPI will not build without it.
+  * **Minimum version required:** |event_min_version|
+  * **Version embedded in Open MPI distribution:**
+    |event_embedded_version|
+
+* `PMIx <https://pmix.org/>`_
+
+  * This library is required; Open MPI will not build without it.
+  * **Minimum version required when building without PRRTE:**
+    |pmix_min_version|
+  * **Minimum version required when building with PRRTE:** `See the
+    PRRTE project documentation <https://docs.prrte.org/>`_.
+  * **Version embedded in Open MPI distribution:**
+    |pmix_embedded_version|
+
+* `PRRTE <https://github.com/openpmix/prrte>`_
+
+  * This library is optional in some environments. See below.
+  * **Minimum version required:** |prte_min_version|
+
+    .. note:: While building Open MPI with PRRTE |prte_min_version|
+              *works*, you will not get a fully-populated
+              ``mpirun(1)`` man page.  The Open MPI community
+              recommends that you use PRRTE version 3.0.1 or higher.
+
+  * **Version embedded in Open MPI distribution:**
+    |prte_embedded_version|
+
+  PRRTE provides Open MPI's full-featured ``mpirun`` / ``mpiexec`` MPI
+  application launchers (the two commands are identical; they are
+  symbolic links to the same executable).
+
+  .. warning:: If you are building the PRRTE that is embedded in the
+               Open MPI |ompi_ver| distribution:
+
+               * If you are also building the PMIx that is embedded in
+                 the Open MPI |ompi_ver| distribution, that
+                 combination of packages is supported.
+
+               * If you are building against an external PMIx
+                 installation (i.e., a version of PMIx that is not
+                 embedded in the Open MPI |ompi_ver| distribution),
+                 you should check `the PRRTE project documentation
+                 <https://docs.prrte.org/>`_ to see what minimum
+                 version of PMIx is required.
+
+  * If your environment uses another MPI application launcher (e.g.,
+    Slurm users can use the ``srun`` launcher to "direct launch" Open
+    MPI applications), then the use of PRRTE is optional.
+  * If your environment has no other MPI application launcher, then
+    you need to install PRRTE and build Open MPI with PRRTE support.
+  * Open MPI can use the copy of PRRTE embedded in its source code
+    tree, or compile/link against an external PRRTE installation.
+    :ref:`See this section for details about how to specify each
+    method
+    <label-building-ompi-cli-options-required-support-libraries>`.
+
+.. note:: In general, the Open MPI community recommends using the most
+          recent versions of both the :ref:`required support libraries
+          <label-install-required-support-libraries>` and any other
+          optional support libraries.  This is because more recent
+          versions typically tend to include bug fixes, sometimes
+          affecting Open MPI functionality.  As a specific example,
+          there is a known issue with `Hardware Locality
+          <https://www.open-mpi.org/projects/hwloc/>`_ releases older
+          than v2.8.0 on systems with Intel Ponte Vecchio
+          accelerators.  If you run Open MPI on such systems, you need
+          to use Hwloc v2.8.0 or newer, or you will experience
+          undefined behavior.  This effect is not unique to the
+          Hardware Locality library; this is why the Open MPI
+          community recommends using as recent as possible versions of
+          *all* support libraries.
+
 Library dependencies
 --------------------
 
@@ -145,6 +198,8 @@ example |mdash| only Libevent and Hwloc, that somewhat simplifies the
 final Open MPI configuration, and therefore avoids some potentially
 erroneous configurations.
 
+.. _required-support-libraries-configure-discovery-label:
+
 How ``configure`` finds the required libraries
 ----------------------------------------------
 
@@ -264,7 +319,7 @@ on Mac OS because:
    tarballs).
 #. In MacOS, it is common for `Homebrew <https://brew.sh/>`_ or
    `MacPorts <https://www.macports.org/>`_ to install:
-   
+
    * `Hardware Locality <https://www.open-mpi.org/projects/hwloc/>`_
    * `Libevent <https://libevent.org/>`_