Updating to tensorflow 2.13 and Macos arm issues (#2138)

* Updating to tensorflow 2.12

* Using legacy Adam optimizer.

* Switching to install grpcio with pip (to match tensorflow)

* Trying tensorflow 2.12 on windows.

* Switching to tensorflow 2.11, the newest that works in windows.

* Making smt optional since it is not available for macos amd64.

* Make importing BayesianOptimizer optional.

* Making BayesianOptimizer optional in Factory.

* Skip registering BayesianOptimizer if smt not available.

* Skip Bayesian tests if smt not available.

* Adding grpcio.

* Trying newer ray.

* Trying tensorflow 2.13 again on windows.

* Dependency cleanup.

pedantic no-longer seems to need to be specified with ray 2.6
updating to netcdf 1.6
consolidating redundant os lines.

* Fix pca tests.

* Adding zero threshold to SparseGrid tests.

* Rel_err for redudantInputs for arm64

* Updating rel_err for simionescuConstrainedInvLin for arm64

* Updating rel_err for arm64

* Adding rel_err for polyCorrelation for arm64

* Adding rel_err to CobraTF test for arm64.

* Adding rel_err for NDGridProbabilityWeightValue for arm64

* Switching to only use smt on x86_64

* Adding in ability to substitute machine type names

* Reverting rel_err change for simionescuConstrainedInvLin

dependencies.xml

@@ -13,6 +13,7 @@ Note all install methods after "main" take
  add with no pinned version: <library/>
  remove a library:           <library>remove</library>
  action for limited OS:      <library os='windows'>...
+ action for certain machine  <library machine='arm64'>...
  add library from forge:     <library source='forge'>...
  optional:                   <library optional='True'>...
  skip run/install check:     <library skip_check='True'>...
@@ -42,12 +43,12 @@ Note all install methods after "main" take
     <pandas/>
     <!-- Note most versions of xarray work, but some (such as 0.20) don't -->
     <xarray>2023</xarray>
-    <netcdf4 source="pip">1.5</netcdf4>
+    <netcdf4 source="pip">1.6</netcdf4>
     <matplotlib>3.5</matplotlib>
     <statsmodels>0.13</statsmodels>
     <cloudpickle>2.2</cloudpickle>
-    <tensorflow source="pip" os='mac,linux'>2.10</tensorflow>
-    <tensorflow source="pip" os='windows'>2.10</tensorflow>
+    <tensorflow source="pip">2.13</tensorflow>
+    <grpcio source="pip" />
     <!-- conda is really slow on windows if the version is not specified.-->
     <python skip_check='True' os='windows'>3.8</python>
     <python skip_check='True' os='mac,linux'>3</python>
@@ -64,13 +65,11 @@ Note all install methods after "main" take
     <numexpr os='linux'/>
     <cmake skip_check='True' optional='True'/>
     <dask source="pip" pip_extra="[complete]"/>
-    <ray source="pip" pip_extra="[default]" os='mac,linux'>2.2</ray>
-    <ray source="pip" pip_extra="[default]" os='windows'>1.13</ray>
-    <pydantic source="pip" os='mac,linux'>1</pydantic> <!-- ray 2.2 can't use pydantic 2 -->
+    <ray source="pip" pip_extra="[default]">2.6</ray>
     <!-- redis is needed by ray, but on windows, this seems to need to be explicitly stated -->
     <redis source="pip" os='windows'/>
     <imageio source="pip">2.22</imageio>
-    <smt/>
+    <smt machine='x86_64'/> <!-- not available on macos arm64 -->
     <line_profiler optional='True'/>
     <!-- <ete3 optional='True'/> -->
     <pywavelets optional='True'>1.1</pywavelets>

ravenframework/Optimizers/Factory.py

@@ -24,10 +24,14 @@
 from .GradientDescent import GradientDescent
 from .SimulatedAnnealing import SimulatedAnnealing
 from .GeneticAlgorithm import GeneticAlgorithm
-from .BayesianOptimizer import BayesianOptimizer
 
 factory = EntityFactory('Optimizer')
 factory.registerType('GradientDescent', GradientDescent)
 factory.registerType('SimulatedAnnealing', SimulatedAnnealing)
 factory.registerType('GeneticAlgorithm', GeneticAlgorithm)
-factory.registerType('BayesianOptimizer', BayesianOptimizer)
+
+try:
+    from .BayesianOptimizer import BayesianOptimizer
+    factory.registerType('BayesianOptimizer', BayesianOptimizer)
+except ModuleNotFoundError as error:
+    print("ERROR: Unable to import BayesianOptimizer", error)

ravenframework/Optimizers/__init__.py

@@ -24,6 +24,9 @@
 from .RavenSampled import RavenSampled
 from .GradientDescent import GradientDescent
 from .SimulatedAnnealing import SimulatedAnnealing
-from .BayesianOptimizer import BayesianOptimizer
+try:
+    from .BayesianOptimizer import BayesianOptimizer
+except ModuleNotFoundError as error:
+  print("ERROR: Unable to import BayesianOptimizer", error)
 
 from .Factory import factory

ravenframework/SupervisedLearning/KerasBase.py

@@ -1910,7 +1910,9 @@ def __init__(self):
       # continues learning even when many updates have been done.
       self.availOptimizer['adadelta'] = tf.keras.optimizers.Adadelta
       # Adam optimzer
-      self.availOptimizer['adam'] = tf.keras.optimizers.Adam
+      self.availOptimizer['adam_new'] = tf.keras.optimizers.Adam
+      # Adam legacy optimizer
+      self.availOptimizer['adam'] = tf.keras.optimizers.legacy.Adam
       # Adamax optimizer from Adam paper's section 7
       self.availOptimizer['adamax'] = tf.keras.optimizers.Adamax
       # Nesterov Adam optimizer

scripts/library_handler.py

@@ -449,6 +449,14 @@ def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources, re
     # if this library's OS's don't match the requested OS, then we move on
     if opSys not in [x.lower().strip() for x in libOS.split(',')]:
       return # nothing to do
+  libMachine = libNode.attrib.get('machine', None)
+  if libMachine is not None:
+    machineType = platform.machine()
+    machineSubst = {'AMD64':'x86_64'} #substitutions for different names
+    if machineType in machineSubst:
+      machineType = machineSubst[machineType] #convert to standard name.
+    if machineType not in [x.lower().strip() for x in libMachine.split(',')]:
+      return # nothing to do since not machine type specified
   # check optional
   ## note that None means "not optional" in this case
   ## further note anything besides "True" is taken to mean "not optional"

tests/framework/CodeInterfaceTests/CobraTF/tests

@@ -19,8 +19,13 @@
   [./test3]
     type   = 'RavenFramework'
     input  = 'test3_cobratf_code_interface.xml'
-    text   = 'CobraTFinterfaceTest3/testDummyStep_MC/1/sample.inp CobraTFinterfaceTest3/testDummyStep_MC/1/vuq_param.txt CobraTFinterfaceTest3/testDummyStep_MC/2/sample.inp CobraTFinterfaceTest3/testDummyStep_MC/2/vuq_param.txt'
+    text   = 'CobraTFinterfaceTest3/testDummyStep_MC/1/sample.inp CobraTFinterfaceTest3/testDummyStep_MC/2/sample.inp'
     csv    = 'CobraTFinterfaceTest3/cobra_out.csv'
+    [./text]
+      type = NumericText
+      output = ' CobraTFinterfaceTest3/testDummyStep_MC/1/vuq_param.txt CobraTFinterfaceTest3/testDummyStep_MC/2/vuq_param.txt'
+      rel_err = 1e-8
+    [../]
     test_interface_only = True
   [../]
 

tests/framework/Optimizers/BayesianOptimizer/tests

@@ -3,6 +3,7 @@
     type = 'RavenFramework'
     input = 'Basic/BayesianBasic.xml'
     skip_if_OS = 'linux'
+    required_libraries = 'smt'
     [./data]
       type = OrderedCSV
       output = 'Basic/opt_export_0.csv'
@@ -17,6 +18,7 @@
     type = 'RavenFramework'
     input = 'Basic/BayesianBasic.xml'
     skip_if_OS = 'mac,windows'
+    required_libraries = 'smt'
     [./data]
       type = OrderedCSV
       output = 'Basic/final_opt.csv'
@@ -29,6 +31,7 @@
     type = 'RavenFramework'
     input = 'MishraBirdConstrained/BayesianMishraBirdConstrained.xml'
     skip_if_OS = 'linux'
+    required_libraries = 'smt'
     [./data]
       type = OrderedCSV
       output = 'MishraBirdConstrained/opt_export_0.csv'
@@ -43,6 +46,7 @@
     type = 'RavenFramework'
     input = 'MishraBirdConstrained/BayesianMishraBirdConstrained.xml'
     skip_if_OS = 'mac,windows'
+    required_libraries = 'smt'
     [./data]
       type = OrderedCSV
       output = 'MishraBirdConstrained/final_opt.csv'
@@ -55,6 +59,7 @@
     type = 'RavenFramework'
     input = 'Matyas/BayesianMatyas.xml'
     skip_if_OS = 'linux'
+    required_libraries = 'smt'
     [./data]
       type = OrderedCSV
       output = 'Matyas/opt_export_0.csv'
@@ -69,6 +74,7 @@
     type = 'RavenFramework'
     input = 'Matyas/BayesianMatyas.xml'
     skip_if_OS = 'mac,windows'
+    required_libraries = 'smt'
     [./data]
       type = OrderedCSV
       output = 'Matyas/final_opt.csv'

tests/framework/PostProcessors/LimitSurface/tests

@@ -39,6 +39,7 @@
    output = 'limitSurfaceBoundingError/LimitSurfaceWeightedPb_dump.csv'
    windows_gold = 'limitSurfaceBoundingError/WindowsLimitSurfaceWeightedPb_dump.csv'
    mac_gold =  'limitSurfaceBoundingError/MacLimitSurfaceWeightedPb_dump.csv'
+   rel_err = 0.002
    [../]
  [../]
 

tests/framework/Samplers/SparseGrid/tests

@@ -62,6 +62,7 @@
   input = 'test_scgpc_normal.xml'
   UnorderedXml = 'scgpc/Ndumprom.xml'
   csv = 'scgpc/Ndump.csv'
+  zero_threshold = 1e-12
  [../]
  [./gamma]
   type = 'RavenFramework'
@@ -96,6 +97,7 @@
   input = 'test_scgpc_triang.xml'
   UnorderedXml = 'scgpc/Tdumprom.xml'
   csv = 'scgpc/Tdump.csv'
+  zero_threshold = 1e-12
  [../]
  [./exponential]
   type = 'RavenFramework'

tests/framework/pca_adaptive_sgc/gold/Adaptive/dump.xml

@@ -2,6 +2,8 @@
   <rom type="Static">
     <ROM>
       <type>GAUSSgpcROM</type>
+      <Features>y1,y2</Features>
+      <Targets>ans,ans2</Targets>
     </ROM>
     <ans>
       <mean>1.434</mean>

tests/framework/pca_adaptive_sgc/tests

@@ -3,7 +3,7 @@
   type = 'RavenFramework'
   input = 'test_adaptive_sgc_pca_analytic.xml'
   UnorderedCsv = 'Adaptive/data.csv'
-  UnorderedXml = 'Adaptive/dump.xml'
+  xml = 'Adaptive/dump.xml'
   zero_threshold = 1e-12
  [../]
 
@@ -35,6 +35,7 @@
   UnorderedCsv = 'polynomial/csv_database.csv'
   UnorderedXml = 'polynomial/stats_td1.xml'                    #analytic, "Tensor Polynomial (First-Order) with Multivariate Normal Distribution"
   max_time = 500 # windows test box is slow, so very slow
+  zero_threshold = 1e-12
  [../]
 
 []

tests/framework/pca_sparseGridCollocation/tests

@@ -35,6 +35,8 @@
   input = 'test_poly_correlation.xml'
   csv = 'polyCorrelation/csv_database.csv'
   UnorderedXml = 'polyCorrelation/stats_td1.xml'
+  zero_threshold = 1e-12
+  rel_err = 1e-8
  [../]
 
  [./polyCorrelationMC]

tests/framework/tests

@@ -228,6 +228,7 @@
    input = 'test_redundant_inputs.xml'
    output = 'RedundantInputs/mc_out.xml RedundantInputs/grid_out.xml'
    csv = 'RedundantInputs/mc_out.csv RedundantInputs/grid_out.csv'
+   rel_err = 2.0e-3
  [../]
  [./redundantInputsSCgPC]
    type = 'RavenFramework'
@@ -262,6 +263,7 @@
   type = 'RavenFramework'
   input = 'test_simple_ND_grid_probabilityWeight_value.xml'
   csv = 'NDGridPWvalue/testPrintPointSet_dump.csv'
+  rel_err = 1.0e-9
  [../]
 
  [./LHSVariables]