[luci/service] Support dynamic shape inference for reshape

.ahub/sam/exclude.txt

@@ -26,10 +26,6 @@ runtime/libs/circle-schema/include/circle_schema_generated.h
 runtime/libs/circle-schema/include/circle_traininfo_generated.h
 runtime/onert/core/src/loader/tflite_schema_generated.h
 
-# External code: Android NN API
-runtime/onert/api/nnapi/include/NeuralNetworks.h
-runtime/onert/api/nnapi/include/NeuralNetworksExtensions.h
-
 # External code: Tensorflow lite
 runtime/libs/profiling/
 

compiler/common-artifacts/exclude.lst

@@ -7,6 +7,7 @@
 ## TensorFlowLiteRecipes
 optimize(Add_STR_000) # STRING is not supported
 optimize(Add_STR_001) # STRING is not supported
+optimize(Net_Gather_SparseToDense_AddV2_000) # Constant folding is not generally supported
 
 ## CircleRecipes
 

compiler/fm-equalize/fmelib/ComputeParam.py

@@ -37,3 +37,13 @@ def _channelwiseMinMax(tensors: Tensors, channel: int) -> Tuple[List[float], Lis
         channel_wise_min.append(float(np.min(min_act[:, :, :, c])))
         channel_wise_max.append(float(np.max(max_act[:, :, :, c])))
     return channel_wise_min, channel_wise_max
+
+
+def getActivationMax(tensor: np.ndarray) -> np.ndarray:
+    """
+    Get max values of activation.
+    :param tensors: a list of numpy array (each is a tensor)
+    :return: 1D array of max absolute values for each channel
+    """
+    # max along with last dimension
+    return np.abs(tensor.reshape(-1, tensor.shape[-1])).max(axis=0)

compiler/fm-equalize/fmelib/DumpFMEParams.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script that dumps scale parameters for FM equalization
+# NOTE This script runs on dalgona
+
+from typing import Dict
+
+import numpy as np
+import json
+import os
+
+from ComputeParam import getActivationMax
+
+
+# Recursively visit items and round floats with ndigits
+def _pretty_float(item, ndigits=6):
+    if isinstance(item, list):
+        return [_pretty_float(x, ndigits) for x in item]
+    if isinstance(item, float):
+        return round(item, ndigits)
+    return item
+
+
+# Dump scale parameters for FM equalization
+# The parameters are saved in the json file given as an input
+#
+# Contents of the json file ('<model_name>.fme_patterns.json')
+#
+# Before
+# [
+#   {
+#     “front”: <tensor_name>,
+#     “back”: <tensor_name>,
+#     “type”: ScaleOnly,
+#   },
+#   …
+# ]
+#
+# After
+# [
+#   {
+#     “front”: <tensor_name>,
+#     “back”: <tensor_name>,
+#     “type”: ScaleOnly,
+#     “act_scale”: [..],
+#   },
+#   …
+# ]
+class DumpFMEParams:
+
+    # Return path to the data
+    # self._dir_path/<tid>.<data_idx>.npy
+    def _data_path(self, tid: int, data_idx: int):
+        assert (self._dir_path != None)  # FIX_CALLER_UNLESS
+        return self._dir_path + '/' + str(tid) + '.' + str(data_idx) + '.npy'
+
+    def record_activaion_max(self, tensor: dict):
+        tensor_name = tensor['name']
+
+        act_max = getActivationMax(tensor['data'])
+        old_act_max = self.activation_max.get(tensor_name, act_max)
+        self.activation_max[tensor_name] = np.maximum.reduce([old_act_max, act_max])
+
+    def StartAnalysis(self, args: str):
+        """Called when the analysis starts"""
+        self._json_path = args
+        self._dir_path = os.path.dirname(args)
+        # Data structure to save tensor information
+        # {
+        #   <tensor_name>: <activation max>
+        # }
+        self.activation_max: Dict[str, float] = {}
+
+        # Names of target tensors ('back' of equalization pattern)
+        self._target_tensors = []
+
+        with open(args, 'r') as f:
+            patterns = json.load(f)
+            for pattern in patterns:
+                self._target_tensors.append(pattern['front'])
+            self._patterns = patterns
+
+    def DefaultOpPost(self, name, opcode, inputs, output):
+        # DO NOTHING
+        pass
+
+    def Conv2DPost(self, name, input, filter, bias, padding, stride, dilation, output,
+                   fused_act):
+        if name in self._target_tensors:
+            self.record_activaion_max(output)
+
+    def DepthwiseConv2DPost(self, name, input, filter, bias, padding, stride,
+                            depthMultiplier, dilation, output, fused_act):
+        if name in self._target_tensors:
+            self.record_activaion_max(output)
+
+    def FullyConnectedPost(self, name, input, weights, bias, output, fused_act):
+        if name in self._target_tensors:
+            self.record_activaion_max(output)
+
+    def InstanceNormPost(self, name, input, gamma, beta, epsilon, output, fused_act):
+        if name in self._target_tensors:
+            self.record_activaion_max(output)
+
+    def TransposeConvPost(self, name, input, filter, output_shape, bias, padding, stride,
+                          output):
+        if name in self._target_tensors:
+            self.record_activaion_max(output)
+
+    def EndAnalysis(self):
+        """Called when the analysis ends"""
+
+        res = []
+        for pattern in self._patterns:
+            # pattern['front'] is the input of pattern['back']
+            tensor_name = pattern['front']
+            eq_type = pattern['type']
+
+            act_max = self.activation_max[tensor_name]
+
+            if eq_type == 'ScaleOnly':
+                pattern['act_scale'] = _pretty_float(act_max).tolist()
+            else:
+                raise ValueError('Unknown equalization type: ' + eq_type)
+
+            res.append(pattern)
+
+        # Overwrite
+        with open(self._json_path, 'w') as json_file:
+            json.dump(res, json_file)

compiler/fme-apply/src/EqualizePattern.h

@@ -35,8 +35,8 @@ struct EqualizePattern
   std::string front;
   std::string back;
   Type type = Type::Invalid;
-  std::vector<float> scale; // channel-wise scale
-  std::vector<float> shift; // channel-wise shift
+  std::vector<float> act_scale; // activation channel scales
+  std::vector<float> scale;     // channel-wise scale
 };
 
 } // namespace fme_apply

compiler/fme-apply/src/EqualizePatternCheck.cpp

@@ -28,6 +28,12 @@ using namespace fme_apply;
 namespace fme_apply
 {
 
+/**
+ * It checks if given patterns are valid as follows.
+ *
+ * - "scale" is empty.
+ * - "front" and "back" of the patterns are in the graph.
+ */
 void check_patterns_valid(loco::Graph *g, const std::vector<EqualizePattern> &patterns)
 {
   // Create a map to find node by its name
@@ -40,27 +46,21 @@ void check_patterns_valid(loco::Graph *g, const std::vector<EqualizePattern> &pa
     }
   }
 
-  auto check_negative_scale_across_relu = [&node_by_name](const EqualizePattern *p) {
-    auto front = node_by_name.at(p->front); // FIX_ME_UNLESS
-    auto node =
-      dynamic_cast<const luci::CircleNodeMixin<luci::CircleNodeTrait::FusedActFunc> *>(front);
-    if (not node)
-      return;
-
-    if (node->fusedActivationFunction() != luci::FusedActFunc::RELU)
-      return;
-
-    if (p->type != EqualizePattern::Type::ScaleOnly && p->type != EqualizePattern::Type::ScaleShift)
-      return;
-
-    for (auto s : p->scale)
-      if (s < 0.0)
-        throw std::runtime_error("Negative scale cannot be fused across ReLU");
-  };
-
-  for (const auto &pattern : patterns)
+  for (const auto &p : patterns)
   {
-    check_negative_scale_across_relu(&pattern);
+    // "scale" is empty.
+    // "scale" is calculated in the runtime.
+    if (not p.scale.empty())
+    {
+      throw std::runtime_error{"'scale' shouldn't exist."};
+    }
+
+    // "front" and "back" of the patterns are in the graph.
+    if (node_by_name.find(p.front) == node_by_name.end() or
+        node_by_name.find(p.back) == node_by_name.end())
+    {
+      throw std::runtime_error{"Given front or back don't exist in the graph."};
+    }
   }
 }
 

compiler/fme-apply/src/EqualizePatternCheck.test.cpp

@@ -86,7 +86,7 @@ TEST(EqualizePatternCheckTest, simple)
     pattern.back = "conv2";
     pattern.type = EqualizePattern::Type::ScaleOnly;
     for (uint32_t i = 0; i < 16; i++)
-      pattern.scale.push_back(1.0);
+      pattern.act_scale.push_back(1.0);
   }
   p.emplace_back(pattern);
 
@@ -96,16 +96,16 @@ TEST(EqualizePatternCheckTest, simple)
   EXPECT_NO_THROW(check_patterns_valid(g.g(), p));
 }
 
-TEST(EqualizePatternCheckTest, negative_scale_across_relu_scaleonly_NEG)
+TEST(EqualizePatternCheckTest, invalid_names_NEG)
 {
   std::vector<EqualizePattern> p;
   EqualizePattern pattern;
   {
     pattern.front = "conv1";
-    pattern.back = "conv2";
+    pattern.back = "conv3"; // invalid
     pattern.type = EqualizePattern::Type::ScaleOnly;
     for (uint32_t i = 0; i < 16; i++)
-      pattern.scale.push_back(-1.0); // Negative value
+      pattern.act_scale.push_back(1.0);
   }
   p.emplace_back(pattern);
 
@@ -116,16 +116,16 @@ TEST(EqualizePatternCheckTest, negative_scale_across_relu_scaleonly_NEG)
   EXPECT_ANY_THROW(check_patterns_valid(g.g(), p));
 }
 
-TEST(EqualizePatternCheckTest, negative_scale_across_relu_scalshift_NEG)
+TEST(EqualizePatternCheckTest, invalid_scale_NEG)
 {
   std::vector<EqualizePattern> p;
   EqualizePattern pattern;
   {
     pattern.front = "conv1";
     pattern.back = "conv2";
-    pattern.type = EqualizePattern::Type::ScaleShift;
+    pattern.type = EqualizePattern::Type::ScaleOnly;
     for (uint32_t i = 0; i < 16; i++)
-      pattern.scale.push_back(-1.0); // Negative value
+      pattern.scale.push_back(1.0); // invalid
   }
   p.emplace_back(pattern);
 

compiler/fme-apply/src/EqualizePatternRead.cpp

@@ -101,14 +101,7 @@ std::vector<EqualizePattern> read(const std::string &filename)
       switch (p.type)
       {
         case EqualizePattern::Type::ScaleOnly:
-          p.scale = get_fp32_array("scale");
-          break;
-        case EqualizePattern::Type::ShiftOnly:
-          p.shift = get_fp32_array("shift");
-          break;
-        case EqualizePattern::Type::ScaleShift:
-          p.scale = get_fp32_array("scale");
-          p.shift = get_fp32_array("shift");
+          p.act_scale = get_fp32_array("act_scale");
           break;
         default:
           throw std::runtime_error("Unsupported EqualizePattern type");

compiler/fme-apply/src/EqualizePatternRead.test.cpp

@@ -58,7 +58,7 @@ TEST_F(EqualizePatternReadTest, simple)
                                "\"front\": \"f\","
                                "\"back\": \"b\","
                                "\"type\": \"ScaleOnly\","
-                               "\"scale\": [1.0, 2.0, 3.0]"
+                               "\"act_scale\": [4.0, 5.0, 6.0]"
                                "}"
                                "]";
 
@@ -70,11 +70,10 @@ TEST_F(EqualizePatternReadTest, simple)
   EXPECT_EQ("f", patterns[0].front);
   EXPECT_EQ("b", patterns[0].back);
   EXPECT_EQ(EqualizePattern::Type::ScaleOnly, patterns[0].type);
-  EXPECT_EQ(3, patterns[0].scale.size());
-  EXPECT_FLOAT_EQ(1.0, patterns[0].scale[0]);
-  EXPECT_FLOAT_EQ(2.0, patterns[0].scale[1]);
-  EXPECT_FLOAT_EQ(3.0, patterns[0].scale[2]);
-  EXPECT_TRUE(patterns[0].shift.empty());
+  EXPECT_EQ(3, patterns[0].act_scale.size());
+  EXPECT_FLOAT_EQ(4.0, patterns[0].act_scale[0]);
+  EXPECT_FLOAT_EQ(5.0, patterns[0].act_scale[1]);
+  EXPECT_FLOAT_EQ(6.0, patterns[0].act_scale[2]);
 }
 
 TEST_F(EqualizePatternReadTest, no_file_NEG) { EXPECT_ANY_THROW(fme_apply::read(_filename)); }

compiler/fme-apply/src/FMEqualizer.cpp

@@ -37,8 +37,8 @@ using namespace fme_apply;
 namespace
 {
 
-// Throw exception if virtual Op (Scale/Shift) exists
-void check_no_scale_shift(loco::Graph *g)
+// Throw exception if virtual Op (Scale) exists
+void check_no_scale(loco::Graph *g)
 {
   for (auto node : loco::active_nodes(loco::output_nodes(g)))
   {
@@ -50,8 +50,8 @@ void check_no_scale_shift(loco::Graph *g)
       continue;
 
     const auto code = custom->custom_code();
-    if (code == "PreScale" or code == "PreShift" or code == "PostScale" or code == "PostShift")
-      throw std::runtime_error("Virtual node(" + code + ") remains. " + custom->name());
+    if (code == "scale")
+      throw std::runtime_error("Virtual node(" + code + ") remains.");
   }
 }
 
@@ -60,7 +60,7 @@ void check_no_scale_shift(loco::Graph *g)
 namespace fme_apply
 {
 
-void FMEqualizer::equalize(loco::Graph *g, const std::vector<EqualizePattern> &p)
+void FMEqualizer::equalize(loco::Graph *g, std::vector<EqualizePattern> &p)
 {
   THROW_UNLESS(g != nullptr, "Invalid argument g");
 
@@ -78,7 +78,7 @@ void FMEqualizer::equalize(loco::Graph *g, const std::vector<EqualizePattern> &p
   phase.emplace_back(std::make_unique<luci::CircleShapeInferencePass>());
   phase.emplace_back(std::make_unique<luci::CircleTypeInferencePass>());
 
-  // Fuse Pre/Post Scale/Shift
+  // Fuse Pre/Post Scale
   phase.emplace_back(std::make_unique<fme_apply::FusePreScalePass>());
   phase.emplace_back(std::make_unique<fme_apply::FusePostScalePass>());
 
@@ -88,7 +88,7 @@ void FMEqualizer::equalize(loco::Graph *g, const std::vector<EqualizePattern> &p
   phase_runner.run(phase);
 
   // Check if all Scale/Shift nodes are removed
-  check_no_scale_shift(g);
+  check_no_scale(g);
 }
 
 } // namespace fme_apply

compiler/fme-apply/src/FMEqualizer.h

@@ -27,7 +27,7 @@ namespace fme_apply
 class FMEqualizer final
 {
 public:
-  void equalize(loco::Graph *g, const std::vector<EqualizePattern> &p);
+  void equalize(loco::Graph *g, std::vector<EqualizePattern> &p);
 };
 
 } // namespace fme_apply