feat: add MoveNet and PoseNet notebooks for experiment

fall_prediction.py

@@ -24,6 +24,7 @@ def _fall_detect_config():
         'labels': _good_labels,
         'top_k': 3,
         'confidence_threshold': 0.6,
+        'model_name':'mobilenet'
     }
     return config
 

install_requirements.sh

@@ -34,17 +34,9 @@ pip3 install -r requirements.txt
 
 
 echo "Installing tflite for x86 CPU"
-if python3 --version | grep -q 3.8
-then
-  pip3 install --force-reinstall https://github.com/google-coral/pycoral/releases/download/v1.0.1/tflite_runtime-2.5.0-cp38-cp38-linux_x86_64.whl
-else
-  pip3 install https://dl.google.com/coral/python/tflite_runtime-1.14.0-cp37-cp37m-linux_x86_64.whl
-fi
 
+pip3 install --index-url https://google-coral.github.io/py-repo/ tflite_runtime
 pip3 list
-pip3 show tflite-runtime
-
-
 
 # [Cleanup]
 sudo apt-get -y autoremove

posenet_models_unit_test.csv

@@ -0,0 +1,83 @@
+image,model,LShoulder_score,RShoulder_score,LHip_score,RHip_score,Inference_time
+fall_img_46.png,movenet,0.6819305,0.5842324,0.56410754,0.6362491,0.2540985039999999
+fall_img_46.png,posenet,0.69342387,0.36183468,0.74672127,0.5799404,0.14426371400000004
+fall_img_2.png,movenet,0.5514568,0.6855382,0.5712503,0.4971534,0.20016966699999994
+fall_img_2.png,posenet,0.9842503,0.82194954,0.87478703,0.94453293,0.10588090700000041
+fall_img_31.jpg,movenet,0.51481885,0.48256937,0.6345951,0.47572482,0.18377915499999986
+fall_img_31.jpg,posenet,0.8169527,0.78705454,0.61934024,0.43284088,0.14038745399999986
+fall_img_45.jpg,movenet,0.29324666,0.28072846,0.4282155,0.46244907,0.21849277500000053
+fall_img_45.jpg,posenet,0.08013292,0.06459336,0.13733537,0.16932018,0.12332088599999924
+fall_img_43.jpg,movenet,0.29134363,0.4097841,0.50563425,0.55706006,0.1958281520000007
+fall_img_43.jpg,posenet,0.3587505,0.38052747,0.4743314,0.5782926,0.10769053200000034
+fall_img_27.jpg,movenet,0.38543415,0.44316107,0.4570407,0.45220768,0.18684843200000056
+fall_img_27.jpg,posenet,0.99310285,0.8825601,0.9892623,0.9479455,0.09993119099999959
+fall_img_36.jpg,movenet,0.5792162,0.608703,0.1727243,0.13766456,0.19952614900000043
+fall_img_36.jpg,posenet,0.43793395,0.13267146,0.21109623,0.09329173,0.13237763100000066
+fall_img_18.png,movenet,0.67180467,0.5602175,0.70882577,0.7714026,0.19587414399999936
+fall_img_18.png,posenet,0.99404085,0.83740944,0.99810404,0.9752334,0.11412095999999927
+fall_img_6.png,movenet,0.57402754,0.6617981,0.48006332,0.48390344,0.20785609900000068
+fall_img_6.png,posenet,0.9666462,0.94852865,0.9261057,0.74862593,0.1090223839999993
+fall_img_38.jpg,movenet,0.4046827,0.32330233,0.6284069,0.5317362,0.188134655999999
+fall_img_38.jpg,posenet,0.9942818,0.981373,0.99523616,0.9924943,0.12237617199999917
+fall_img_22.jpg,movenet,0.06042713,0.090580404,0.24557343,0.34067115,0.18726404600000102
+fall_img_22.jpg,posenet,0.06480862,0.060696777,0.02943276,0.04805087,0.1124404590000001
+fall_img_8.png,movenet,0.57642716,0.5935232,0.65205187,0.6925719,0.18800517800000094
+fall_img_8.png,posenet,0.99942774,0.97129595,0.92931855,0.8278,0.11419718400000001
+fall_img_41.jpg,movenet,0.48216346,0.4079069,0.3897511,0.35610485,0.19650474100000004
+fall_img_41.jpg,posenet,0.6761217,0.7182537,0.50929534,0.6189367,0.13334337700000098
+fall_img_29.jpg,movenet,0.41138604,0.3875395,0.38787916,0.43213737,0.20753362600000003
+fall_img_29.jpg,posenet,0.028314507,0.037679303,0.026444392,0.043686103,0.10527827700000003
+fall_img_9.png,movenet,0.27230793,0.3088425,0.19749063,0.171467,0.18010737900000073
+fall_img_9.png,posenet,0.011118463,0.017692,0.0056906105,0.0065444713,0.11892035999999884
+fall_img_19.png,movenet,0.0651651,0.1590397,0.078885645,0.121895164,0.16386596199999914
+fall_img_19.png,posenet,0.64090765,0.66178703,0.30963737,0.31163716,0.10894279299999887
+fall_img_42.jpg,movenet,0.47542182,0.57622933,0.7953565,0.67832226,0.18384632799999956
+fall_img_42.jpg,posenet,0.5225503,0.90776765,0.9359411,0.754716,0.12065812200000003
+fall_img_1.png,movenet,0.7220231,0.518078,0.55900866,0.5907552,0.1954106909999993
+fall_img_1.png,posenet,0.99347764,0.9972915,0.9935631,0.99635535,0.09136559699999935
+fall_img_39.jpg,movenet,0.4636351,0.65149367,0.5330208,0.66735494,0.20252261999999988
+fall_img_39.jpg,posenet,0.990964,0.8839682,0.9993354,0.9543344,0.12125916199999942
+fall_img_5.png,movenet,0.6493088,0.57786906,0.57014275,0.53305167,0.22903751900000024
+fall_img_5.png,posenet,0.9845722,0.9413432,0.94040734,0.92478985,0.11749330499999999
+fall_img_26.jpg,movenet,0.35181302,0.28550175,0.2578727,0.3013627,0.19634643799999907
+fall_img_26.jpg,posenet,0.78985244,0.86632884,0.80895525,0.77620715,0.11366006599999956
+fall_img_47.png,movenet,0.6989045,0.49237436,0.5286144,0.5482457,0.223991701000001
+fall_img_47.png,posenet,0.9559954,0.96266806,0.8468235,0.86478263,0.1144607889999989
+fall_img_23.jpg,movenet,0.06042713,0.090580404,0.24557343,0.34067115,0.17779585699999956
+fall_img_23.jpg,posenet,0.06480862,0.060696777,0.02943276,0.04805087,0.11074089499999928
+fall_img_48.png,movenet,0.70592093,0.6292974,0.57736087,0.6445884,0.19012303999999958
+fall_img_48.png,posenet,0.8908973,0.70779157,0.99134,0.8552165,0.11257309600000198
+fall_img_28.jpg,movenet,0.7090164,0.48855498,0.6736608,0.54246265,0.19425879299999949
+fall_img_28.jpg,posenet,0.8323339,0.9378493,0.7245066,0.84253985,0.11928166899999937
+fall_img_7.png,movenet,0.46757096,0.43270636,0.53369397,0.44386715,0.19764870699999904
+fall_img_7.png,posenet,0.9385126,0.9922037,0.8956691,0.90781033,0.11499780200000131
+fall_img_4.png,movenet,0.20349178,0.15909463,0.29380772,0.35532477,0.20916026100000096
+fall_img_4.png,posenet,0.004773718,0.0055866074,0.0012511327,0.0019156403,0.10847501699999995
+fall_img_34.jpg,movenet,0.5638154,0.6443332,0.536523,0.57594264,0.20660395600000214
+fall_img_34.jpg,posenet,0.9325178,0.95144737,0.8590275,0.97449934,0.11204048700000158
+fall_img_40.jpg,movenet,0.60853237,0.6954137,0.5840774,0.57719195,0.17327212000000003
+fall_img_40.jpg,posenet,0.9946446,0.9982699,0.99097717,0.988087,0.10860424499999866
+fall_img_30.jpg,movenet,0.6644391,0.45011955,0.5667578,0.6276376,0.1967397869999985
+fall_img_30.jpg,posenet,0.9091012,0.72147924,0.9856206,0.8084575,0.1476578410000009
+background.jpg,movenet,0.011379272,0.027954906,0.012384951,0.03174165,0.17961108699999784
+background.jpg,posenet,0.0031433364,0.0015060705,0.009571217,0.00648168,0.14815919699999824
+fall_img_0.png,movenet,0.3685925,0.40784594,0.5072441,0.53182495,0.20680322599999812
+fall_img_0.png,posenet,0.9005406,0.9945664,0.9415373,0.9941527,0.14672335699999906
+fall_img_21.jpg,movenet,0.19473442,0.15379488,0.13752794,0.10585514,0.18752656699999903
+fall_img_21.jpg,posenet,0.0031907763,0.0052951495,0.004225612,0.005557035,0.11748993999999868
+fall_img_35.jpg,movenet,0.45273593,0.5616107,0.61524975,0.6321429,0.2044615250000028
+fall_img_35.jpg,posenet,0.9274382,0.8731172,0.94449514,0.9481014,0.09953262199999813
+fall_img_44.jpg,movenet,0.4848252,0.6009575,0.63397104,0.4999005,0.19058707299999966
+fall_img_44.jpg,posenet,0.78244233,0.9479431,0.9306459,0.99197817,0.10536027899999922
+fall_img_37.jpg,movenet,0.42246965,0.64756715,0.6421648,0.6165342,0.20145623499999843
+fall_img_37.jpg,posenet,0.9412618,0.934462,0.9845023,0.9655075,0.11534595300000205
+fall_img_32.jpg,movenet,0.28776717,0.22512242,0.15157163,0.13117048,0.19345090800000264
+fall_img_32.jpg,posenet,0.049259413,0.039405648,0.015937367,0.019278647,0.11347303399999831
+fall_img_24.jpg,movenet,0.09763062,0.09028259,0.11482483,0.15692073,0.1777742759999974
+fall_img_24.jpg,posenet,0.13359007,0.08598422,0.10951048,0.13772556,0.11474216100000234
+fall_img_20.png,movenet,0.16873437,0.16755846,0.6144976,0.57287884,0.20607699099999977
+fall_img_20.png,posenet,0.64510906,0.94567925,0.89202195,0.9924114,0.09962109599999991
+fall_img_3.png,movenet,0.6288005,0.67481524,0.6196538,0.794412,0.18099559899999917
+fall_img_3.png,posenet,0.9559911,0.9079693,0.85024244,0.7811313,0.11437555899999907
+fall_img_33.jpg,movenet,0.36124235,0.6049675,0.29604268,0.3293897,0.21695801299999928
+fall_img_33.jpg,posenet,0.9879984,0.95383805,0.96492946,0.8616411,0.10618814500000084

src/pipeline/fall_detect.py

@@ -18,6 +18,7 @@ def __init__(self,
                  model=None,
                  labels=None,
                  confidence_threshold=0.15,
+                 model_name=None,
                  **kwargs
                  ):
         """Initialize detector with config parameters.
@@ -36,6 +37,7 @@ def __init__(self,
                         model=model,
                         labels=labels,
                         confidence_threshold=confidence_threshold)
+        self.model_name = model_name
 
         self._sys_data_dir = DEFAULT_DATA_DIR
         self._sys_data_dir = Path(self._sys_data_dir)
@@ -64,7 +66,7 @@ def __init__(self,
         # self._prev_data[1] : store data of frame at t-1
         self._prev_data[0] = self._prev_data[1] = _dix
 
-        self._pose_engine = PoseEngine(self._tfengine)
+        self._pose_engine = PoseEngine(self._tfengine, self.model_name)
         self._fall_factor = 60
         self.confidence_threshold = confidence_threshold
         log.debug(f"Initializing FallDetector with conficence threshold: \
@@ -291,9 +293,9 @@ def draw_lines(self, thumbnail, pose_dix, score):
         timestr = int(time.monotonic()*1000)
         debug_image_file_name = \
             f'tmp-fall-detect-thumbnail-{timestr}-score-{score}.jpg'
-        thumbnail.save(
-                       Path(self._sys_data_dir, debug_image_file_name),
-                       format='JPEG')
+        # thumbnail.save(
+        #                Path(self._sys_data_dir, debug_image_file_name),
+        #                format='JPEG')
         print(Path(self._sys_data_dir, debug_image_file_name))
         return body_lines_drawn
 

src/pipeline/inference.py

@@ -79,16 +79,16 @@ def __init__(self,
             .format(labels)
         self._model_labels_path = labels
         self._confidence_threshold = confidence_threshold
-        log.info('Loading AI model:\n'
-                  'TFLite graph: %r\n'
-                  'EdgeTPU graph: %r\n'
-                  'Labels %r.'
-                  'Condidence threshod: %.0f%%'
-                  'top-k: %d',
-                  model_tflite,
-                  model_edgetpu,
-                  labels,
-                  confidence_threshold*100)
+        # log.info('Loading AI model:\n'
+        #           'TFLite graph: %r\n'
+        #           'EdgeTPU graph: %r\n'
+        #           'Labels %r.'
+        #           'Condidence threshod: %.0f%%'
+        #           'top-k: %d',
+        #           model_tflite,
+        #           model_edgetpu,
+        #           labels,
+        #           confidence_threshold*100)
         # EdgeTPU is not available in testing and other environments
         # load dynamically as needed
 #        edgetpu_class = 'DetectionEngine'

src/pipeline/movenet_model.py

@@ -0,0 +1,89 @@
+from src.pipeline.pose_base import AbstractPoseModel
+import numpy as np
+import time
+
+
+class Movenet(AbstractPoseModel):
+    '''The class for pose estimation using Movenet implementation.'''
+
+    def __init__(self, tfengine):
+        super().__init__(tfengine)
+
+
+    def parse_output(self, keypoints_with_scores, height, width):
+        '''
+            Parse Output of TFLite model and get keypoints with score.
+        '''
+
+        keypoints_all = []
+        num_instances, _, _, _ = keypoints_with_scores.shape
+
+        for idx in range(num_instances):
+
+            kpts_y = keypoints_with_scores[0, idx, :, 1]
+            kpts_x = keypoints_with_scores[0, idx, :, 0]
+
+            kpts_scores = keypoints_with_scores[0, idx, :, 2]
+
+            kpts_absolute_xy = np.stack([width * np.array(kpts_x), height * np.array(kpts_y), kpts_scores], axis=-1)
+            keypoints_all.append(kpts_absolute_xy)
+
+        if keypoints_all:
+            keypoints_xy = np.concatenate(keypoints_all, axis=0)
+        else:
+            keypoints_xy = np.zeros((0, 17, 2))
+
+        return keypoints_xy
+
+
+    def execute_model(self, img):
+        ''' Run TFLite model.
+
+        :Parameters:
+        ----------
+        img: PIL.Image
+            Input Image for AI model detection.
+        :Returns:
+        -------
+        kps:
+            A list of Pose objects with keypoints and confidence scores
+        template_image: PIL.Image
+            Input resized image.
+        thumbnail: PIL.Image
+            Thumbnail input image
+        _inference_time: float
+            Model inference time in seconds
+        '''
+
+        _tensor_input_size = (self._tensor_image_width,
+                              self._tensor_image_height)
+
+        # thumbnail is a proportionately resized image
+        thumbnail = self.thumbnail(image=img,
+                                               desired_size=_tensor_input_size)
+        # convert thumbnail into an image with the exact size
+        # as the input tensor preserving proportions by padding with
+        # a solid color as needed
+        template_image = self.resize(image=thumbnail,
+                                desired_size=_tensor_input_size)
+
+        start_time = time.process_time()
+
+        template_input = np.expand_dims(template_image.copy(), axis=0)
+        floating_model = self._tfengine.input_details[0]['dtype'] == np.float32
+
+        if floating_model:
+            template_input = template_input.astype(np.float32)
+
+        self.tf_interpreter().\
+            set_tensor(self._tfengine.input_details[0]['index'],
+                       template_input)
+        self.tf_interpreter().invoke()
+
+        keypoints_with_scores = self.tf_interpreter().get_tensor(self._tfengine.output_details[0]['index'])
+        kps = self.parse_output(keypoints_with_scores, self._tensor_image_height, self._tensor_image_width)
+
+
+        _inference_time = time.process_time() - start_time
+
+        return kps, template_image, thumbnail, _inference_time

src/pipeline/pose_base.py

@@ -0,0 +1,131 @@
+from abc import ABC, abstractmethod
+import numpy as np
+from PIL import ImageOps
+
+import logging
+log = logging.getLogger(__name__)
+
+class AbstractPoseModel(ABC):
+    """
+        Abstract class for pose estimation models.
+    """
+
+    def __init__(self, tfengine):
+
+        """Initialize posenet-base class with Tensorflow inference engine.
+        :Parameters:
+        ----------
+        tfengine: Tensorflow inference engine.
+        """
+
+        self._tfengine = tfengine
+
+        self._input_tensor_shape = self.get_input_tensor_shape()
+
+        _, self._tensor_image_height, self._tensor_image_width, self._tensor_image_depth = \
+            self.get_input_tensor_shape()
+
+        self.confidence_threshold = self._tfengine.confidence_threshold
+        log.debug(f"Initializing PoseEngine with confidence threshold \
+            {self.confidence_threshold}")
+
+
+    def get_input_tensor_shape(self):
+        """Get the shape of the input tensor structure.
+        Gets the shape required for the input tensor.
+        For models trained for image classification / detection, the shape is
+        always [1, height, width, channels].
+        To be used as input for :func:`run_inference`,
+        this tensor shape must be flattened into a 1-D array with size
+        ``height * width * channels``. To instead get that 1-D array size, use
+        :func:`required_input_array_size`.
+        Returns:
+        A 1-D array (:obj:`numpy.ndarray`) representing the required input
+        tensor shape.
+        """
+        return self._tfengine.input_details[0]['shape']
+
+
+    def tf_interpreter(self):
+        return self._tfengine._tf_interpreter
+
+
+    def thumbnail(self, image=None, desired_size=None):
+        """Resizes original image as close as possible to desired size.
+        Preserves aspect ratio of original image.
+        Does not modify the original image.
+        :Parameters:
+        ----------
+        image : PIL.Image
+            Input Image for AI model detection.
+        desired_size : (width, height)
+            Size expected by the AI model.
+        :Returns:
+        -------
+        PIL.Image
+            Resized image fitting for the AI model input tensor.
+        """
+        assert image
+        assert desired_size
+        log.debug('input image size = %r', image.size)
+        thumb = image.copy()
+        w, h = desired_size
+        try:
+            # convert from numpy to native Python int type
+            # that PIL expects
+            if isinstance(w, np.generic):
+                w = w.item()
+                w = int(w)
+                h = h.item()
+                h = int(h)
+            thumb.thumbnail((w, h))
+        except Exception as e:
+            msg = (f"Exception in "
+                   f"PIL.image.thumbnail(desired_size={desired_size}):"
+                   f"type(width)={type(w)}, type(height)={type(h)}"
+                   f"\n{e}"
+                   )
+            log.exception(msg)
+            raise RuntimeError(msg)
+        log.debug('thmubnail image size = %r', thumb.size)
+        return thumb
+
+
+    def resize(self, image=None, desired_size=None):
+        """Pad original image to exact size expected by input tensor.
+        Preserve aspect ratio to avoid confusing the AI model with
+        unnatural distortions. Pad the resulting image
+        with solid black color pixels to fill the desired size.
+        Do not modify the original image.
+        :Parameters:
+        ----------
+        image : PIL.Image
+            Input Image sized to fit an input tensor but without padding.
+            Its possible that one size fits one tensor dimension exactly
+            but the other size is smaller than
+            the input tensor other dimension.
+        desired_size : (width, height)
+            Exact size expected by the AI model.
+        :Returns:
+        -------
+        PIL.Image
+            Resized image fitting exactly the AI model input tensor.
+        """
+        assert image
+        assert desired_size
+        log.debug('input image size = %r', image.size)
+        thumb = image.copy()
+        delta_w = desired_size[0] - thumb.size[0]
+        delta_h = desired_size[1] - thumb.size[1]
+        padding = (0, 0, delta_w, delta_h)
+        new_im = ImageOps.expand(thumb, padding)
+        log.debug('new image size = %r', new_im.size)
+        assert new_im.size == desired_size
+        return new_im
+
+
+    @abstractmethod
+    def execute_model(self, img):
+        '''
+            Execute Pose Estimation Model.
+        '''