#26 parse function explanation added

wisdomify · Nov 7, 2021 · 4345fdc · 4345fdc
1 parent 108f777
commit 4345fdc
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/storyteller/preprocess.py b/storyteller/preprocess.py
@@ -18,12 +18,18 @@ def parse(df: pd.DataFrame) -> pd.DataFrame:
     """
 
     df['eg'] = df['eg'].apply(
+        # return list of example only on 'eg' column (proverb is converted to [WISDOM])
         lambda r: list(map(
+            # while iterating list of 'hits'
+            # convert <em> ... lalib ... </em> to [WISDOM]
             lambda hit: re.sub(r"<em>.*</em>", "[WISDOM]", hit['highlight']['sents'][0]),
+            # loading json to dict -> taking dict['hits']['hits']
             json.loads(r)['hits']['hits']
         ))
     )
 
+    # 'eg' column contains list object
+    # -> converted to single value with multiple columns
     df = df.explode('eg')
 
     return df