Feature/fix pop menu

agent/agent.py

@@ -118,10 +118,19 @@ def set_action_set_tag(self, tag: str) -> None:
     @beartype
     def next_action(
         self, trajectory: Trajectory, intent: str, meta_data: dict[str, Any]
-    ) -> Action:
+    ) -> tuple[Action, list[str]]:
         prompt = self.prompt_constructor.construct(
             trajectory, intent, meta_data
         )
+        pattern = r"\[(\d+)\] (combobox 'Sort By' hasPopup: menu expanded: True|menuitem 'Product Name' selected: False|menuitem 'Price' selected: False|menuitem 'Relevance' selected: True)"
+
+        # Finding all matches including the numbers
+        matches_with_numbers = re.findall(pattern, prompt[-1]['content'])
+        if not matches_with_numbers:
+            hard_coded_menu_ele_numbers = []
+        else:
+            hard_coded_menu_ele_numbers = [match[0] for match in matches_with_numbers]
+
         lm_config = self.lm_config
         n = 0
         while True:
@@ -151,7 +160,7 @@ def next_action(
                     action["raw_prediction"] = response
                     break
 
-        return action
+        return (action, hard_coded_menu_ele_numbers)
 
     def reset(self, test_config_file: str) -> None:
         pass

config_files/test.raw.json

@@ -7619,9 +7619,9 @@
     "geolocation": null,
     "intent_template": "I am doing a market survey for one stop market, show me the most expensive product from {{product_category}} category",
     "instantiation_dict": {
-      "product_category": "competative swimwear"
+      "product_category": "competitive swimwear"
     },
-    "intent": "I am doing a market survey for one stop market, show me the most expensive product from competative swimwear category",
+    "intent": "I am doing a market survey for one stop market, show me the most expensive product from competitive swimwear category",
     "require_reset": false,
     "eval": {
       "eval_types": [
@@ -10963,10 +10963,10 @@
     "geolocation": null,
     "intent_template": "List products from {{product_category}} category by {{order}} price",
     "instantiation_dict": {
-      "product_category": "competative swimwear",
+      "product_category": "competitive swimwear",
       "order": "ascending"
     },
-    "intent": "List products from competative swimwear category by ascending price",
+    "intent": "List products from competitive swimwear category by ascending price",
     "require_reset": false,
     "eval": {
       "eval_types": [

run.py

@@ -293,9 +293,14 @@ def test(
                     action = create_stop_action(f"Early stop: {stop_info}")
                 else:
                     try:
-                        action = agent.next_action(
+                        action, hard_coded_menu_ele_numbers = agent.next_action(
                             trajectory, intent, meta_data=meta_data
                         )
+                        if action['element_id'] in hard_coded_menu_ele_numbers:
+                            if action['element_id'] == hard_coded_menu_ele_numbers[0]:
+                                action = {'action_type': ActionTypes.GOTO_URL, 'element_id': action['element_id'], 'url': env.page.url + '&product_list_order=name', 'raw_prediction': 'go to URL'}
+                            if action['element_id'] == hard_coded_menu_ele_numbers[1]:
+                                action = {'action_type': ActionTypes.GOTO_URL, 'element_id': action['element_id'], 'url': env.page.url + '&product_list_order=price', 'raw_prediction': 'go to URL'}
                     except ValueError as e:
                         # get the error message
                         action = create_stop_action(f"ERROR: {str(e)}")