feat(app): implement stall recoveries (#17002)

Adds recovery from stall or collision errors to the app.

This should implement these flows:
https://www.figma.com/design/OGssKRmCOvuXSqUpK2qXrV/Feature%3A-Error-Recovery-November-Release?node-id=9765-66609&t=65NkMVGZlPdCG7z6-4

When there's a stall, which can happen on pretty much any command, we
should now prompt the user to home and retry. To home, they have to make
sure the machine is safe, so we will go through DTWiz.

## Reviews
- [ ] did i miss anything

## Testing
- [x] stalls should get you the home and retry button
- [x] you should enter the DTWiz if >0 pipettes have a tip
- [x] if 0 pipettes have a tip the DTwiz should be skipped (or if there
are no pipettes)
- [x] Retrying should in fact work

Closes EXEC-725

api/src/opentrons/hardware_control/api.py

@@ -921,6 +921,16 @@ def engaged_axes(self) -> Dict[Axis, bool]:
     async def disengage_axes(self, which: List[Axis]) -> None:
         await self._backend.disengage_axes([ot2_axis_to_string(ax) for ax in which])
 
+    def axis_is_present(self, axis: Axis) -> bool:
+        is_ot2 = axis in Axis.ot2_axes()
+        if not is_ot2:
+            return False
+        if axis in Axis.pipette_axes():
+            mount = Axis.to_ot2_mount(axis)
+            if self.attached_pipettes.get(mount) is None:
+                return False
+        return True
+
     @ExecutionManagerProvider.wait_for_running
     async def _fast_home(self, axes: Sequence[str], margin: float) -> Dict[str, float]:
         converted_axes = "".join(axes)

api/src/opentrons/hardware_control/ot3api.py

@@ -1674,7 +1674,12 @@ async def disengage_axes(self, which: List[Axis]) -> None:
         await self._backend.disengage_axes(which)
 
     async def engage_axes(self, which: List[Axis]) -> None:
-        await self._backend.engage_axes(which)
+        await self._backend.engage_axes(
+            [axis for axis in which if self._backend.axis_is_present(axis)]
+        )
+
+    def axis_is_present(self, axis: Axis) -> bool:
+        return self._backend.axis_is_present(axis)
 
     async def get_limit_switches(self) -> Dict[Axis, bool]:
         res = await self._backend.get_limit_switches()

api/src/opentrons/hardware_control/protocols/hardware_manager.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 from typing_extensions import Protocol
 
-from ..types import SubSystem, SubSystemState
+from ..types import SubSystem, SubSystemState, Axis
 
 
 class HardwareManager(Protocol):
@@ -45,3 +45,7 @@ def attached_subsystems(self) -> Dict[SubSystem, SubSystemState]:
     async def get_serial_number(self) -> Optional[str]:
         """Get the robot serial number, if provisioned. If not provisioned, will be None."""
         ...
+
+    def axis_is_present(self, axis: Axis) -> bool:
+        """Get whether a motor axis is present on the machine."""
+        ...

api/src/opentrons/protocol_engine/commands/unsafe/unsafe_engage_axes.py

@@ -52,10 +52,7 @@ async def execute(
         """Enable exes."""
         ot3_hardware_api = ensure_ot3_hardware(self._hardware_api)
         await ot3_hardware_api.engage_axes(
-            [
-                self._gantry_mover.motor_axis_to_hardware_axis(axis)
-                for axis in params.axes
-            ]
+            self._gantry_mover.motor_axes_to_present_hardware_axes(params.axes)
         )
         return SuccessData(
             public=UnsafeEngageAxesResult(),

api/src/opentrons/protocol_engine/commands/unsafe/update_position_estimators.py

@@ -58,10 +58,7 @@ async def execute(
         """Update axis position estimators from their encoders."""
         ot3_hardware_api = ensure_ot3_hardware(self._hardware_api)
         await ot3_hardware_api.update_axis_position_estimations(
-            [
-                self._gantry_mover.motor_axis_to_hardware_axis(axis)
-                for axis in params.axes
-            ]
+            self._gantry_mover.motor_axes_to_present_hardware_axes(params.axes)
         )
         return SuccessData(
             public=UpdatePositionEstimatorsResult(),

api/src/opentrons/protocol_engine/execution/gantry_mover.py

@@ -159,6 +159,12 @@ def pick_mount_from_axis_map(self, axis_map: Dict[MotorAxis, float]) -> Mount:
         """Find a mount axis in the axis_map if it exists otherwise default to left mount."""
         ...
 
+    def motor_axes_to_present_hardware_axes(
+        self, motor_axes: List[MotorAxis]
+    ) -> List[HardwareAxis]:
+        """Transform a list of engine axes into a list of hardware axes, filtering out non-present axes."""
+        ...
+
 
 class HardwareGantryMover(GantryMover):
     """Hardware API based gantry movement handler."""
@@ -167,6 +173,18 @@ def __init__(self, hardware_api: HardwareControlAPI, state_view: StateView) -> N
         self._hardware_api = hardware_api
         self._state_view = state_view
 
+    def motor_axes_to_present_hardware_axes(
+        self, motor_axes: List[MotorAxis]
+    ) -> List[HardwareAxis]:
+        """Get hardware axes from engine axes while filtering out non-present axes."""
+        return [
+            self.motor_axis_to_hardware_axis(motor_axis)
+            for motor_axis in motor_axes
+            if self._hardware_api.axis_is_present(
+                self.motor_axis_to_hardware_axis(motor_axis)
+            )
+        ]
+
     def motor_axis_to_hardware_axis(self, motor_axis: MotorAxis) -> HardwareAxis:
         """Transform an engine motor axis into a hardware axis."""
         return _MOTOR_AXIS_TO_HARDWARE_AXIS[motor_axis]
@@ -643,6 +661,14 @@ async def prepare_for_mount_movement(self, mount: Mount) -> None:
         """Retract the 'idle' mount if necessary."""
         pass
 
+    def motor_axes_to_present_hardware_axes(
+        self, motor_axes: List[MotorAxis]
+    ) -> List[HardwareAxis]:
+        """Get present hardware axes from a list of engine axes. In simulation, all axes are present."""
+        return [
+            self.motor_axis_to_hardware_axis(motor_axis) for motor_axis in motor_axes
+        ]
+
 
 def create_gantry_mover(
     state_view: StateView, hardware_api: HardwareControlAPI

api/tests/opentrons/protocol_engine/commands/unsafe/test_engage_axes.py

@@ -22,21 +22,28 @@ async def test_engage_axes_implementation(
     )
 
     data = UnsafeEngageAxesParams(
-        axes=[MotorAxis.LEFT_Z, MotorAxis.LEFT_PLUNGER, MotorAxis.X, MotorAxis.Y]
-    )
-
-    decoy.when(gantry_mover.motor_axis_to_hardware_axis(MotorAxis.LEFT_Z)).then_return(
-        Axis.Z_L
+        axes=[
+            MotorAxis.LEFT_Z,
+            MotorAxis.LEFT_PLUNGER,
+            MotorAxis.X,
+            MotorAxis.Y,
+            MotorAxis.RIGHT_Z,
+            MotorAxis.RIGHT_PLUNGER,
+        ]
     )
     decoy.when(
-        gantry_mover.motor_axis_to_hardware_axis(MotorAxis.LEFT_PLUNGER)
-    ).then_return(Axis.P_L)
-    decoy.when(gantry_mover.motor_axis_to_hardware_axis(MotorAxis.X)).then_return(
-        Axis.X
-    )
-    decoy.when(gantry_mover.motor_axis_to_hardware_axis(MotorAxis.Y)).then_return(
-        Axis.Y
-    )
+        gantry_mover.motor_axes_to_present_hardware_axes(
+            [
+                MotorAxis.LEFT_Z,
+                MotorAxis.LEFT_PLUNGER,
+                MotorAxis.X,
+                MotorAxis.Y,
+                MotorAxis.RIGHT_Z,
+                MotorAxis.RIGHT_PLUNGER,
+            ]
+        )
+    ).then_return([Axis.Z_L, Axis.P_L, Axis.X, Axis.Y])
+
     decoy.when(
         await ot3_hardware_api.update_axis_position_estimations(
             [Axis.Z_L, Axis.P_L, Axis.X, Axis.Y]

api/tests/opentrons/protocol_engine/commands/unsafe/test_update_position_estimators.py

@@ -22,21 +22,27 @@ async def test_update_position_estimators_implementation(
     )
 
     data = UpdatePositionEstimatorsParams(
-        axes=[MotorAxis.LEFT_Z, MotorAxis.LEFT_PLUNGER, MotorAxis.X, MotorAxis.Y]
-    )
-
-    decoy.when(gantry_mover.motor_axis_to_hardware_axis(MotorAxis.LEFT_Z)).then_return(
-        Axis.Z_L
+        axes=[
+            MotorAxis.LEFT_Z,
+            MotorAxis.LEFT_PLUNGER,
+            MotorAxis.X,
+            MotorAxis.Y,
+            MotorAxis.RIGHT_Z,
+            MotorAxis.RIGHT_PLUNGER,
+        ]
     )
     decoy.when(
-        gantry_mover.motor_axis_to_hardware_axis(MotorAxis.LEFT_PLUNGER)
-    ).then_return(Axis.P_L)
-    decoy.when(gantry_mover.motor_axis_to_hardware_axis(MotorAxis.X)).then_return(
-        Axis.X
-    )
-    decoy.when(gantry_mover.motor_axis_to_hardware_axis(MotorAxis.Y)).then_return(
-        Axis.Y
-    )
+        gantry_mover.motor_axes_to_present_hardware_axes(
+            [
+                MotorAxis.LEFT_Z,
+                MotorAxis.LEFT_PLUNGER,
+                MotorAxis.X,
+                MotorAxis.Y,
+                MotorAxis.RIGHT_Z,
+                MotorAxis.RIGHT_PLUNGER,
+            ]
+        )
+    ).then_return([Axis.Z_L, Axis.P_L, Axis.X, Axis.Y])
 
     result = await subject.execute(data)
 

app/src/assets/localization/en/error_recovery.json

@@ -8,6 +8,7 @@
   "blowout_failed": "Blowout failed",
   "cancel_run": "Cancel run",
   "canceling_run": "Canceling run",
+  "carefully_move_labware": "<block>Carefully move any misplaced labware and clean up any spilled liquid.</block><block>Close the robot door before proceeding.</block>",
   "change_location": "Change location",
   "change_tip_pickup_location": "Change tip pick-up location",
   "choose_a_recovery_action": "Choose a recovery action",
@@ -32,6 +33,9 @@
   "gripper_errors_occur_when": "Gripper errors occur when the gripper stalls or collides with another object on the deck and are usually caused by improperly placed labware or inaccurate labware offsets",
   "gripper_releasing_labware": "Gripper releasing labware",
   "gripper_will_release_in_s": "Gripper will release labware in {{seconds}} seconds",
+  "home_and_retry": "Home gantry and retry step",
+  "home_gantry": "Home gantry",
+  "home_now": "Home now",
   "homing_pipette_dangerous": "Homing the <bold>{{mount}} pipette</bold> with liquid in the tips may damage it. You must remove all tips before using the pipette again.",
   "if_issue_persists_gripper_error": " If the issue persists, cancel the run and rerun gripper calibration",
   "if_issue_persists_overpressure": " If the issue persists, cancel the run and make the necessary changes to the protocol",
@@ -57,7 +61,9 @@
   "overpressure_is_usually_caused": "Overpressure is usually caused by a tip contacting labware, a clog, or moving viscous liquid too quickly",
   "pick_up_tips": "Pick up tips",
   "pipette_overpressure": "Pipette overpressure",
+  "prepare_deck_for_homing": "Prepare deck for homing",
   "proceed_to_cancel": "Proceed to cancel",
+  "proceed_to_home": "Proceed to home",
   "proceed_to_tip_selection": "Proceed to tip selection",
   "recovery_action_failed": "{{action}} failed",
   "recovery_mode": "Recovery mode",
@@ -96,6 +102,8 @@
   "skip_to_next_step_same_tips": "Skip to next step with same tips",
   "skipping_to_step_succeeded": "Skipping to step {{step}} succeeded.",
   "skipping_to_step_succeeded_na": "Skipping to next step succeeded.",
+  "stall_or_collision_detected_when": "A stall or collision is detected when the robot's motors are blocked",
+  "stall_or_collision_error": "Stall or collision",
   "stand_back": "Stand back, robot is in motion",
   "stand_back_picking_up_tips": "Stand back, picking up tips",
   "stand_back_resuming": "Stand back, resuming current step",
@@ -105,7 +113,9 @@
   "take_necessary_actions": "<block>First, take any necessary actions to prepare the robot to retry the failed step.</block><block>Then, close the robot door before proceeding.</block>",
   "take_necessary_actions_failed_pickup": "<block>First, take any necessary actions to prepare the robot to retry the failed tip pickup.</block><block>Then, close the robot door before proceeding.</block>",
   "take_necessary_actions_failed_tip_drop": "<block>First, take any necessary actions to prepare the robot to retry the failed tip drop.</block><block>Then, close the robot door before proceeding.</block>",
+  "take_necessary_actions_home": "<block>Take any necessary actions to prepare the robot to move the gantry to its home position.</block><block>Close the robot door before proceeding.</block>",
   "terminate_remote_activity": "Terminate remote activity",
+  "the_robot_must_return_to_home_position": "The robot must return to its home position before proceeding",
   "tip_drop_failed": "Tip drop failed",
   "tip_not_detected": "Tip not detected",
   "tip_presence_errors_are_caused": "Tip presence errors are usually caused by improperly placed labware or inaccurate labware offsets",

app/src/organisms/ErrorRecoveryFlows/ErrorRecoveryWizard.tsx

@@ -19,6 +19,7 @@ import {
   IgnoreErrorSkipStep,
   ManualMoveLwAndSkip,
   ManualReplaceLwAndRetry,
+  HomeAndRetry,
 } from './RecoveryOptions'
 import {
   useErrorDetailsModal,
@@ -225,6 +226,10 @@ export function ErrorRecoveryContent(props: RecoveryContentProps): JSX.Element {
     return <RecoveryDoorOpenSpecial {...props} />
   }
 
+  const buildHomeAndRetry = (): JSX.Element => {
+    return <HomeAndRetry {...props} />
+  }
+
   switch (props.recoveryMap.route) {
     case RECOVERY_MAP.OPTION_SELECTION.ROUTE:
       return buildSelectRecoveryOption()
@@ -264,6 +269,8 @@ export function ErrorRecoveryContent(props: RecoveryContentProps): JSX.Element {
       return buildRecoveryInProgress()
     case RECOVERY_MAP.ROBOT_DOOR_OPEN.ROUTE:
       return buildManuallyRouteToDoorOpen()
+    case RECOVERY_MAP.HOME_AND_RETRY.ROUTE:
+      return buildHomeAndRetry()
     default:
       return buildSelectRecoveryOption()
   }

app/src/organisms/ErrorRecoveryFlows/RecoveryOptions/CancelRun.tsx

@@ -35,7 +35,9 @@ export function CancelRun(props: RecoveryContentProps): JSX.Element {
       case CANCEL_RUN.STEPS.CONFIRM_CANCEL:
         return <CancelRunConfirmation {...props} />
       default:
-        console.warn(`${step} in ${route} not explicitly handled. Rerouting.`)
+        console.warn(
+          `CancelRun: ${step} in ${route} not explicitly handled. Rerouting.`
+        )
         return <SelectRecoveryOption {...props} />
     }
   }

app/src/organisms/ErrorRecoveryFlows/RecoveryOptions/FillWellAndSkip.tsx

@@ -34,7 +34,9 @@ export function FillWellAndSkip(props: RecoveryContentProps): JSX.Element {
       case CANCEL_RUN.STEPS.CONFIRM_CANCEL:
         return <CancelRun {...props} />
       default:
-        console.warn(`${step} in ${route} not explicitly handled. Rerouting.`)
+        console.warn(
+          `FillWellAndSkip: ${step} in ${route} not explicitly handled. Rerouting.`
+        )
         return <SelectRecoveryOption {...props} />
     }
   }