Skip to content

Commit

Permalink
Fix/workflow retry (#11999)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zxhlyh
zxhlyh authored Dec 23, 2024
1 parent e068bbe commit ef95b12
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 29 deletions.
71 changes: 61 additions & 10 deletions web/app/components/workflow/hooks/use-workflow-run.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,9 @@ export const useWorkflowRun = () => {
if (nodeIndex !== -1) {
currIteration[nodeIndex] = {
...currIteration[nodeIndex],
...(currIteration[nodeIndex].retryDetail
? { retryDetail: currIteration[nodeIndex].retryDetail }
: {}),
...data,
} as any
}
Expand Down Expand Up @@ -626,26 +629,74 @@ export const useWorkflowRun = () => {
const {
workflowRunningData,
setWorkflowRunningData,
iterParallelLogMap,
setIterParallelLogMap,
} = workflowStore.getState()
const {
getNodes,
setNodes,
} = store.getState()

const nodes = getNodes()
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const currentRetryNodeIndex = tracing.findIndex(trace => trace.node_id === data.node_id)
const currentNode = nodes.find(node => node.id === data.node_id)!
const nodeParent = nodes.find(node => node.id === currentNode.parentId)
if (nodeParent) {
if (!data.execution_metadata.parallel_mode_run_id) {
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const iteration = tracing.find(trace => trace.node_id === nodeParent.id)

if (currentRetryNodeIndex > -1) {
const currentRetryNode = tracing[currentRetryNodeIndex]
if (currentRetryNode.retryDetail)
draft.tracing![currentRetryNodeIndex].retryDetail!.push(data as NodeTracing)
if (iteration && iteration.details?.length) {
const currentNodeRetry = iteration.details[nodeParent.data._iterationIndex - 1]?.find(item => item.node_id === data.node_id)

else
draft.tracing![currentRetryNodeIndex].retryDetail = [data as NodeTracing]
if (currentNodeRetry) {
if (currentNodeRetry?.retryDetail)
currentNodeRetry?.retryDetail.push(data as NodeTracing)
else
currentNodeRetry.retryDetail = [data as NodeTracing]
}
}
}))
}
}))
else {
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const iteration = tracing.find(trace => trace.node_id === nodeParent.id)

if (iteration && iteration.details?.length) {
const iterRunID = data.execution_metadata?.parallel_mode_run_id

const currIteration = iterParallelLogMap.get(iteration.node_id)?.get(iterRunID)
const currentNodeRetry = currIteration?.find(item => item.node_id === data.node_id)

if (currentNodeRetry) {
if (currentNodeRetry?.retryDetail)
currentNodeRetry?.retryDetail.push(data as NodeTracing)
else
currentNodeRetry.retryDetail = [data as NodeTracing]
}
setIterParallelLogMap(iterParallelLogMap)
const iterLogMap = iterParallelLogMap.get(iteration.node_id)
if (iterLogMap)
iteration.details = Array.from(iterLogMap.values())
}
}))
}
}
else {
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const currentRetryNodeIndex = tracing.findIndex(trace => trace.node_id === data.node_id)

if (currentRetryNodeIndex > -1) {
const currentRetryNode = tracing[currentRetryNodeIndex]
if (currentRetryNode.retryDetail)
draft.tracing![currentRetryNodeIndex].retryDetail!.push(data as NodeTracing)
else
draft.tracing![currentRetryNodeIndex].retryDetail = [data as NodeTracing]
}
}))
}
const newNodes = produce(nodes, (draft) => {
const currentNode = draft.find(node => node.id === data.node_id)!

Expand Down
4 changes: 2 additions & 2 deletions web/app/components/workflow/nodes/_base/components/retry/retry-on-node.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const RetryOnNode = ({
}, [data._runningStatus, showSelectedBorder])
const showDefault = !isRunning && !isSuccessful && !isException && !isFailed

if (!retry_config)
if (!retry_config?.retry_enabled)
return null

return (
Expand Down Expand Up @@ -74,7 +74,7 @@ const RetryOnNode = ({
}
</div>
{
!showDefault && (
!showDefault && !!data._retryIndex && (
<div>
{data._retryIndex}/{data.retry_config?.max_retries}
</div>
Expand Down
39 changes: 33 additions & 6 deletions web/app/components/workflow/run/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,24 @@ const RunPanel: FC<RunProps> = ({ hideResult, activeTab = 'RESULT', runID, getRe

const groupMap = nodeGroupMap.get(iterationNode.node_id)!

if (!groupMap.has(runId))
if (!groupMap.has(runId)) {
groupMap.set(runId, [item])
}
else {
if (item.status === 'retry') {
const retryNode = groupMap.get(runId)!.find(node => node.node_id === item.node_id)

else
groupMap.get(runId)!.push(item)
if (retryNode) {
if (retryNode?.retryDetail)
retryNode.retryDetail.push(item)
else
retryNode.retryDetail = [item]
}
}
else {
groupMap.get(runId)!.push(item)
}
}

if (item.status === 'failed') {
iterationNode.status = 'failed'
Expand All @@ -94,10 +107,24 @@ const RunPanel: FC<RunProps> = ({ hideResult, activeTab = 'RESULT', runID, getRe
const updateSequentialModeGroup = (index: number, item: NodeTracing, iterationNode: NodeTracing) => {
const { details } = iterationNode
if (details) {
if (!details[index])
if (!details[index]) {
details[index] = [item]
else
details[index].push(item)
}
else {
if (item.status === 'retry') {
const retryNode = details[index].find(node => node.node_id === item.node_id)

if (retryNode) {
if (retryNode?.retryDetail)
retryNode.retryDetail.push(item)
else
retryNode.retryDetail = [item]
}
}
else {
details[index].push(item)
}
}
}

if (item.status === 'failed') {
Expand Down
40 changes: 29 additions & 11 deletions web/app/components/workflow/run/iteration-result-panel.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
import { ArrowNarrowLeft } from '../../base/icons/src/vender/line/arrows'
import { NodeRunningStatus } from '../types'
import TracingPanel from './tracing-panel'
import RetryResultPanel from './retry-result-panel'
import { Iteration } from '@/app/components/base/icons/src/vender/workflow'
import cn from '@/utils/classnames'
import type { IterationDurationMap, NodeTracing } from '@/types/workflow'
Expand Down Expand Up @@ -41,8 +42,8 @@ const IterationResultPanel: FC<Props> = ({
}))
}, [])
const countIterDuration = (iteration: NodeTracing[], iterDurationMap: IterationDurationMap): string => {
const IterRunIndex = iteration[0].execution_metadata.iteration_index as number
const iterRunId = iteration[0].execution_metadata.parallel_mode_run_id
const IterRunIndex = iteration[0]?.execution_metadata?.iteration_index as number
const iterRunId = iteration[0]?.execution_metadata?.parallel_mode_run_id
const iterItem = iterDurationMap[iterRunId || IterRunIndex]
const duration = iterItem
return `${(duration && duration > 0.01) ? duration.toFixed(2) : 0.01}s`
Expand Down Expand Up @@ -74,6 +75,10 @@ const IterationResultPanel: FC<Props> = ({
</>
)
}
const [retryRunResult, setRetryRunResult] = useState<Record<string, NodeTracing[]> | undefined>()
const handleRetryDetail = (v: number, detail?: NodeTracing[]) => {
setRetryRunResult({ ...retryRunResult, [v]: detail })
}

const main = (
<>
Expand Down Expand Up @@ -116,15 +121,28 @@ const IterationResultPanel: FC<Props> = ({
{expandedIterations[index] && <div
className="flex-grow h-px bg-divider-subtle"
></div>}
<div className={cn(
'overflow-hidden transition-all duration-200',
expandedIterations[index] ? 'max-h-[1000px] opacity-100' : 'max-h-0 opacity-0',
)}>
<TracingPanel
list={iteration}
className='bg-background-section-burn'
/>
</div>
{
!retryRunResult?.[index] && (
<div className={cn(
'overflow-hidden transition-all duration-200',
expandedIterations[index] ? 'max-h-[1000px] opacity-100' : 'max-h-0 opacity-0',
)}>
<TracingPanel
list={iteration}
className='bg-background-section-burn'
onShowRetryDetail={v => handleRetryDetail(index, v)}
/>
</div>
)
}
{
retryRunResult?.[index] && (
<RetryResultPanel
list={retryRunResult[index]}
onBack={() => handleRetryDetail(index, undefined)}
/>
)
}
</div>
))}
</div>
Expand Down
5 changes: 5 additions & 0 deletions web/app/components/workflow/run/node.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,11 @@ const NodePanel: FC<Props> = ({
{nodeInfo.error}
</StatusContainer>
)}
{nodeInfo.status === 'retry' && (
<StatusContainer status='failed'>
{nodeInfo.error}
</StatusContainer>
)}
</div>
{nodeInfo.inputs && (
<div className={cn('mb-1')}>
Expand Down

0 comments on commit ef95b12

Please sign in to comment.