Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/workflow retry #11999

Merged
merged 4 commits into from
Dec 23, 2024
Merged

Fix/workflow retry #11999

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dbb05c7
fix: workflow-retry log
zxhlyh Dec 23, 2024
d0807a8
fix: workflow-retry log
zxhlyh Dec 23, 2024
db1f1d8
fix: workflow-retry log
zxhlyh Dec 23, 2024
95b254b
fix: workflow-retry log
zxhlyh Dec 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 61 additions & 10 deletions web/app/components/workflow/hooks/use-workflow-run.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,6 +387,9 @@ export const useWorkflowRun = () => {
if (nodeIndex !== -1) {
currIteration[nodeIndex] = {
...currIteration[nodeIndex],
...(currIteration[nodeIndex].retryDetail
? { retryDetail: currIteration[nodeIndex].retryDetail }
: {}),
...data,
} as any
}
Expand Down Expand Up @@ -626,26 +629,74 @@ export const useWorkflowRun = () => {
const {
workflowRunningData,
setWorkflowRunningData,
iterParallelLogMap,
setIterParallelLogMap,
} = workflowStore.getState()
const {
getNodes,
setNodes,
} = store.getState()

const nodes = getNodes()
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const currentRetryNodeIndex = tracing.findIndex(trace => trace.node_id === data.node_id)
const currentNode = nodes.find(node => node.id === data.node_id)!
const nodeParent = nodes.find(node => node.id === currentNode.parentId)
if (nodeParent) {
if (!data.execution_metadata.parallel_mode_run_id) {
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const iteration = tracing.find(trace => trace.node_id === nodeParent.id)

if (currentRetryNodeIndex > -1) {
const currentRetryNode = tracing[currentRetryNodeIndex]
if (currentRetryNode.retryDetail)
draft.tracing![currentRetryNodeIndex].retryDetail!.push(data as NodeTracing)
if (iteration && iteration.details?.length) {
const currentNodeRetry = iteration.details[nodeParent.data._iterationIndex - 1]?.find(item => item.node_id === data.node_id)

else
draft.tracing![currentRetryNodeIndex].retryDetail = [data as NodeTracing]
if (currentNodeRetry) {
if (currentNodeRetry?.retryDetail)
currentNodeRetry?.retryDetail.push(data as NodeTracing)
else
currentNodeRetry.retryDetail = [data as NodeTracing]
}
}
}))
}
}))
else {
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const iteration = tracing.find(trace => trace.node_id === nodeParent.id)

if (iteration && iteration.details?.length) {
const iterRunID = data.execution_metadata?.parallel_mode_run_id

const currIteration = iterParallelLogMap.get(iteration.node_id)?.get(iterRunID)
const currentNodeRetry = currIteration?.find(item => item.node_id === data.node_id)

if (currentNodeRetry) {
if (currentNodeRetry?.retryDetail)
currentNodeRetry?.retryDetail.push(data as NodeTracing)
else
currentNodeRetry.retryDetail = [data as NodeTracing]
}
setIterParallelLogMap(iterParallelLogMap)
const iterLogMap = iterParallelLogMap.get(iteration.node_id)
if (iterLogMap)
iteration.details = Array.from(iterLogMap.values())
}
}))
}
}
else {
setWorkflowRunningData(produce(workflowRunningData!, (draft) => {
const tracing = draft.tracing!
const currentRetryNodeIndex = tracing.findIndex(trace => trace.node_id === data.node_id)

if (currentRetryNodeIndex > -1) {
const currentRetryNode = tracing[currentRetryNodeIndex]
if (currentRetryNode.retryDetail)
draft.tracing![currentRetryNodeIndex].retryDetail!.push(data as NodeTracing)
else
draft.tracing![currentRetryNodeIndex].retryDetail = [data as NodeTracing]
}
}))
}
const newNodes = produce(nodes, (draft) => {
const currentNode = draft.find(node => node.id === data.node_id)!

Expand Down
4 changes: 2 additions & 2 deletions web/app/components/workflow/nodes/_base/components/retry/retry-on-node.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ const RetryOnNode = ({
}, [data._runningStatus, showSelectedBorder])
const showDefault = !isRunning && !isSuccessful && !isException && !isFailed

if (!retry_config)
if (!retry_config?.retry_enabled)
return null

return (
Expand Down Expand Up @@ -74,7 +74,7 @@ const RetryOnNode = ({
}
</div>
{
!showDefault && (
!showDefault && !!data._retryIndex && (
<div>
{data._retryIndex}/{data.retry_config?.max_retries}
</div>
Expand Down
39 changes: 33 additions & 6 deletions web/app/components/workflow/run/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,24 @@ const RunPanel: FC<RunProps> = ({ hideResult, activeTab = 'RESULT', runID, getRe

const groupMap = nodeGroupMap.get(iterationNode.node_id)!

if (!groupMap.has(runId))
if (!groupMap.has(runId)) {
groupMap.set(runId, [item])
}
else {
if (item.status === 'retry') {
const retryNode = groupMap.get(runId)!.find(node => node.node_id === item.node_id)

else
groupMap.get(runId)!.push(item)
if (retryNode) {
if (retryNode?.retryDetail)
retryNode.retryDetail.push(item)
else
retryNode.retryDetail = [item]
}
}
else {
groupMap.get(runId)!.push(item)
}
}

if (item.status === 'failed') {
iterationNode.status = 'failed'
Expand All @@ -94,10 +107,24 @@ const RunPanel: FC<RunProps> = ({ hideResult, activeTab = 'RESULT', runID, getRe
const updateSequentialModeGroup = (index: number, item: NodeTracing, iterationNode: NodeTracing) => {
const { details } = iterationNode
if (details) {
if (!details[index])
if (!details[index]) {
details[index] = [item]
else
details[index].push(item)
}
else {
if (item.status === 'retry') {
const retryNode = details[index].find(node => node.node_id === item.node_id)

if (retryNode) {
if (retryNode?.retryDetail)
retryNode.retryDetail.push(item)
else
retryNode.retryDetail = [item]
}
}
else {
details[index].push(item)
}
}
}

if (item.status === 'failed') {
Expand Down
40 changes: 29 additions & 11 deletions web/app/components/workflow/run/iteration-result-panel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
import { ArrowNarrowLeft } from '../../base/icons/src/vender/line/arrows'
import { NodeRunningStatus } from '../types'
import TracingPanel from './tracing-panel'
import RetryResultPanel from './retry-result-panel'
import { Iteration } from '@/app/components/base/icons/src/vender/workflow'
import cn from '@/utils/classnames'
import type { IterationDurationMap, NodeTracing } from '@/types/workflow'
Expand Down Expand Up @@ -41,8 +42,8 @@ const IterationResultPanel: FC<Props> = ({
}))
}, [])
const countIterDuration = (iteration: NodeTracing[], iterDurationMap: IterationDurationMap): string => {
const IterRunIndex = iteration[0].execution_metadata.iteration_index as number
const iterRunId = iteration[0].execution_metadata.parallel_mode_run_id
const IterRunIndex = iteration[0]?.execution_metadata?.iteration_index as number
const iterRunId = iteration[0]?.execution_metadata?.parallel_mode_run_id
const iterItem = iterDurationMap[iterRunId || IterRunIndex]
const duration = iterItem
return `${(duration && duration > 0.01) ? duration.toFixed(2) : 0.01}s`
Expand Down Expand Up @@ -74,6 +75,10 @@ const IterationResultPanel: FC<Props> = ({
</>
)
}
const [retryRunResult, setRetryRunResult] = useState<Record<string, NodeTracing[]> | undefined>()
const handleRetryDetail = (v: number, detail?: NodeTracing[]) => {
setRetryRunResult({ ...retryRunResult, [v]: detail })
}

const main = (
<>
Expand Down Expand Up @@ -116,15 +121,28 @@ const IterationResultPanel: FC<Props> = ({
{expandedIterations[index] && <div
className="flex-grow h-px bg-divider-subtle"
></div>}
<div className={cn(
'overflow-hidden transition-all duration-200',
expandedIterations[index] ? 'max-h-[1000px] opacity-100' : 'max-h-0 opacity-0',
)}>
<TracingPanel
list={iteration}
className='bg-background-section-burn'
/>
</div>
{
!retryRunResult?.[index] && (
<div className={cn(
'overflow-hidden transition-all duration-200',
expandedIterations[index] ? 'max-h-[1000px] opacity-100' : 'max-h-0 opacity-0',
)}>
<TracingPanel
list={iteration}
className='bg-background-section-burn'
onShowRetryDetail={v => handleRetryDetail(index, v)}
/>
</div>
)
}
{
retryRunResult?.[index] && (
<RetryResultPanel
list={retryRunResult[index]}
onBack={() => handleRetryDetail(index, undefined)}
/>
)
}
</div>
))}
</div>
Expand Down
5 changes: 5 additions & 0 deletions web/app/components/workflow/run/node.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,11 @@ const NodePanel: FC<Props> = ({
{nodeInfo.error}
</StatusContainer>
)}
{nodeInfo.status === 'retry' && (
<StatusContainer status='failed'>
{nodeInfo.error}
</StatusContainer>
)}
</div>
{nodeInfo.inputs && (
<div className={cn('mb-1')}>
Expand Down
Loading