Merge branch 'feat/firecrawl-fe' into deploy/dev

langgenius · Jun 7, 2024 · 6c238d8 · 6c238d8
2 parents fb7842e + 6e86c73
commit 6c238d8
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 13 deletions.
diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
@@ -110,7 +110,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
         <StepsNavBar step={step} datasetId={datasetId} />
       </div>
       <div className="grow bg-white">
-        <div className={step === 1 ? 'block h-full' : 'none'}>
+        <div className={step === 1 ? 'block h-full' : 'hidden'}>
           <StepOne
             hasConnection={hasConnection}
             onSetting={() => setShowAccountSettingModal({ payload: 'data-source' })}

diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -96,27 +96,42 @@ const FireCrawl: FC<Props> = ({
   const [crawlHasError, setCrawlHasError] = useState(false)
 
   const waitForCrawlFinished = useCallback(async (jobId: string) => {
-    const res = await checkFirecrawlTaskStatus(jobId) as any
-    if (res.status === 'completed') {
-      return {
-        isError: false,
-        data: {
-          ...res,
-          total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
-        },
+    try {
+      const res = await checkFirecrawlTaskStatus(jobId) as any
+      if (res.status === 'completed') {
+        return {
+          isError: false,
+          data: {
+            ...res,
+            total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
+          },
+        }
       }
+      if (res.status === 'error') {
+        // can't get the error message from the firecrawl api
+        return {
+          isError: true,
+          data: {
+            data: [],
+          },
+        }
+      }
+      // update the progress
+      setCrawlResult({
+        ...res,
+        total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
+      })
+      await sleep(2500)
+      return await waitForCrawlFinished(jobId)
     }
-    if (res.status === 'error') {
-      // can't get the error message from the firecrawl api
+    catch (e) {
       return {
         isError: true,
         data: {
           data: [],
         },
       }
     }
-    await sleep(2500)
-    return await waitForCrawlFinished(jobId)
   }, [crawlOptions.limit])
 
   const handleRun = useCallback(async (url: string) => {