From 4b58b549c64c1376b07ecfbb55a4b623fb953257 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 31 May 2024 14:44:26 +0800
Subject: [PATCH 01/31] chore: notion source to common component

---
 .../data-source-notion/index.tsx              | 149 ++++++------------
 .../data-source-notion/operate/index.tsx      |  14 +-
 .../data-source-page/panel/config-item.tsx    |  78 +++++++++
 .../data-source-page/panel/index.tsx          | 113 +++++++++++++
 .../style.module.css                          |   0
 .../data-source-page/panel/types.ts           |   4 +
 6 files changed, 251 insertions(+), 107 deletions(-)
 create mode 100644 web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
 create mode 100644 web/app/components/header/account-setting/data-source-page/panel/index.tsx
 rename web/app/components/header/account-setting/data-source-page/{data-source-notion => panel}/style.module.css (100%)
 create mode 100644 web/app/components/header/account-setting/data-source-page/panel/types.ts

diff --git a/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx
index 8f72421451eae5..4929b2b403fcb3 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx
@@ -1,23 +1,34 @@
-import { useEffect, useState } from 'react'
+'use client'
+import type { FC } from 'react'
+import React, { useEffect, useState } from 'react'
 import useSWR from 'swr'
-import { useTranslation } from 'react-i18next'
-import { PlusIcon } from '@heroicons/react/24/solid'
-import cn from 'classnames'
-import Indicator from '../../../indicator'
-import Operate from './operate'
-import s from './style.module.css'
-import NotionIcon from '@/app/components/base/notion-icon'
+import Panel from '../panel'
+import { DataSourceType } from '../panel/types'
 import type { DataSourceNotion as TDataSourceNotion } from '@/models/common'
 import { useAppContext } from '@/context/app-context'
 import { fetchNotionConnection } from '@/service/common'
+import NotionIcon from '@/app/components/base/notion-icon'
 
-type DataSourceNotionProps = {
+const Icon: FC<{
+  src: string
+  name: string
+  className: string
+}> = ({ src, name, className }) => {
+  return (
+    <NotionIcon
+      src={src}
+      name={name}
+      className={className}
+    />
+  )
+}
+type Props = {
   workspaces: TDataSourceNotion[]
 }
-const DataSourceNotion = ({
+
+const DataSourceNotion: FC<Props> = ({
   workspaces,
-}: DataSourceNotionProps) => {
-  const { t } = useTranslation()
+}) => {
   const { isCurrentWorkspaceManager } = useAppContext()
   const [canConnectNotion, setCanConnectNotion] = useState(false)
   const { data } = useSWR(canConnectNotion ? '/oauth/data-source/notion' : null, fetchNotionConnection)
@@ -42,95 +53,31 @@ const DataSourceNotion = ({
     if (data?.data)
       window.location.href = data.data
   }, [data])
-
   return (
-    <div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
-      <div className='flex items-center px-3 py-[9px]'>
-        <div className={cn(s['notion-icon'], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
-        <div className='grow'>
-          <div className='leading-5 text-sm font-medium text-gray-800'>
-            {t('common.dataSource.notion.title')}
-          </div>
-          {
-            !connected && (
-              <div className='leading-5 text-xs text-gray-500'>
-                {t('common.dataSource.notion.description')}
-              </div>
-            )
-          }
-        </div>
-        {
-          connected
-            ? (
-              <div
-                className={
-                  `flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
-                  rounded-md text-xs font-medium text-gray-700
-                  ${isCurrentWorkspaceManager ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
-                }
-                onClick={handleConnectNotion}
-              >
-                {t('common.dataSource.connect')}
-              </div>
-            )
-            : (
-              <div
-                className={
-                  `flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
-                  ${isCurrentWorkspaceManager ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
-                }
-                onClick={handleConnectNotion}
-              >
-                <PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
-                {t('common.dataSource.notion.addWorkspace')}
-              </div>
-            )
-        }
-      </div>
-      {
-        connected && (
-          <div className='flex items-center px-3 h-[18px]'>
-            <div className='text-xs font-medium text-gray-500'>
-              {t('common.dataSource.notion.connectedWorkspace')}
-            </div>
-            <div className='grow ml-3 border-t border-t-gray-100' />
-          </div>
-        )
-      }
-      {
-        connected && (
-          <div className='px-3 pt-2 pb-3'>
-            {
-              workspaces.map(workspace => (
-                <div className={cn(s['workspace-item'], 'flex items-center mb-1 py-1 pr-1 bg-white rounded-lg')} key={workspace.id}>
-                  <NotionIcon
-                    className='ml-3 mr-[6px]'
-                    src={workspace.source_info.workspace_icon}
-                    name={workspace.source_info.workspace_name}
-                  />
-                  <div className='grow py-[7px] leading-[18px] text-[13px] font-medium text-gray-700 truncate' title={workspace.source_info.workspace_name}>{workspace.source_info.workspace_name}</div>
-                  {
-                    workspace.is_bound
-                      ? <Indicator className='shrink-0 mr-[6px]' />
-                      : <Indicator className='shrink-0 mr-[6px]' color='yellow' />
-                  }
-                  <div className='shrink-0 mr-3 text-xs font-medium'>
-                    {
-                      workspace.is_bound
-                        ? t('common.dataSource.notion.connected')
-                        : t('common.dataSource.notion.disconnected')
-                    }
-                  </div>
-                  <div className='mr-2 w-[1px] h-3 bg-gray-100' />
-                  <Operate workspace={workspace} onAuthAgain={handleAuthAgain} />
-                </div>
-              ))
-            }
-          </div>
-        )
-      }
-    </div>
+    <Panel
+      type={DataSourceType.notion}
+      isConfigured={connected}
+      onConfigure={handleConnectNotion}
+      readonly={!isCurrentWorkspaceManager}
+      configuredList={workspaces.map(workspace => ({
+        id: workspace.id,
+        logo: ({ className }: { className: string }) => (
+          <Icon
+            src={workspace.source_info.workspace_icon!}
+            name={workspace.source_info.workspace_name}
+            className={className}
+          />),
+        name: workspace.source_info.workspace_name,
+        isActive: workspace.is_bound,
+        notionConfig: {
+          total: workspace.source_info.total || 0,
+        },
+      }))}
+      onRemove={() => { }} // handled in operation/index.tsx
+      notionActions={{
+        onChangeAuthorizedPage: handleAuthAgain,
+      }}
+    />
   )
 }
-
-export default DataSourceNotion
+export default React.memo(DataSourceNotion)
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-notion/operate/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-notion/operate/index.tsx
index e115034ff733aa..7b20e5e0a61e13 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-notion/operate/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-notion/operate/index.tsx
@@ -6,17 +6,19 @@ import { EllipsisHorizontalIcon } from '@heroicons/react/24/solid'
 import { Menu, Transition } from '@headlessui/react'
 import { syncDataSourceNotion, updateDataSourceNotionAction } from '@/service/common'
 import Toast from '@/app/components/base/toast'
-import type { DataSourceNotion } from '@/models/common'
 import { FilePlus02 } from '@/app/components/base/icons/src/vender/line/files'
 import { RefreshCw05 } from '@/app/components/base/icons/src/vender/line/arrows'
 import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
 
 type OperateProps = {
-  workspace: DataSourceNotion
+  payload: {
+    id: string
+    total: number
+  }
   onAuthAgain: () => void
 }
 export default function Operate({
-  workspace,
+  payload,
   onAuthAgain,
 }: OperateProps) {
   const itemClassName = `
@@ -37,11 +39,11 @@ export default function Operate({
     mutate({ url: 'data-source/integrates' })
   }
   const handleSync = async () => {
-    await syncDataSourceNotion({ url: `/oauth/data-source/notion/${workspace.id}/sync` })
+    await syncDataSourceNotion({ url: `/oauth/data-source/notion/${payload.id}/sync` })
     updateIntegrates()
   }
   const handleRemove = async () => {
-    await updateDataSourceNotionAction({ url: `/data-source/integrates/${workspace.id}/disable` })
+    await updateDataSourceNotionAction({ url: `/data-source/integrates/${payload.id}/disable` })
     updateIntegrates()
   }
 
@@ -79,7 +81,7 @@ export default function Operate({
                       <div>
                         <div className='leading-5'>{t('common.dataSource.notion.changeAuthorizedPages')}</div>
                         <div className='leading-5 text-xs text-gray-500'>
-                          {workspace.source_info.total} {t('common.dataSource.notion.pagesAuthorized')}
+                          {payload.total} {t('common.dataSource.notion.pagesAuthorized')}
                         </div>
                       </div>
                     </div>
diff --git a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
new file mode 100644
index 00000000000000..a776455432410d
--- /dev/null
+++ b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
@@ -0,0 +1,78 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
+import Indicator from '../../../indicator'
+import Operate from '../data-source-notion/operate'
+import { DataSourceType } from './types'
+import s from './style.module.css'
+import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
+
+export type ConfigItemType = {
+  id: string
+  logo: any
+  name: string
+  isActive: boolean
+  notionConfig?: {
+    total: number
+  }
+}
+
+type Props = {
+  type: DataSourceType
+  payload: ConfigItemType
+  onRemove: () => void
+  notionActions?: {
+    onChangeAuthorizedPage: () => void
+  }
+}
+
+const ConfigItem: FC<Props> = ({
+  type,
+  payload,
+  onRemove,
+  notionActions,
+}) => {
+  const { t } = useTranslation()
+  const isNotion = type === DataSourceType.notion
+  const isWebsite = type === DataSourceType.website
+  const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || function () { }
+
+  return (
+    <div className={cn(s['workspace-item'], 'flex items-center mb-1 py-1 pr-1 bg-white rounded-lg')} key={payload.id}>
+      <payload.logo className='ml-3 mr-[6px]' />
+      <div className='grow py-[7px] leading-[18px] text-[13px] font-medium text-gray-700 truncate' title={payload.name}>{payload.name}</div>
+      {
+        payload.isActive
+          ? <Indicator className='shrink-0 mr-[6px]' />
+          : <Indicator className='shrink-0 mr-[6px]' color='yellow' />
+      }
+      <div className='shrink-0 mr-3 text-xs font-medium'>
+        {
+          payload.isActive
+            ? t('common.dataSource.notion.connected')
+            : t('common.dataSource.notion.disconnected')
+        }
+      </div>
+      <div className='mr-2 w-[1px] h-3 bg-gray-100' />
+      {isNotion && (
+        <Operate payload={{
+          id: payload.id,
+          total: payload.notionConfig?.total || 0,
+        }} onAuthAgain={onChangeAuthorizedPage}
+        />
+      )}
+
+      {
+        isWebsite && (
+          <div className='p-1 text-gray-500 cursor-pointer rounded-md hover:bg-black/5' onClick={onRemove} >
+            <Trash03 className='w-3 h-3 ' />
+          </div>
+        )
+      }
+
+    </div>
+  )
+}
+export default React.memo(ConfigItem)
diff --git a/web/app/components/header/account-setting/data-source-page/panel/index.tsx b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
new file mode 100644
index 00000000000000..37d87773670672
--- /dev/null
+++ b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
@@ -0,0 +1,113 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { PlusIcon } from '@heroicons/react/24/solid'
+import cn from 'classnames'
+import type { ConfigItemType } from './config-item'
+import ConfigItem from './config-item'
+
+import s from './style.module.css'
+import type { DataSourceType } from './types'
+
+type Props = {
+  type: DataSourceType
+  isConfigured: boolean
+  onConfigure: () => void
+  readonly: boolean
+  isSupportList?: boolean
+  configuredList: ConfigItemType[]
+  onRemove: () => void
+  notionActions?: {
+    onChangeAuthorizedPage: () => void
+  }
+}
+
+const Panel: FC<Props> = ({
+  type,
+  isConfigured,
+  onConfigure,
+  readonly,
+  configuredList,
+  isSupportList,
+  onRemove,
+  notionActions,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
+      <div className='flex items-center px-3 py-[9px]'>
+        <div className={cn(s['notion-icon'], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
+        <div className='grow'>
+          <div className='leading-5 text-sm font-medium text-gray-800'>
+            {t(`common.dataSource.${type}.title`)}
+          </div>
+          {
+            !isConfigured && (
+              <div className='leading-5 text-xs text-gray-500'>
+                {t(`common.dataSource.${type}.description`)}
+              </div>
+            )
+          }
+        </div>
+        {
+          isConfigured
+            ? (
+              <div
+                className={
+                  `flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
+                  rounded-md text-xs font-medium text-gray-700
+                  ${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
+                }
+                onClick={onConfigure}
+              >
+                {t('common.dataSource.connect')}
+              </div>
+            )
+            : (
+              <>
+                {isSupportList && <div
+                  className={
+                    `flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
+                  ${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
+                  }
+                  onClick={onConfigure}
+                >
+                  <PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
+                  {t('common.dataSource.notion.addWorkspace')}
+                </div>}
+              </>
+            )
+        }
+      </div>
+      {
+        configuredList && (
+          <div className='flex items-center px-3 h-[18px]'>
+            <div className='text-xs font-medium text-gray-500'>
+              {t('common.dataSource.notion.connectedWorkspace')}
+            </div>
+            <div className='grow ml-3 border-t border-t-gray-100' />
+          </div>
+        )
+      }
+      {
+        isConfigured && (
+          <div className='px-3 pt-2 pb-3'>
+            {
+              configuredList.map(item => (
+                <ConfigItem
+                  key={item.id}
+                  type={type}
+                  payload={item}
+                  onRemove={onRemove}
+                  notionActions={notionActions} />
+              ))
+            }
+          </div>
+        )
+      }
+    </div>
+  )
+}
+export default React.memo(Panel)
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-notion/style.module.css b/web/app/components/header/account-setting/data-source-page/panel/style.module.css
similarity index 100%
rename from web/app/components/header/account-setting/data-source-page/data-source-notion/style.module.css
rename to web/app/components/header/account-setting/data-source-page/panel/style.module.css
diff --git a/web/app/components/header/account-setting/data-source-page/panel/types.ts b/web/app/components/header/account-setting/data-source-page/panel/types.ts
new file mode 100644
index 00000000000000..345bc10f8121dc
--- /dev/null
+++ b/web/app/components/header/account-setting/data-source-page/panel/types.ts
@@ -0,0 +1,4 @@
+export enum DataSourceType {
+  notion = 'notion',
+  website = 'website',
+}

From ddfac3f4aab9cb8106b5731354ae606f7e040397 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 31 May 2024 17:35:28 +0800
Subject: [PATCH 02/31] feat: website panel

---
 web/app/(commonLayout)/apps/Apps.tsx          | 16 +++-
 .../data-source-notion/index.tsx              |  1 +
 .../data-source-website/index.tsx             | 80 +++++++++++++++++++
 .../data-source-page/index.tsx                |  2 +
 .../data-source-page/panel/index.tsx          |  4 +-
 .../data-source-page/panel/style.module.css   |  5 ++
 web/i18n/en-US/common.ts                      |  4 +
 web/i18n/zh-Hans/common.ts                    |  4 +
 web/models/common.ts                          | 26 ++++++
 web/service/common.ts                         |  5 ++
 10 files changed, 141 insertions(+), 6 deletions(-)
 create mode 100644 web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx

diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx
index 744bb9c9d70981..f4bac7e44bbf7f 100644
--- a/web/app/(commonLayout)/apps/Apps.tsx
+++ b/web/app/(commonLayout)/apps/Apps.tsx
@@ -24,6 +24,7 @@ import SearchInput from '@/app/components/base/search-input'
 import { useStore as useTagStore } from '@/app/components/base/tag-management/store'
 import TagManagementModal from '@/app/components/base/tag-management'
 import TagFilter from '@/app/components/base/tag-management/filter'
+import { useModalContext } from '@/context/modal-context'
 
 const getKey = (
   pageIndex: number,
@@ -49,6 +50,13 @@ const getKey = (
 }
 
 const Apps = () => {
+  const { setShowAccountSettingModal } = useModalContext()
+  useEffect(() => {
+    setShowAccountSettingModal({
+      payload: 'data-source',
+    })
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
   const { t } = useTranslation()
   const { isCurrentWorkspaceManager } = useAppContext()
   const showTagManagementModal = useTagStore(s => s.showTagManagementModal)
@@ -73,10 +81,10 @@ const Apps = () => {
 
   const anchorRef = useRef<HTMLDivElement>(null)
   const options = [
-    { value: 'all', text: t('app.types.all'), icon: <DotsGrid className='w-[14px] h-[14px] mr-1'/> },
-    { value: 'chat', text: t('app.types.chatbot'), icon: <ChatBot className='w-[14px] h-[14px] mr-1'/> },
-    { value: 'agent-chat', text: t('app.types.agent'), icon: <CuteRobot className='w-[14px] h-[14px] mr-1'/> },
-    { value: 'workflow', text: t('app.types.workflow'), icon: <Route className='w-[14px] h-[14px] mr-1'/> },
+    { value: 'all', text: t('app.types.all'), icon: <DotsGrid className='w-[14px] h-[14px] mr-1' /> },
+    { value: 'chat', text: t('app.types.chatbot'), icon: <ChatBot className='w-[14px] h-[14px] mr-1' /> },
+    { value: 'agent-chat', text: t('app.types.agent'), icon: <CuteRobot className='w-[14px] h-[14px] mr-1' /> },
+    { value: 'workflow', text: t('app.types.workflow'), icon: <Route className='w-[14px] h-[14px] mr-1' /> },
   ]
 
   useEffect(() => {
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx
index 4929b2b403fcb3..f5541999a492bc 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-notion/index.tsx
@@ -59,6 +59,7 @@ const DataSourceNotion: FC<Props> = ({
       isConfigured={connected}
       onConfigure={handleConnectNotion}
       readonly={!isCurrentWorkspaceManager}
+      isSupportList
       configuredList={workspaces.map(workspace => ({
         id: workspace.id,
         logo: ({ className }: { className: string }) => (
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
new file mode 100644
index 00000000000000..81b5f91a0fa317
--- /dev/null
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
@@ -0,0 +1,80 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useBoolean } from 'ahooks'
+import Panel from '../panel'
+import { DataSourceType } from '../panel/types'
+import { fetchWebsiteDataSource } from '@/service/common'
+import type {
+  DataSourceWebsiteItem,
+} from '@/models/common'
+import { useAppContext } from '@/context/app-context'
+
+import {
+  DataSourceCategory,
+  WebsiteProvider,
+} from '@/models/common'
+
+type Props = {}
+
+const isUseMock = false
+const mockList: DataSourceWebsiteItem[] = [
+  {
+    id: '1',
+    category: DataSourceCategory.website,
+    provider: WebsiteProvider.fireCrawl,
+    credentials: {
+      auth_type: 'bearer',
+      config: {
+        base_url: 'https://xxx',
+        api_key: '123456',
+      },
+    },
+    created_at: 1627584000,
+    updated_at: 1627584000,
+  },
+]
+
+const DataSourceWebsite: FC<Props> = () => {
+  const { isCurrentWorkspaceManager } = useAppContext()
+  const [list, setList] = useState<DataSourceWebsiteItem[]>(isUseMock ? mockList : [])
+  useEffect(() => {
+    (async () => {
+      const { data } = await fetchWebsiteDataSource()
+      const list = data.settings.filter(item => item.provider === WebsiteProvider.fireCrawl)
+
+      setList(list)
+    })()
+  }, [])
+
+  const [isShowConfig, {
+    setTrue: showConfig,
+    setFalse: hideConfig,
+  }] = useBoolean(false)
+
+  const handleRemove = useCallback(() => {
+
+  }, [])
+
+  console.log(list)
+
+  return (
+    <Panel
+      type={DataSourceType.website}
+      isConfigured={list.length > 0}
+      onConfigure={showConfig}
+      readonly={!isCurrentWorkspaceManager}
+      configuredList={list.map(item => ({
+        id: item.id,
+        logo: ({ className }: { className: string }) => (
+          <div className={className}>L</div>
+        ),
+        name: 'FireCrawl',
+        isActive: true,
+      }))}
+      onRemove={handleRemove}
+
+    />
+  )
+}
+export default React.memo(DataSourceWebsite)
diff --git a/web/app/components/header/account-setting/data-source-page/index.tsx b/web/app/components/header/account-setting/data-source-page/index.tsx
index 761d9cbfe3ddef..ede83152b223e1 100644
--- a/web/app/components/header/account-setting/data-source-page/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/index.tsx
@@ -1,6 +1,7 @@
 import useSWR from 'swr'
 import { useTranslation } from 'react-i18next'
 import DataSourceNotion from './data-source-notion'
+import DataSourceWebsite from './data-source-website'
 import { fetchDataSource } from '@/service/common'
 
 export default function DataSourcePage() {
@@ -12,6 +13,7 @@ export default function DataSourcePage() {
     <div className='mb-8'>
       <div className='mb-2 text-sm font-medium text-gray-900'>{t('common.dataSource.add')}</div>
       <DataSourceNotion workspaces={notionWorkspaces} />
+      <DataSourceWebsite />
     </div>
   )
 }
diff --git a/web/app/components/header/account-setting/data-source-page/panel/index.tsx b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
index 37d87773670672..fc6657681138a7 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
@@ -38,7 +38,7 @@ const Panel: FC<Props> = ({
   return (
     <div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
       <div className='flex items-center px-3 py-[9px]'>
-        <div className={cn(s['notion-icon'], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
+        <div className={cn(s[`${type}-icon`], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
         <div className='grow'>
           <div className='leading-5 text-sm font-medium text-gray-800'>
             {t(`common.dataSource.${type}.title`)}
@@ -82,7 +82,7 @@ const Panel: FC<Props> = ({
         }
       </div>
       {
-        configuredList && (
+        isConfigured && (
           <div className='flex items-center px-3 h-[18px]'>
             <div className='text-xs font-medium text-gray-500'>
               {t('common.dataSource.notion.connectedWorkspace')}
diff --git a/web/app/components/header/account-setting/data-source-page/panel/style.module.css b/web/app/components/header/account-setting/data-source-page/panel/style.module.css
index ede323072dad24..a11d4758fbaec8 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/style.module.css
+++ b/web/app/components/header/account-setting/data-source-page/panel/style.module.css
@@ -3,6 +3,11 @@
   background-size: 20px 20px;
 }
 
+.website-icon {
+  background: #ffffff url(../../../../datasets/create/assets/web.svg) center center no-repeat;
+  background-size: 20px 20px;
+}
+
 .workspace-item {
   box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);
 }
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts
index 586ca28df96a51..d0d7398772e5a6 100644
--- a/web/i18n/en-US/common.ts
+++ b/web/i18n/en-US/common.ts
@@ -357,6 +357,10 @@ const translation = {
         preview: 'PREVIEW',
       },
     },
+    website: {
+      title: 'Website',
+      description: 'Import content from websites using web crawler.',
+    },
   },
   plugin: {
     serpapi: {
diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts
index 0e6feb7cf562c0..d7fc1ab3873685 100644
--- a/web/i18n/zh-Hans/common.ts
+++ b/web/i18n/zh-Hans/common.ts
@@ -356,6 +356,10 @@ const translation = {
         addPages: '添加页面',
         preview: '预览',
       },
+      website: {
+        title: '网站',
+        description: '使用网络爬虫从网站导入内容。',
+      },
     },
   },
   plugin: {
diff --git a/web/models/common.ts b/web/models/common.ts
index d5ebdc3faa3e10..a8322e2478206f 100644
--- a/web/models/common.ts
+++ b/web/models/common.ts
@@ -171,6 +171,32 @@ export type DataSourceNotion = {
   source_info: DataSourceNotionWorkspace
 }
 
+export enum DataSourceCategory {
+  website = 'website',
+}
+export enum WebsiteProvider {
+  fireCrawl = 'firecrawl',
+}
+
+export type WebsiteCredentials = {
+  auth_type: 'bearer'
+  config: {
+    base_url: string
+    api_key: string
+  }
+}
+export type DataSourceWebsiteItem = {
+  id: string
+  category: DataSourceCategory.website
+  provider: WebsiteProvider
+  credentials: WebsiteCredentials
+  created_at: number
+  updated_at: number
+}
+export type DataSourceWebsite = {
+  settings: DataSourceWebsiteItem[]
+}
+
 export type GithubRepo = {
   stargazers_count: number
 }
diff --git a/web/service/common.ts b/web/service/common.ts
index d7f7c6c8bcc0b8..8c645b225eb466 100644
--- a/web/service/common.ts
+++ b/web/service/common.ts
@@ -6,6 +6,7 @@ import type {
   CodeBasedExtension,
   CommonResponse,
   DataSourceNotion,
+  DataSourceWebsite,
   FileUploadConfigResponse,
   ICurrentWorkspace,
   IWorkspace,
@@ -143,6 +144,10 @@ export const updateDataSourceNotionAction: Fetcher<CommonResponse, { url: string
   return patch<CommonResponse>(url)
 }
 
+export const fetchWebsiteDataSource = () => {
+  return get<{ data: DataSourceWebsite }>('/api-key-auth/data-source')
+}
+
 export const fetchPluginProviders: Fetcher<PluginProvider[] | null, string> = (url) => {
   return get<PluginProvider[] | null>(url)
 }

From 8f1a662c4abfcf5ed6b57faf56e77c864459922d Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 31 May 2024 18:25:17 +0800
Subject: [PATCH 03/31] feat: with fire crawl label

---
 .../account-setting/data-source-page/panel/index.tsx | 12 +++++++++---
 web/i18n/en-US/common.ts                             |  1 +
 web/i18n/zh-Hans/common.ts                           |  1 +
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/web/app/components/header/account-setting/data-source-page/panel/index.tsx b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
index fc6657681138a7..6d6fc144abdb69 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
@@ -8,7 +8,7 @@ import type { ConfigItemType } from './config-item'
 import ConfigItem from './config-item'
 
 import s from './style.module.css'
-import type { DataSourceType } from './types'
+import { DataSourceType } from './types'
 
 type Props = {
   type: DataSourceType
@@ -34,14 +34,20 @@ const Panel: FC<Props> = ({
   notionActions,
 }) => {
   const { t } = useTranslation()
+  const isWebsite = type === DataSourceType.website
 
   return (
     <div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
       <div className='flex items-center px-3 py-[9px]'>
         <div className={cn(s[`${type}-icon`], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
         <div className='grow'>
-          <div className='leading-5 text-sm font-medium text-gray-800'>
-            {t(`common.dataSource.${type}.title`)}
+          <div className='flex items-center h-5'>
+            <div className='text-sm font-medium text-gray-800'>{t(`common.dataSource.${type}.title`)}</div>
+            {isWebsite && (
+              <div className='ml-1 leading-[18px] px-1.5 rounded-md bg-white border border-gray-100 text-xs font-medium text-gray-700'>
+                <span className='text-gray-500'>{t('common.dataSource.website.with')}</span> 🔥 FireCrawl
+              </div>
+            )}
           </div>
           {
             !isConfigured && (
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts
index d0d7398772e5a6..06b7961773b807 100644
--- a/web/i18n/en-US/common.ts
+++ b/web/i18n/en-US/common.ts
@@ -360,6 +360,7 @@ const translation = {
     website: {
       title: 'Website',
       description: 'Import content from websites using web crawler.',
+      with: 'With',
     },
   },
   plugin: {
diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts
index d7fc1ab3873685..af67c57e13117c 100644
--- a/web/i18n/zh-Hans/common.ts
+++ b/web/i18n/zh-Hans/common.ts
@@ -359,6 +359,7 @@ const translation = {
       website: {
         title: '网站',
         description: '使用网络爬虫从网站导入内容。',
+        with: '使用',
       },
     },
   },

From bde44fd2a26a6382997ea95fe838a22dfc8ef890 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 31 May 2024 18:35:29 +0800
Subject: [PATCH 04/31] fix: not show configure and i18n problem

---
 .../data-source-page/panel/index.tsx          | 71 ++++++++++++-------
 web/i18n/en-US/common.ts                      |  1 +
 web/i18n/zh-Hans/common.ts                    | 11 +--
 3 files changed, 52 insertions(+), 31 deletions(-)

diff --git a/web/app/components/header/account-setting/data-source-page/panel/index.tsx b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
index 6d6fc144abdb69..75bef952f8ede0 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
@@ -34,6 +34,7 @@ const Panel: FC<Props> = ({
   notionActions,
 }) => {
   const { t } = useTranslation()
+  const isNotion = type === DataSourceType.notion
   const isWebsite = type === DataSourceType.website
 
   return (
@@ -57,35 +58,53 @@ const Panel: FC<Props> = ({
             )
           }
         </div>
-        {
-          isConfigured
-            ? (
-              <div
-                className={
-                  `flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
+        {isNotion && (
+          <>
+            {
+              isConfigured
+                ? (
+                  <div
+                    className={
+                      `flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
                   rounded-md text-xs font-medium text-gray-700
                   ${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
-                }
-                onClick={onConfigure}
-              >
-                {t('common.dataSource.connect')}
-              </div>
-            )
-            : (
-              <>
-                {isSupportList && <div
-                  className={
-                    `flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
+                    }
+                    onClick={onConfigure}
+                  >
+                    {t('common.dataSource.configure')}
+                  </div>
+                )
+                : (
+                  <>
+                    {isSupportList && <div
+                      className={
+                        `flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
                   ${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
-                  }
-                  onClick={onConfigure}
-                >
-                  <PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
-                  {t('common.dataSource.notion.addWorkspace')}
-                </div>}
-              </>
-            )
-        }
+                      }
+                      onClick={onConfigure}
+                    >
+                      <PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
+                      {t('common.dataSource.notion.addWorkspace')}
+                    </div>}
+                  </>
+                )
+            }
+          </>
+        )}
+
+        {isWebsite && !isConfigured && (
+          <div
+            className={
+              `flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
+        rounded-md text-xs font-medium text-gray-700
+        ${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
+            }
+            onClick={onConfigure}
+          >
+            {t('common.dataSource.configure')}
+          </div>
+        )}
+
       </div>
       {
         isConfigured && (
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts
index 06b7961773b807..6984b89072607c 100644
--- a/web/i18n/en-US/common.ts
+++ b/web/i18n/en-US/common.ts
@@ -338,6 +338,7 @@ const translation = {
   dataSource: {
     add: 'Add a data source',
     connect: 'Connect',
+    configure: 'Configure',
     notion: {
       title: 'Notion',
       description: 'Using Notion as a data source for the Knowledge.',
diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts
index af67c57e13117c..64d0a367bc41a6 100644
--- a/web/i18n/zh-Hans/common.ts
+++ b/web/i18n/zh-Hans/common.ts
@@ -338,6 +338,7 @@ const translation = {
   dataSource: {
     add: '添加数据源',
     connect: '绑定',
+    configure: '配置',
     notion: {
       title: 'Notion',
       description: '使用 Notion 作为知识库的数据源。',
@@ -356,11 +357,11 @@ const translation = {
         addPages: '添加页面',
         preview: '预览',
       },
-      website: {
-        title: '网站',
-        description: '使用网络爬虫从网站导入内容。',
-        with: '使用',
-      },
+    },
+    website: {
+      title: '网站',
+      description: '使用网络爬虫从网站导入内容。',
+      with: '使用',
     },
   },
   plugin: {

From 04857a29f86ceb239d88bce9a00ee6e3e771eb07 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 31 May 2024 18:41:45 +0800
Subject: [PATCH 05/31] feat: firecrawl item

---
 .../data-source-page/data-source-website/index.tsx           | 5 +++--
 .../account-setting/data-source-page/panel/config-item.tsx   | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
index 81b5f91a0fa317..c572f5d3b0dd9e 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
@@ -2,6 +2,7 @@
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useState } from 'react'
 import { useBoolean } from 'ahooks'
+import cn from 'classnames'
 import Panel from '../panel'
 import { DataSourceType } from '../panel/types'
 import { fetchWebsiteDataSource } from '@/service/common'
@@ -17,7 +18,7 @@ import {
 
 type Props = {}
 
-const isUseMock = false
+const isUseMock = true
 const mockList: DataSourceWebsiteItem[] = [
   {
     id: '1',
@@ -67,7 +68,7 @@ const DataSourceWebsite: FC<Props> = () => {
       configuredList={list.map(item => ({
         id: item.id,
         logo: ({ className }: { className: string }) => (
-          <div className={className}>L</div>
+          <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
         ),
         name: 'FireCrawl',
         isActive: true,
diff --git a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
index a776455432410d..2e6e0c64c63958 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
+++ b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
@@ -41,7 +41,7 @@ const ConfigItem: FC<Props> = ({
 
   return (
     <div className={cn(s['workspace-item'], 'flex items-center mb-1 py-1 pr-1 bg-white rounded-lg')} key={payload.id}>
-      <payload.logo className='ml-3 mr-[6px]' />
+      <payload.logo className='ml-3 mr-1.5' />
       <div className='grow py-[7px] leading-[18px] text-[13px] font-medium text-gray-700 truncate' title={payload.name}>{payload.name}</div>
       {
         payload.isActive

From 10e0f43e416c0dd7b6d9daddf66a15d41e1f45a4 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Tue, 4 Jun 2024 11:46:03 +0800
Subject: [PATCH 06/31] chore: crawel text

---
 .../data-source-page/panel/config-item.tsx             | 10 +++++-----
 .../account-setting/data-source-page/panel/index.tsx   |  2 +-
 web/i18n/en-US/common.ts                               |  3 +++
 web/i18n/zh-Hans/common.ts                             |  3 +++
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
index 2e6e0c64c63958..376c4aea7b3318 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
+++ b/web/app/components/header/account-setting/data-source-page/panel/config-item.tsx
@@ -48,11 +48,11 @@ const ConfigItem: FC<Props> = ({
           ? <Indicator className='shrink-0 mr-[6px]' />
           : <Indicator className='shrink-0 mr-[6px]' color='yellow' />
       }
-      <div className='shrink-0 mr-3 text-xs font-medium'>
+      <div className='shrink-0 mr-3 text-xs font-medium uppercase'>
         {
           payload.isActive
-            ? t('common.dataSource.notion.connected')
-            : t('common.dataSource.notion.disconnected')
+            ? t(isNotion ? 'common.dataSource.notion.connected' : 'common.dataSource.website.active')
+            : t(isNotion ? 'common.dataSource.notion.disconnected' : 'common.dataSource.website.inactive')
         }
       </div>
       <div className='mr-2 w-[1px] h-3 bg-gray-100' />
@@ -66,8 +66,8 @@ const ConfigItem: FC<Props> = ({
 
       {
         isWebsite && (
-          <div className='p-1 text-gray-500 cursor-pointer rounded-md hover:bg-black/5' onClick={onRemove} >
-            <Trash03 className='w-3 h-3 ' />
+          <div className='p-2 text-gray-500 cursor-pointer rounded-md hover:bg-black/5' onClick={onRemove} >
+            <Trash03 className='w-4 h-4 ' />
           </div>
         )
       }
diff --git a/web/app/components/header/account-setting/data-source-page/panel/index.tsx b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
index 75bef952f8ede0..b0f6f4ad13ac95 100644
--- a/web/app/components/header/account-setting/data-source-page/panel/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/panel/index.tsx
@@ -110,7 +110,7 @@ const Panel: FC<Props> = ({
         isConfigured && (
           <div className='flex items-center px-3 h-[18px]'>
             <div className='text-xs font-medium text-gray-500'>
-              {t('common.dataSource.notion.connectedWorkspace')}
+              {isNotion ? t('common.dataSource.notion.connectedWorkspace') : t('common.dataSource.website.configuredCrawlers')}
             </div>
             <div className='grow ml-3 border-t border-t-gray-100' />
           </div>
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts
index 6984b89072607c..c0a64cc0b6e56c 100644
--- a/web/i18n/en-US/common.ts
+++ b/web/i18n/en-US/common.ts
@@ -362,6 +362,9 @@ const translation = {
       title: 'Website',
       description: 'Import content from websites using web crawler.',
       with: 'With',
+      configuredCrawlers: 'Configured crawlers',
+      active: 'Active',
+      inactive: 'Inactive',
     },
   },
   plugin: {
diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts
index 64d0a367bc41a6..c3e82cb7aaa488 100644
--- a/web/i18n/zh-Hans/common.ts
+++ b/web/i18n/zh-Hans/common.ts
@@ -362,6 +362,9 @@ const translation = {
       title: '网站',
       description: '使用网络爬虫从网站导入内容。',
       with: '使用',
+      configuredCrawlers: '已配置的爬虫',
+      active: '可用',
+      inactive: '不可用',
     },
   },
   plugin: {

From df97c6b291ddecebb3e42eb41fbf19f251ddce8c Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Tue, 4 Jun 2024 16:48:38 +0800
Subject: [PATCH 07/31] feat: firecrawl no data

---
 .../assets/vender/line/others/icon-3-dots.svg |  5 +++
 .../src/vender/line/others/Icon3Dots.json     | 39 +++++++++++++++++++
 .../src/vender/line/others/Icon3Dots.tsx      | 16 ++++++++
 .../icons/src/vender/line/others/index.ts     |  1 +
 web/app/components/datasets/create/index.tsx  |  2 +-
 .../datasets/create/step-one/index.tsx        |  9 +++--
 .../create/website/import-firecrawl.tsx       | 15 +++++++
 .../datasets/create/website/index.tsx         | 18 +++++++++
 .../datasets/create/website/no-data.tsx       | 36 +++++++++++++++++
 web/i18n/en-US/dataset-creation.ts            |  5 +++
 web/i18n/zh-Hans/dataset-creation.ts          |  5 +++
 11 files changed, 147 insertions(+), 4 deletions(-)
 create mode 100644 web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg
 create mode 100644 web/app/components/base/icons/src/vender/line/others/Icon3Dots.json
 create mode 100644 web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx
 create mode 100644 web/app/components/datasets/create/website/import-firecrawl.tsx
 create mode 100644 web/app/components/datasets/create/website/index.tsx
 create mode 100644 web/app/components/datasets/create/website/no-data.tsx

diff --git a/web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg b/web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg
new file mode 100644
index 00000000000000..bba42851f61829
--- /dev/null
+++ b/web/app/components/base/icons/assets/vender/line/others/icon-3-dots.svg
@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Icon-3-dots">
+<path id="Icon" d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+</svg>
diff --git a/web/app/components/base/icons/src/vender/line/others/Icon3Dots.json b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.json
new file mode 100644
index 00000000000000..0942222f39e0a4
--- /dev/null
+++ b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.json
@@ -0,0 +1,39 @@
+{
+	"icon": {
+		"type": "element",
+		"isRootNode": true,
+		"name": "svg",
+		"attributes": {
+			"width": "16",
+			"height": "16",
+			"viewBox": "0 0 16 16",
+			"fill": "none",
+			"xmlns": "http://www.w3.org/2000/svg"
+		},
+		"children": [
+			{
+				"type": "element",
+				"name": "g",
+				"attributes": {
+					"id": "Icon-3-dots"
+				},
+				"children": [
+					{
+						"type": "element",
+						"name": "path",
+						"attributes": {
+							"id": "Icon",
+							"d": "M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103",
+							"stroke": "currentColor",
+							"stroke-width": "2",
+							"stroke-linecap": "round",
+							"stroke-linejoin": "round"
+						},
+						"children": []
+					}
+				]
+			}
+		]
+	},
+	"name": "Icon3Dots"
+}
\ No newline at end of file
diff --git a/web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx
new file mode 100644
index 00000000000000..1f9eb767a8f2fb
--- /dev/null
+++ b/web/app/components/base/icons/src/vender/line/others/Icon3Dots.tsx
@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './Icon3Dots.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'Icon3Dots'
+
+export default Icon
diff --git a/web/app/components/base/icons/src/vender/line/others/index.ts b/web/app/components/base/icons/src/vender/line/others/index.ts
index 648792f22b40f0..554f14b55bd650 100644
--- a/web/app/components/base/icons/src/vender/line/others/index.ts
+++ b/web/app/components/base/icons/src/vender/line/others/index.ts
@@ -3,4 +3,5 @@ export { default as Colors } from './Colors'
 export { default as DragHandle } from './DragHandle'
 export { default as Exchange02 } from './Exchange02'
 export { default as FileCode } from './FileCode'
+export { default as Icon3Dots } from './Icon3Dots'
 export { default as Tools } from './Tools'
diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
index ce32f29fc528fd..83d74b55baefd6 100644
--- a/web/app/components/datasets/create/index.tsx
+++ b/web/app/components/datasets/create/index.tsx
@@ -23,7 +23,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const { t } = useTranslation()
   const { setShowAccountSettingModal } = useModalContext()
   const [hasConnection, setHasConnection] = useState(true)
-  const [dataSourceType, setDataSourceType] = useState<DataSourceType>(DataSourceType.FILE)
+  const [dataSourceType, setDataSourceType] = useState<DataSourceType>(DataSourceType.WEB) // TODO: for test. DataSourceType.FILE
   const [step, setStep] = useState(1)
   const [indexingTypeCache, setIndexTypeCache] = useState('')
   const [fileList, setFiles] = useState<FileItem[]>([])
diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx
index 66df1dd2083466..7d46d5eb5d0bc1 100644
--- a/web/app/components/datasets/create/step-one/index.tsx
+++ b/web/app/components/datasets/create/step-one/index.tsx
@@ -6,6 +6,7 @@ import FilePreview from '../file-preview'
 import FileUploader from '../file-uploader'
 import NotionPagePreview from '../notion-page-preview'
 import EmptyDatasetCreationModal from '../empty-dataset-creation-modal'
+import Website from '../website'
 import s from './index.module.css'
 import type { FileItem } from '@/models/datasets'
 import type { NotionPage } from '@/models/common'
@@ -150,10 +151,9 @@ const StepOne = ({
                   {t('datasetCreation.stepOne.dataSourceType.notion')}
                 </div>
                 <div
-                  className={cn(s.dataSourceItem, s.disabled, dataSourceType === DataSourceType.WEB && s.active)}
-                // onClick={() => changeType(DataSourceType.WEB)}
+                  className={cn(s.dataSourceItem, dataSourceType === DataSourceType.WEB ? s.active : s.disabled)}
+                  onClick={() => changeType(DataSourceType.WEB)}
                 >
-                  <span className={s.comingTag}>Coming soon</span>
                   <span className={cn(s.datasetIcon, s.web)} />
                   {t('datasetCreation.stepOne.dataSourceType.web')}
                 </div>
@@ -201,6 +201,9 @@ const StepOne = ({
               )}
             </>
           )}
+          {dataSourceType === DataSourceType.WEB && (
+            <Website />
+          )}
           {!datasetId && (
             <>
               <div className={s.dividerLine} />
diff --git a/web/app/components/datasets/create/website/import-firecrawl.tsx b/web/app/components/datasets/create/website/import-firecrawl.tsx
new file mode 100644
index 00000000000000..7fafaf32bc93da
--- /dev/null
+++ b/web/app/components/datasets/create/website/import-firecrawl.tsx
@@ -0,0 +1,15 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+
+type Props = {
+
+}
+
+const ImportFireCrawl: FC<Props> = () => {
+  return (
+    <div>
+    </div>
+  )
+}
+export default React.memo(ImportFireCrawl)
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
new file mode 100644
index 00000000000000..cd82d45f7c588c
--- /dev/null
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -0,0 +1,18 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import NoData from './no-data'
+
+type Props = {
+
+}
+
+const WebsitePreview: FC<Props> = () => {
+  const handleOnConfig = useCallback(() => { }, [])
+  return (
+    <div>
+      <NoData onConfig={handleOnConfig} />
+    </div>
+  )
+}
+export default React.memo(WebsitePreview)
diff --git a/web/app/components/datasets/create/website/no-data.tsx b/web/app/components/datasets/create/website/no-data.tsx
new file mode 100644
index 00000000000000..35eb35f7095dac
--- /dev/null
+++ b/web/app/components/datasets/create/website/no-data.tsx
@@ -0,0 +1,36 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
+import Button from '@/app/components/base/button'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onConfig: () => void
+}
+
+const NoData: FC<Props> = ({
+  onConfig,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='max-w-[640px] p-6 rounded-2xl bg-gray-50'>
+      <div className='flex w-11 h-11 items-center justify-center bg-gray-50 rounded-xl border-[0.5px] border-gray-100 shadow-lg'>
+        🔥
+      </div>
+      <div className='my-2'>
+        <span className='text-gray-700 font-semibold'>{t(`${I18N_PREFIX}.fireCrawlNotConfigured`)}<Icon3Dots className='inline relative -top-3 -left-1.5' /></span>
+        <div className='mt-1 pb-3 text-gray-500 text-[13px] font-normal'>
+          {t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`)}
+        </div>
+      </div>
+      <Button type='primary' onClick={onConfig} className='!h-8 text-[13px] font-medium ' >
+        {t(`${I18N_PREFIX}.configure`)}
+      </Button>
+    </div>
+  )
+}
+export default React.memo(NoData)
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 8923170f7f4539..9c7f44f1d0472c 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -50,6 +50,11 @@ const translation = {
       confirmButton: 'Create',
       failed: 'Creation failed',
     },
+    website: {
+      fireCrawlNotConfigured: 'Firecrawl is not configured',
+      fireCrawlNotConfiguredDescription: 'Configure Firecrawl with API key to use it.',
+      configure: 'Configure',
+    },
   },
   stepTwo: {
     segmentation: 'Chunk settings',
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index d36850dc3dc833..208312f01f971d 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -50,6 +50,11 @@ const translation = {
       confirmButton: '创建',
       failed: '创建失败',
     },
+    website: {
+      fireCrawlNotConfigured: 'Firecrawl 未配置',
+      fireCrawlNotConfiguredDescription: '请配置 Firecrawl 的 API 密钥以使用它。',
+      configure: 'Configure',
+    },
   },
   stepTwo: {
     segmentation: '分段设置',

From 5aef25b5fe01fab8fa7064fe51ab12773be93aea Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Tue, 4 Jun 2024 16:53:25 +0800
Subject: [PATCH 08/31] feat: handle show config firecrawl

---
 web/app/components/datasets/create/website/index.tsx  | 11 ++++++++++-
 .../data-source-page/data-source-website/index.tsx    |  2 --
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
index cd82d45f7c588c..96af30faf3ab3b 100644
--- a/web/app/components/datasets/create/website/index.tsx
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -2,13 +2,22 @@
 import type { FC } from 'react'
 import React, { useCallback } from 'react'
 import NoData from './no-data'
+import { useModalContext } from '@/context/modal-context'
 
 type Props = {
 
 }
 
 const WebsitePreview: FC<Props> = () => {
-  const handleOnConfig = useCallback(() => { }, [])
+  const { setShowAccountSettingModal } = useModalContext()
+
+  const handleOnConfig = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: 'data-source',
+    })
+  }, [setShowAccountSettingModal])
+
+  // TODO: on Hide
   return (
     <div>
       <NoData onConfig={handleOnConfig} />
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
index c572f5d3b0dd9e..489af9842cbc03 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
@@ -57,8 +57,6 @@ const DataSourceWebsite: FC<Props> = () => {
 
   }, [])
 
-  console.log(list)
-
   return (
     <Panel
       type={DataSourceType.website}

From fdae549b267f3ec53ddb9762963b0c6942712540 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Tue, 4 Jun 2024 18:30:27 +0800
Subject: [PATCH 09/31] feat: firecrawl header

---
 .../create/website/firecrawl/base/field.tsx   |  0
 .../create/website/firecrawl/base/input.tsx   | 25 +++++++++++
 .../website/firecrawl/base/url-input.tsx      | 41 ++++++++++++++++++
 .../create/website/firecrawl/header.tsx       | 42 +++++++++++++++++++
 .../create/website/firecrawl/index.tsx        | 32 ++++++++++++++
 .../website/firecrawl/options-error.tsx       |  0
 .../create/website/firecrawl/options.tsx      |  0
 .../create/website/import-firecrawl.tsx       | 15 -------
 .../datasets/create/website/index.tsx         | 19 +++++++--
 web/i18n/en-US/dataset-creation.ts            |  4 ++
 web/i18n/zh-Hans/dataset-creation.ts          |  6 ++-
 11 files changed, 165 insertions(+), 19 deletions(-)
 create mode 100644 web/app/components/datasets/create/website/firecrawl/base/field.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/base/input.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/header.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/index.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/options-error.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/options.tsx
 delete mode 100644 web/app/components/datasets/create/website/import-firecrawl.tsx

diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx
new file mode 100644
index 00000000000000..0d3337848d41b0
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx
@@ -0,0 +1,25 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+
+type Props = {
+  value: string
+  onChange: (value: string) => void
+  placeholder?: string
+}
+
+const Input: FC<Props> = ({
+  value,
+  onChange,
+  placeholder = '',
+}) => {
+  return (
+    <input
+      value={value}
+      onChange={e => onChange(e.target.value)}
+      className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
+      placeholder={placeholder}
+    />
+  )
+}
+export default React.memo(Input)
diff --git a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
new file mode 100644
index 00000000000000..915db90d5c5a77
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
@@ -0,0 +1,41 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import Input from './input'
+import Button from '@/app/components/base/button'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onRun: (url: string) => void
+}
+
+const UrlInput: FC<Props> = ({
+  onRun,
+}) => {
+  const { t } = useTranslation()
+  const [url, setUrl] = useState('')
+
+  const handleOnRun = useCallback(() => {
+    onRun(url)
+  }, [onRun, url])
+
+  return (
+    <div className='flex items-center justify-between'>
+      <Input
+        value={url}
+        onChange={setUrl}
+        placeholder='https://docs.dify.ai'
+      />
+      <Button
+        type='primary'
+        onClick={handleOnRun}
+        className='ml-2 !h-8 text-[13px] font-medium'
+      >
+        {t(`${I18N_PREFIX}.run`)}
+      </Button>
+    </div>
+  )
+}
+export default React.memo(UrlInput)
diff --git a/web/app/components/datasets/create/website/firecrawl/header.tsx b/web/app/components/datasets/create/website/firecrawl/header.tsx
new file mode 100644
index 00000000000000..d89cfce46a55ce
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/header.tsx
@@ -0,0 +1,42 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { Settings01 } from '@/app/components/base/icons/src/vender/line/general'
+import { BookOpen01 } from '@/app/components/base/icons/src/vender/line/education'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onSetting: () => void
+}
+
+const Header: FC<Props> = ({
+  onSetting,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex h-6 items-center justify-between'>
+      <div className='flex items-center'>
+        <div className='text-base font-medium text-gray-700'>{t(`${I18N_PREFIX}.firecrawlTitle`)}</div>
+        <div className='ml-2 mr-1 w-px h-3.5 bg-gray-200'></div>
+        <div
+          className='p-1 rounded-md hover:bg-black/5 cursor-pointer'
+          onClick={onSetting}
+        >
+          <Settings01 className='w-3.5 h-3.5 text-gray-500' />
+        </div>
+      </div>
+      <a
+        href={t(`${I18N_PREFIX}.firecrawlDocLink`)!} // TODO: firecrawl doc URL
+        target='_blank' rel='noopener noreferrer'
+        className='flex items-center text-xs text-primary-600'
+      >
+        <BookOpen01 className='mr-1 w-3.5 h-3.5 text-primary-600' />
+        {t(`${I18N_PREFIX}.firecrawlDoc`)}
+      </a>
+    </div>
+  )
+}
+export default React.memo(Header)
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
new file mode 100644
index 00000000000000..85dd5fbc7d6b00
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -0,0 +1,32 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import Header from './header'
+import UrlInput from './base/url-input'
+import { useModalContext } from '@/context/modal-context'
+
+type Props = {
+
+}
+
+const FireCrawl: FC<Props> = () => {
+  const { setShowAccountSettingModal } = useModalContext()
+  const handleSetting = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: 'data-source',
+    })
+  }, [setShowAccountSettingModal])
+
+  const handleRun = useCallback((url: string) => {
+    console.log(url)
+  }, [])
+  return (
+    <div>
+      <Header onSetting={handleSetting} />
+      <div className='mt-2 p-3 pb-4 rounded-xl border border-gray-200'>
+        <UrlInput onRun={handleRun} />
+      </div>
+    </div>
+  )
+}
+export default React.memo(FireCrawl)
diff --git a/web/app/components/datasets/create/website/firecrawl/options-error.tsx b/web/app/components/datasets/create/website/firecrawl/options-error.tsx
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/web/app/components/datasets/create/website/import-firecrawl.tsx b/web/app/components/datasets/create/website/import-firecrawl.tsx
deleted file mode 100644
index 7fafaf32bc93da..00000000000000
--- a/web/app/components/datasets/create/website/import-firecrawl.tsx
+++ /dev/null
@@ -1,15 +0,0 @@
-'use client'
-import type { FC } from 'react'
-import React from 'react'
-
-type Props = {
-
-}
-
-const ImportFireCrawl: FC<Props> = () => {
-  return (
-    <div>
-    </div>
-  )
-}
-export default React.memo(ImportFireCrawl)
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
index 96af30faf3ab3b..b85d986f798418 100644
--- a/web/app/components/datasets/create/website/index.tsx
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -1,7 +1,8 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback } from 'react'
+import React, { useCallback, useState } from 'react'
 import NoData from './no-data'
+import Firecrawl from './firecrawl'
 import { useModalContext } from '@/context/modal-context'
 
 type Props = {
@@ -10,6 +11,8 @@ type Props = {
 
 const WebsitePreview: FC<Props> = () => {
   const { setShowAccountSettingModal } = useModalContext()
+  const [isLoaded, setIsLoaded] = useState(false)
+  const [isConfigured, setIsConfigured] = useState(true)
 
   const handleOnConfig = useCallback(() => {
     setShowAccountSettingModal({
@@ -17,10 +20,20 @@ const WebsitePreview: FC<Props> = () => {
     })
   }, [setShowAccountSettingModal])
 
-  // TODO: on Hide
+  // TODO: on Hide account setting modal
+
+  if (isLoaded)
+    return null
+
   return (
     <div>
-      <NoData onConfig={handleOnConfig} />
+      {isConfigured
+        ? (
+          <Firecrawl />
+        )
+        : (
+          <NoData onConfig={handleOnConfig} />
+        )}
     </div>
   )
 }
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 9c7f44f1d0472c..c3506485d5d8b4 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -54,6 +54,10 @@ const translation = {
       fireCrawlNotConfigured: 'Firecrawl is not configured',
       fireCrawlNotConfiguredDescription: 'Configure Firecrawl with API key to use it.',
       configure: 'Configure',
+      run: 'Run',
+      firecrawlTitle: 'Extract web content with 🔥Firecrawl',
+      firecrawlDoc: 'Firecrawl docs',
+      firecrawlDocLink: '',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index 208312f01f971d..7956c98931db90 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -53,7 +53,11 @@ const translation = {
     website: {
       fireCrawlNotConfigured: 'Firecrawl 未配置',
       fireCrawlNotConfiguredDescription: '请配置 Firecrawl 的 API 密钥以使用它。',
-      configure: 'Configure',
+      configure: '配置',
+      run: '运行',
+      firecrawlTitle: '使用 🔥Firecrawl 提取网页内容',
+      firecrawlDoc: 'Firecrawl 文档',
+      firecrawlDocLink: '',
     },
   },
   stepTwo: {

From 93113766a1ad0a416304f1ba966986c97574a3a2 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Tue, 4 Jun 2024 22:48:20 +0800
Subject: [PATCH 10/31] just add

---
 .../website/firecrawl/base/options-wrap.tsx   | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx

diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
new file mode 100644
index 00000000000000..9f1fe32b7f2742
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -0,0 +1,37 @@
+'use client'
+import { useBoolean } from 'ahooks'
+import type { FC } from 'react'
+import React from 'react'
+import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
+
+type Props = {
+  children: React.ReactNode
+}
+
+const OptionsWrap: FC<Props> = ({
+  children,
+}) => {
+  const [fold, {
+    setTrue: foldTrue,
+    setFalse: foldFalse,
+    toggle: foldToggle,
+  }] = useBoolean(false)
+  return (
+    <div>
+      <div className='flex justify-between items-center h-[18px] py-1'>
+        <div className='flex items-center text-gray-700'>
+          <Settings04 className='mr-1 w-4 h-4' />
+          <div>Options</div>
+        </div>
+
+      </div>
+      {!fold && (
+        <div>
+          {children}
+        </div>
+      )}
+
+    </div>
+  )
+}
+export default React.memo(OptionsWrap)

From baad0a7a5f903c57af37e2fa124b616660dbc208 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 11:11:07 +0800
Subject: [PATCH 11/31] options temp

---
 .../create/website/firecrawl/base/options-wrap.tsx         | 7 ++++++-
 .../components/datasets/create/website/firecrawl/index.tsx | 6 +++++-
 web/i18n/en-US/dataset-creation.ts                         | 1 +
 web/i18n/zh-Hans/dataset-creation.ts                       | 1 +
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index 9f1fe32b7f2742..32b4d3c4b5e9b6 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -2,8 +2,11 @@
 import { useBoolean } from 'ahooks'
 import type { FC } from 'react'
 import React from 'react'
+import { useTranslation } from 'react-i18next'
 import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
 
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
 type Props = {
   children: React.ReactNode
 }
@@ -11,6 +14,8 @@ type Props = {
 const OptionsWrap: FC<Props> = ({
   children,
 }) => {
+  const { t } = useTranslation()
+
   const [fold, {
     setTrue: foldTrue,
     setFalse: foldFalse,
@@ -21,7 +26,7 @@ const OptionsWrap: FC<Props> = ({
       <div className='flex justify-between items-center h-[18px] py-1'>
         <div className='flex items-center text-gray-700'>
           <Settings04 className='mr-1 w-4 h-4' />
-          <div>Options</div>
+          <div>{t(`${I18N_PREFIX}.options`)}</div>
         </div>
 
       </div>
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 85dd5fbc7d6b00..d813ae6e94f2dd 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -3,8 +3,8 @@ import type { FC } from 'react'
 import React, { useCallback } from 'react'
 import Header from './header'
 import UrlInput from './base/url-input'
+import OptionsWrap from './base/options-wrap'
 import { useModalContext } from '@/context/modal-context'
-
 type Props = {
 
 }
@@ -25,6 +25,10 @@ const FireCrawl: FC<Props> = () => {
       <Header onSetting={handleSetting} />
       <div className='mt-2 p-3 pb-4 rounded-xl border border-gray-200'>
         <UrlInput onRun={handleRun} />
+
+        <OptionsWrap>
+          <div>contents</div>
+        </OptionsWrap>
       </div>
     </div>
   )
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index c3506485d5d8b4..9bb86765520f59 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -58,6 +58,7 @@ const translation = {
       firecrawlTitle: 'Extract web content with 🔥Firecrawl',
       firecrawlDoc: 'Firecrawl docs',
       firecrawlDocLink: '',
+      options: 'Options',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index 7956c98931db90..9c4793c0a393b4 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -58,6 +58,7 @@ const translation = {
       firecrawlTitle: '使用 🔥Firecrawl 提取网页内容',
       firecrawlDoc: 'Firecrawl 文档',
       firecrawlDocLink: '',
+      options: '选项',
     },
   },
   stepTwo: {

From 5a0bfb69a37ea13b5d5fc35ad75f05d1863510af Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 14:32:14 +0800
Subject: [PATCH 12/31] feat: options main

---
 .../firecrawl/base/checkbox-with-label.tsx    | 27 +++++++
 .../create/website/firecrawl/base/field.tsx   | 41 +++++++++++
 .../create/website/firecrawl/base/input.tsx   | 19 +++--
 .../website/firecrawl/base/options-wrap.tsx   | 26 +++++--
 .../create/website/firecrawl/index.tsx        | 19 +++--
 .../create/website/firecrawl/options.tsx      | 71 +++++++++++++++++++
 web/models/datasets.ts                        |  9 +++
 7 files changed, 197 insertions(+), 15 deletions(-)
 create mode 100644 web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx

diff --git a/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx
new file mode 100644
index 00000000000000..f25e45724512e7
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx
@@ -0,0 +1,27 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import Checkbox from '@/app/components/base/checkbox'
+
+type Props = {
+  className?: string
+  isChecked: boolean
+  onChange: (isChecked: boolean) => void
+  label: string
+}
+
+const CheckboxWithLabel: FC<Props> = ({
+  className = '',
+  isChecked,
+  onChange,
+  label,
+}) => {
+  return (
+    <label className={cn(className, 'flex items-center h-7 space-x-2')}>
+      <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
+      <div className='text-sm font-normal text-gray-800'>{label}</div>
+    </label>
+  )
+}
+export default React.memo(CheckboxWithLabel)
diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
index e69de29bb2d1d6..d9fb18aa48c420 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
@@ -0,0 +1,41 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import Input from './input'
+
+type Props = {
+  className?: string
+  label: string
+  value: string | number
+  onChange: (value: string | number) => void
+  isRequired?: boolean
+  placeholder?: string
+  isNumber?: boolean
+}
+
+const Field: FC<Props> = ({
+  className,
+  label,
+  value,
+  onChange,
+  isRequired = false,
+  placeholder = '',
+  isNumber = false,
+}) => {
+  return (
+    <div className={cn(className)}>
+      <div>
+        <span>{label} </span>
+        {isRequired && <span>*</span>}
+      </div>
+      <Input
+        value={value}
+        onChange={onChange}
+        placeholder={placeholder}
+        isNumber={isNumber}
+      />
+    </div>
+  )
+}
+export default React.memo(Field)
diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx
index 0d3337848d41b0..a042b1460a78ba 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/input.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx
@@ -1,22 +1,33 @@
 'use client'
 import type { FC } from 'react'
-import React from 'react'
+import React, { useCallback } from 'react'
 
 type Props = {
-  value: string
-  onChange: (value: string) => void
+  value: string | number
+  onChange: (value: string | number) => void
   placeholder?: string
+  isNumber?: boolean
 }
 
 const Input: FC<Props> = ({
   value,
   onChange,
   placeholder = '',
+  isNumber = false,
 }) => {
+  const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
+    const value = e.target.value
+    if (isNumber) {
+      onChange(parseFloat(value))
+      return
+    }
+    onChange(value)
+  }, [isNumber, onChange])
   return (
     <input
+      type={isNumber ? 'number' : 'text'}
       value={value}
-      onChange={e => onChange(e.target.value)}
+      onChange={handleChange}
       className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
       placeholder={placeholder}
     />
diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index 32b4d3c4b5e9b6..ff66cb0e614ca0 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -3,36 +3,48 @@ import { useBoolean } from 'ahooks'
 import type { FC } from 'react'
 import React from 'react'
 import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
 import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
-
+import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 type Props = {
+  className?: string
   children: React.ReactNode
+  errorMsg?: string
 }
 
 const OptionsWrap: FC<Props> = ({
+  className = '',
   children,
+  errorMsg,
 }) => {
   const { t } = useTranslation()
 
   const [fold, {
-    setTrue: foldTrue,
-    setFalse: foldFalse,
     toggle: foldToggle,
   }] = useBoolean(false)
   return (
-    <div>
-      <div className='flex justify-between items-center h-[18px] py-1'>
+    <div className={className}>
+      <div
+        className='flex justify-between items-center h-[26px] py-1 cursor-pointer select-none'
+        onClick={foldToggle}
+      >
         <div className='flex items-center text-gray-700'>
           <Settings04 className='mr-1 w-4 h-4' />
           <div>{t(`${I18N_PREFIX}.options`)}</div>
         </div>
-
+        <ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
       </div>
       {!fold && (
         <div>
-          {children}
+          {!errorMsg
+            ? children
+            : (
+              <div >
+                {errorMsg}
+              </div>
+            )}
         </div>
       )}
 
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index d813ae6e94f2dd..1746a1a2450bec 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -1,14 +1,23 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback } from 'react'
+import React, { useCallback, useState } from 'react'
 import Header from './header'
 import UrlInput from './base/url-input'
 import OptionsWrap from './base/options-wrap'
+import Options from './options'
 import { useModalContext } from '@/context/modal-context'
+import type { CrawlOptions } from '@/models/datasets'
 type Props = {
 
 }
-
+const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
+  crawl_sub_pages: true,
+  only_main_content: true,
+  includes: '',
+  excludes: '',
+  limit: 100,
+  max_depth: 3,
+}
 const FireCrawl: FC<Props> = () => {
   const { setShowAccountSettingModal } = useModalContext()
   const handleSetting = useCallback(() => {
@@ -17,6 +26,8 @@ const FireCrawl: FC<Props> = () => {
     })
   }, [setShowAccountSettingModal])
 
+  const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
+
   const handleRun = useCallback((url: string) => {
     console.log(url)
   }, [])
@@ -26,8 +37,8 @@ const FireCrawl: FC<Props> = () => {
       <div className='mt-2 p-3 pb-4 rounded-xl border border-gray-200'>
         <UrlInput onRun={handleRun} />
 
-        <OptionsWrap>
-          <div>contents</div>
+        <OptionsWrap className='mt-3 space-y-2'>
+          <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
         </OptionsWrap>
       </div>
     </div>
diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx
index e69de29bb2d1d6..3062ba2d70772e 100644
--- a/web/app/components/datasets/create/website/firecrawl/options.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/options.tsx
@@ -0,0 +1,71 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import cn from 'classnames'
+import CheckboxWithLabel from './base/checkbox-with-label'
+import Field from './base/field'
+import type { CrawlOptions } from '@/models/datasets'
+
+type Props = {
+  className?: string
+  payload: CrawlOptions
+  onChange: (payload: CrawlOptions) => void
+}
+
+const Options: FC<Props> = ({
+  className = '',
+  payload,
+  onChange,
+}) => {
+  const handleChange = useCallback((key: keyof CrawlOptions) => {
+    return (value: any) => {
+      onChange({
+        ...payload,
+        [key]: value,
+      })
+    }
+  }, [payload, onChange])
+  return (
+    <div className={cn(className, ' space-y-2')}>
+      <CheckboxWithLabel label='Crawl subdomains' isChecked={payload.crawl_sub_pages} onChange={handleChange('crawl_sub_pages')} />
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='grow shrink-0'
+          label='Limit'
+          value={payload.limit}
+          onChange={handleChange('limit')}
+          isNumber
+          isRequired
+        />
+        <Field
+          className='grow shrink-0'
+          label='Max depth'
+          value={payload.max_depth}
+          onChange={handleChange('max_depth')}
+          isNumber
+          isRequired
+        />
+      </div>
+
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='grow shrink-0'
+          label='Exclude paths'
+          value={payload.excludes}
+          onChange={handleChange('excludes')}
+          placeholder='blog/*, /about/*'
+        />
+        <Field
+          className='grow shrink-0'
+          label='Include only paths'
+          value={payload.includes}
+          onChange={handleChange('includes')}
+          isNumber
+          placeholder='articles/*'
+        />
+      </div>
+      <CheckboxWithLabel label='Extract only main content (no headers, navs, footers, etc.)' isChecked={payload.only_main_content} onChange={handleChange('only_main_content')} />
+    </div>
+  )
+}
+export default React.memo(Options)
diff --git a/web/models/datasets.ts b/web/models/datasets.ts
index 585c0473d4f31a..8a16b207d13c55 100644
--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@@ -39,6 +39,15 @@ export type CustomFile = File & {
   created_at?: number
 }
 
+export type CrawlOptions = {
+  crawl_sub_pages: boolean
+  only_main_content: boolean
+  includes: string
+  excludes: string
+  limit: number
+  max_depth: number
+}
+
 export type FileItem = {
   fileID: string
   file: CustomFile

From b4a108960f11f0fc264422db8b1e245b6e0f6aa2 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 14:50:28 +0800
Subject: [PATCH 13/31] feat: option styles

---
 .../create/website/firecrawl/base/field.tsx   |  6 ++---
 .../website/firecrawl/base/options-wrap.tsx   |  2 +-
 .../create/website/firecrawl/index.tsx        |  3 +--
 .../create/website/firecrawl/options.tsx      | 26 ++++++++++++++-----
 web/i18n/en-US/dataset-creation.ts            |  6 +++++
 web/i18n/zh-Hans/dataset-creation.ts          |  6 +++++
 6 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
index d9fb18aa48c420..12eaefa9b85f03 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
@@ -25,9 +25,9 @@ const Field: FC<Props> = ({
 }) => {
   return (
     <div className={cn(className)}>
-      <div>
-        <span>{label} </span>
-        {isRequired && <span>*</span>}
+      <div className='flex py-[7px]'>
+        <div className='flex items-center h-[18px] text-[13px] font-medium text-gray-900'>{label} </div>
+        {isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
       </div>
       <Input
         value={value}
diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index ff66cb0e614ca0..62735cfb0ca281 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -32,7 +32,7 @@ const OptionsWrap: FC<Props> = ({
       >
         <div className='flex items-center text-gray-700'>
           <Settings04 className='mr-1 w-4 h-4' />
-          <div>{t(`${I18N_PREFIX}.options`)}</div>
+          <div className='text-[13px] font-semibold text-gray-800 uppercase'>{t(`${I18N_PREFIX}.options`)}</div>
         </div>
         <ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
       </div>
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 1746a1a2450bec..e394ac51f3aea9 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -36,8 +36,7 @@ const FireCrawl: FC<Props> = () => {
       <Header onSetting={handleSetting} />
       <div className='mt-2 p-3 pb-4 rounded-xl border border-gray-200'>
         <UrlInput onRun={handleRun} />
-
-        <OptionsWrap className='mt-3 space-y-2'>
+        <OptionsWrap className='mt-3'>
           <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
         </OptionsWrap>
       </div>
diff --git a/web/app/components/datasets/create/website/firecrawl/options.tsx b/web/app/components/datasets/create/website/firecrawl/options.tsx
index 3062ba2d70772e..b882febd9d3c25 100644
--- a/web/app/components/datasets/create/website/firecrawl/options.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/options.tsx
@@ -2,10 +2,13 @@
 import type { FC } from 'react'
 import React, { useCallback } from 'react'
 import cn from 'classnames'
+import { useTranslation } from 'react-i18next'
 import CheckboxWithLabel from './base/checkbox-with-label'
 import Field from './base/field'
 import type { CrawlOptions } from '@/models/datasets'
 
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
 type Props = {
   className?: string
   payload: CrawlOptions
@@ -17,6 +20,8 @@ const Options: FC<Props> = ({
   payload,
   onChange,
 }) => {
+  const { t } = useTranslation()
+
   const handleChange = useCallback((key: keyof CrawlOptions) => {
     return (value: any) => {
       onChange({
@@ -27,11 +32,15 @@ const Options: FC<Props> = ({
   }, [payload, onChange])
   return (
     <div className={cn(className, ' space-y-2')}>
-      <CheckboxWithLabel label='Crawl subdomains' isChecked={payload.crawl_sub_pages} onChange={handleChange('crawl_sub_pages')} />
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.crawlSubPage`)}
+        isChecked={payload.crawl_sub_pages}
+        onChange={handleChange('crawl_sub_pages')}
+      />
       <div className='flex justify-between space-x-4'>
         <Field
           className='grow shrink-0'
-          label='Limit'
+          label={t(`${I18N_PREFIX}.limit`)}
           value={payload.limit}
           onChange={handleChange('limit')}
           isNumber
@@ -39,7 +48,7 @@ const Options: FC<Props> = ({
         />
         <Field
           className='grow shrink-0'
-          label='Max depth'
+          label={t(`${I18N_PREFIX}.maxDepth`)}
           value={payload.max_depth}
           onChange={handleChange('max_depth')}
           isNumber
@@ -50,21 +59,24 @@ const Options: FC<Props> = ({
       <div className='flex justify-between space-x-4'>
         <Field
           className='grow shrink-0'
-          label='Exclude paths'
+          label={t(`${I18N_PREFIX}.excludePaths`)}
           value={payload.excludes}
           onChange={handleChange('excludes')}
           placeholder='blog/*, /about/*'
         />
         <Field
           className='grow shrink-0'
-          label='Include only paths'
+          label={t(`${I18N_PREFIX}.includeOnlyPaths`)}
           value={payload.includes}
           onChange={handleChange('includes')}
-          isNumber
           placeholder='articles/*'
         />
       </div>
-      <CheckboxWithLabel label='Extract only main content (no headers, navs, footers, etc.)' isChecked={payload.only_main_content} onChange={handleChange('only_main_content')} />
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.extractOnlyMainContent`)}
+        isChecked={payload.only_main_content}
+        onChange={handleChange('only_main_content')}
+      />
     </div>
   )
 }
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 9bb86765520f59..56df52e89df810 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -59,6 +59,12 @@ const translation = {
       firecrawlDoc: 'Firecrawl docs',
       firecrawlDocLink: '',
       options: 'Options',
+      crawlSubPage: 'Crawl sub-pages',
+      limit: 'Limit',
+      maxDepth: 'Max depth',
+      excludePaths: 'Exclude paths',
+      includeOnlyPaths: 'Include only paths',
+      extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index 9c4793c0a393b4..c8ab88147b9906 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -59,6 +59,12 @@ const translation = {
       firecrawlDoc: 'Firecrawl 文档',
       firecrawlDocLink: '',
       options: '选项',
+      crawlSubPage: '爬取子页面',
+      limit: '限制数量',
+      maxDepth: '最大深度',
+      excludePaths: '排除路径',
+      includeOnlyPaths: '仅包含路径',
+      extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）',
     },
   },
   stepTwo: {

From 446f24cb9fea7a198b0e9c6ca10f8f040a571e80 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 15:43:50 +0800
Subject: [PATCH 14/31] feat: valid crawl options

---
 .../create/website/firecrawl/base/input.tsx   | 24 ++++++-
 .../website/firecrawl/base/url-input.tsx      |  8 ++-
 .../create/website/firecrawl/index.tsx        | 67 +++++++++++++++++--
 web/i18n/en-US/common.ts                      |  4 ++
 web/i18n/zh-Hans/common.ts                    |  4 ++
 web/models/datasets.ts                        |  4 +-
 6 files changed, 100 insertions(+), 11 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx
index a042b1460a78ba..06249f57e7469a 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/input.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx
@@ -9,6 +9,8 @@ type Props = {
   isNumber?: boolean
 }
 
+const MIN_VALUE = 1
+
 const Input: FC<Props> = ({
   value,
   onChange,
@@ -18,14 +20,34 @@ const Input: FC<Props> = ({
   const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
     const value = e.target.value
     if (isNumber) {
-      onChange(parseFloat(value))
+      let numberValue = parseInt(value, 10) // integer only
+      if (isNaN(numberValue)) {
+        onChange('')
+        return
+      }
+      if (numberValue < MIN_VALUE)
+        numberValue = MIN_VALUE
+
+      onChange(numberValue)
       return
     }
     onChange(value)
   }, [isNumber, onChange])
+
+  const otherOption = (() => {
+    if (isNumber) {
+      return {
+        min: MIN_VALUE,
+      }
+    }
+    return {
+
+    }
+  })()
   return (
     <input
       type={isNumber ? 'number' : 'text'}
+      {...otherOption}
       value={value}
       onChange={handleChange}
       className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
diff --git a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
index 915db90d5c5a77..1b572eb68e6f41 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
@@ -15,8 +15,10 @@ const UrlInput: FC<Props> = ({
   onRun,
 }) => {
   const { t } = useTranslation()
-  const [url, setUrl] = useState('')
-
+  const [url, setUrl] = useState('https://docs.dify.ai') // TODO: for test
+  const handleUrlChange = useCallback((url: string | number) => {
+    setUrl(url as string)
+  }, [])
   const handleOnRun = useCallback(() => {
     onRun(url)
   }, [onRun, url])
@@ -25,7 +27,7 @@ const UrlInput: FC<Props> = ({
     <div className='flex items-center justify-between'>
       <Input
         value={url}
-        onChange={setUrl}
+        onChange={handleUrlChange}
         placeholder='https://docs.dify.ai'
       />
       <Button
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index e394ac51f3aea9..cca2cdf06d0c4d 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -1,12 +1,18 @@
 'use client'
 import type { FC } from 'react'
 import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
 import Header from './header'
 import UrlInput from './base/url-input'
 import OptionsWrap from './base/options-wrap'
 import Options from './options'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions } from '@/models/datasets'
+import Toast from '@/app/components/base/toast'
+
+const ERROR_I18N_PREFIX = 'common.errorMsg'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
 type Props = {
 
 }
@@ -15,10 +21,13 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
   only_main_content: true,
   includes: '',
   excludes: '',
-  limit: 100,
-  max_depth: 3,
+  limit: 10,
+  max_depth: 2,
 }
+
 const FireCrawl: FC<Props> = () => {
+  const { t } = useTranslation()
+
   const { setShowAccountSettingModal } = useModalContext()
   const handleSetting = useCallback(() => {
     setShowAccountSettingModal({
@@ -27,17 +36,65 @@ const FireCrawl: FC<Props> = () => {
   }, [setShowAccountSettingModal])
 
   const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
+  const checkValid = useCallback((url: string) => {
+    let errorMsg = ''
+    if (!url) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: 'url',
+      })
+    }
+
+    if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
+      errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`)
+
+    if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: t(`${I18N_PREFIX}.limit`),
+      })
+    }
+
+    if (!errorMsg && (crawlOptions.max_depth === null || crawlOptions.max_depth === undefined || crawlOptions.max_depth === '')) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: t(`${I18N_PREFIX}.maxDepth`),
+      })
+    }
+
+    return {
+      isValid: !errorMsg,
+      errorMsg,
+    }
+  }, [crawlOptions, t])
+
+  const [isCrawlFinished, setIsCrawlFinished] = useState(false)
+  const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
 
   const handleRun = useCallback((url: string) => {
-    console.log(url)
-  }, [])
+    const { isValid, errorMsg } = checkValid(url)
+    if (!isValid) {
+      Toast.notify({
+        message: errorMsg!,
+        type: 'error',
+      })
+      return
+    }
+    // TODO: crawl
+    setIsCrawlFinished(true)
+    setCrawlErrorMsg('')
+  }, [checkValid])
+
   return (
     <div>
       <Header onSetting={handleSetting} />
       <div className='mt-2 p-3 pb-4 rounded-xl border border-gray-200'>
         <UrlInput onRun={handleRun} />
         <OptionsWrap className='mt-3'>
-          <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
+          {!isCrawlFinished
+            ? (
+              <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
+            )
+            : (
+              <div>Result list</div>
+            )}
         </OptionsWrap>
       </div>
     </div>
diff --git a/web/i18n/en-US/common.ts b/web/i18n/en-US/common.ts
index c0a64cc0b6e56c..3047ce162016cc 100644
--- a/web/i18n/en-US/common.ts
+++ b/web/i18n/en-US/common.ts
@@ -37,6 +37,10 @@ const translation = {
     duplicate: 'Duplicate',
     rename: 'Rename',
   },
+  errorMsg: {
+    fieldRequired: '{{field}} is required',
+    urlError: 'url should start with http:// or https://',
+  },
   placeholder: {
     input: 'Please enter',
     select: 'Please select',
diff --git a/web/i18n/zh-Hans/common.ts b/web/i18n/zh-Hans/common.ts
index c3e82cb7aaa488..576eec7878466d 100644
--- a/web/i18n/zh-Hans/common.ts
+++ b/web/i18n/zh-Hans/common.ts
@@ -37,6 +37,10 @@ const translation = {
     duplicate: '复制',
     rename: '重命名',
   },
+  errorMsg: {
+    fieldRequired: '{{field}} 为必填项',
+    urlError: 'url 应该以 http:// 或 https:// 开头',
+  },
   placeholder: {
     input: '请输入',
     select: '请选择',
diff --git a/web/models/datasets.ts b/web/models/datasets.ts
index 8a16b207d13c55..d3b752aaf576af 100644
--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@@ -44,8 +44,8 @@ export type CrawlOptions = {
   only_main_content: boolean
   includes: string
   excludes: string
-  limit: number
-  max_depth: number
+  limit: number | string
+  max_depth: number | string
 }
 
 export type FileItem = {

From 33d153364a83907adcc023f3705871d8e9a2d779 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 16:14:07 +0800
Subject: [PATCH 15/31] feat: error info

---
 .../website/firecrawl/base/error-message.tsx  | 28 +++++++++++++++++++
 .../website/firecrawl/base/options-wrap.tsx   |  5 ++--
 .../create/website/firecrawl/index.tsx        | 12 +++++---
 .../website/firecrawl/options-error.tsx       |  0
 web/i18n/en-US/dataset-creation.ts            |  1 +
 web/i18n/zh-Hans/dataset-creation.ts          |  2 ++
 6 files changed, 41 insertions(+), 7 deletions(-)
 create mode 100644 web/app/components/datasets/create/website/firecrawl/base/error-message.tsx
 delete mode 100644 web/app/components/datasets/create/website/firecrawl/options-error.tsx

diff --git a/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx b/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx
new file mode 100644
index 00000000000000..6e78286fe69a7d
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx
@@ -0,0 +1,28 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+
+type Props = {
+  className?: string
+  title: string
+  errorMsg: string
+}
+
+const ErrorMessage: FC<Props> = ({
+  className,
+  title,
+  errorMsg,
+}) => {
+  return (
+    <div className={cn(className, 'py-2 px-4 border-t border-gray-200 bg-[#FFFAEB]')}>
+      <div className='flex items-center h-5'>
+        <AlertTriangle className='mr-2 w-4 h-4 text-[#F79009]' />
+        <div className='text-sm font-medium text-[#DC6803]'>{title}</div>
+      </div>
+      <div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
+    </div>
+  )
+}
+export default React.memo(ErrorMessage)
diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index 62735cfb0ca281..8a27012ebb5227 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -4,6 +4,7 @@ import type { FC } from 'react'
 import React from 'react'
 import { useTranslation } from 'react-i18next'
 import cn from 'classnames'
+import ErrorMessage from './error-message'
 import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
 import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -41,9 +42,7 @@ const OptionsWrap: FC<Props> = ({
           {!errorMsg
             ? children
             : (
-              <div >
-                {errorMsg}
-              </div>
+              <ErrorMessage className='mt-3 relative left-[-12px] w-[calc(100%_+_24px)] rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={errorMsg} />
             )}
         </div>
       )}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index cca2cdf06d0c4d..271c2d1064dbfa 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -2,6 +2,7 @@
 import type { FC } from 'react'
 import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
 import Header from './header'
 import UrlInput from './base/url-input'
 import OptionsWrap from './base/options-wrap'
@@ -13,6 +14,8 @@ import Toast from '@/app/components/base/toast'
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
+// const testCrawlErrorMsg = 'Firecrawl currently does not support social media scraping due to policy restrictions. We are actively working on building support for it.'
+
 type Props = {
 
 }
@@ -65,10 +68,11 @@ const FireCrawl: FC<Props> = () => {
     }
   }, [crawlOptions, t])
 
-  const [isCrawlFinished, setIsCrawlFinished] = useState(false)
+  const [isCrawlFinished, setIsCrawlFinished] = useState(true)
   const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
-
+  const showCrawlError = isCrawlFinished && !!crawlErrorMsg
   const handleRun = useCallback((url: string) => {
+    setIsCrawlFinished(false)
     const { isValid, errorMsg } = checkValid(url)
     if (!isValid) {
       Toast.notify({
@@ -85,9 +89,9 @@ const FireCrawl: FC<Props> = () => {
   return (
     <div>
       <Header onSetting={handleSetting} />
-      <div className='mt-2 p-3 pb-4 rounded-xl border border-gray-200'>
+      <div className={cn(!showCrawlError ? 'pb-4' : 'pb-0', 'mt-2 p-3 rounded-xl border border-gray-200')}>
         <UrlInput onRun={handleRun} />
-        <OptionsWrap className='mt-3'>
+        <OptionsWrap className='mt-3' errorMsg={isCrawlFinished ? crawlErrorMsg : ''} >
           {!isCrawlFinished
             ? (
               <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
diff --git a/web/app/components/datasets/create/website/firecrawl/options-error.tsx b/web/app/components/datasets/create/website/firecrawl/options-error.tsx
deleted file mode 100644
index e69de29bb2d1d6..00000000000000
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 56df52e89df810..b059b6f0d1e885 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -65,6 +65,7 @@ const translation = {
       excludePaths: 'Exclude paths',
       includeOnlyPaths: 'Include only paths',
       extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
+      exceptionErrorTitle: 'An exception occurred while running Firecrawl job:',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index c8ab88147b9906..23df0a9d537732 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -65,6 +65,8 @@ const translation = {
       excludePaths: '排除路径',
       includeOnlyPaths: '仅包含路径',
       extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）',
+      exceptionErrorTitle: '运行 Firecrawl 时发生异常:',
+
     },
   },
   stepTwo: {

From a278e922f9fb77f40805f57df9b2fec6fca15035 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 16:55:31 +0800
Subject: [PATCH 16/31] feat: crawl options struct

---
 .../website/firecrawl/crawled-result-item.tsx | 32 ++++++++++
 .../website/firecrawl/crawled-result.tsx      | 58 +++++++++++++++++++
 .../create/website/firecrawl/index.tsx        | 13 ++++-
 .../website/firecrawl/mock-crawl-result.ts    | 24 ++++++++
 web/models/datasets.ts                        |  7 +++
 5 files changed, 132 insertions(+), 2 deletions(-)
 create mode 100644 web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts

diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx
new file mode 100644
index 00000000000000..6e3735186f19a6
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx
@@ -0,0 +1,32 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
+import Checkbox from '@/app/components/base/checkbox'
+
+type Props = {
+  payload: CrawlResultItemType
+  isChecked: boolean
+  onCheckChange: (checked: boolean) => void
+}
+
+const CrawledResultItem: FC<Props> = ({
+  payload,
+  isChecked,
+  onCheckChange,
+}) => {
+  const handleCheckChange = useCallback(() => {
+    onCheckChange(!isChecked)
+  }, [isChecked, onCheckChange])
+  return (
+    <div className='rounded-md px-2 py-[5px] hover:bg-gray-100'>
+      <div className='flex items-center h-5'>
+        <Checkbox className='shrink-0 self-start ' checked={isChecked} onCheck={handleCheckChange} />
+        <div className='grow w-0 truncate text-sm font-medium text-gray-700' title={payload.title}>{payload.title}</div>
+        <div className='px-2 text-xs font-medium text-gray-500 uppercase'>Preview</div>
+      </div>
+      <div className='mt-2 truncate pl-6' title={payload.source_url}>{payload.source_url}</div>
+    </div>
+  )
+}
+export default React.memo(CrawledResultItem)
diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
new file mode 100644
index 00000000000000..9df4b2b521679c
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
@@ -0,0 +1,58 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import CheckboxWithLabel from './base/checkbox-with-label'
+import CrawledResultItem from './crawled-result-item'
+import type { CrawlResultItem } from '@/models/datasets'
+
+type Props = {
+  list: CrawlResultItem[]
+  checkedList: CrawlResultItem[]
+  onSelectedChange: (selected: CrawlResultItem[]) => void
+}
+
+const CrawledResult: FC<Props> = ({
+  list,
+  checkedList,
+  onSelectedChange,
+}) => {
+  const isCheckAll = checkedList.length === list.length
+
+  const handleCheckedAll = useCallback(() => {
+    if (!isCheckAll)
+      onSelectedChange(list)
+
+    else
+      onSelectedChange([])
+  }, [isCheckAll, list, onSelectedChange])
+
+  const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
+    return (checked: boolean) => {
+      if (checked)
+        onSelectedChange([...checkedList, item])
+
+      else
+        onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
+    }
+  }, [checkedList, onSelectedChange])
+
+  return (
+    <div>
+      <div className='flex items-center justify-between'>
+        <CheckboxWithLabel isChecked={isCheckAll} onChange={handleCheckedAll} label={isCheckAll ? 'Reset All' : 'Select All'} />
+        <div className=''>Scraped 10 pages in total within 12.4 seconds</div>
+      </div>
+      <div>
+        {list.map(item => (
+          <CrawledResultItem
+            key={item.source_url}
+            payload={item}
+            isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
+            onCheckChange={handleItemCheckChange(item)}
+          />
+        ))}
+      </div>
+    </div>
+  )
+}
+export default React.memo(CrawledResult)
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 271c2d1064dbfa..fe640f1d25bea9 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -7,8 +7,10 @@ import Header from './header'
 import UrlInput from './base/url-input'
 import OptionsWrap from './base/options-wrap'
 import Options from './options'
+import mockCrawlResult from './mock-crawl-result'
+import CrawledResult from './crawled-result'
 import { useModalContext } from '@/context/modal-context'
-import type { CrawlOptions } from '@/models/datasets'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -69,6 +71,9 @@ const FireCrawl: FC<Props> = () => {
   }, [crawlOptions, t])
 
   const [isCrawlFinished, setIsCrawlFinished] = useState(true)
+  const [crawlResult, setCrawlResult] = useState<CrawlResultItem[]>(mockCrawlResult)
+  const [checkedCrawlResult, setCheckedCrawlResult] = useState<CrawlResultItem[]>([])
+
   const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
   const showCrawlError = isCrawlFinished && !!crawlErrorMsg
   const handleRun = useCallback((url: string) => {
@@ -97,7 +102,11 @@ const FireCrawl: FC<Props> = () => {
               <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
             )
             : (
-              <div>Result list</div>
+              <CrawledResult
+                list={crawlResult}
+                checkedList={checkedCrawlResult}
+                onSelectedChange={setCheckedCrawlResult}
+              />
             )}
         </OptionsWrap>
       </div>
diff --git a/web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts b/web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts
new file mode 100644
index 00000000000000..8fd5e6636f1617
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/mock-crawl-result.ts
@@ -0,0 +1,24 @@
+import type { CrawlResultItem } from '@/models/datasets'
+
+const result: CrawlResultItem[] = [
+  {
+    title: 'Start the frontend Docker container separately',
+    markdown: 'Markdown 1',
+    description: 'Description 1',
+    source_url: 'https://example.com/1',
+  },
+  {
+    title: 'Advanced Tool Integration',
+    markdown: 'Markdown 2',
+    description: 'Description 2',
+    source_url: 'https://example.com/2',
+  },
+  {
+    title: 'Local Source Code Start | English | Dify',
+    markdown: 'Markdown 3',
+    description: 'Description 3',
+    source_url: 'https://example.com/3',
+  },
+]
+
+export default result
diff --git a/web/models/datasets.ts b/web/models/datasets.ts
index d3b752aaf576af..1edb63748faadd 100644
--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@@ -48,6 +48,13 @@ export type CrawlOptions = {
   max_depth: number | string
 }
 
+export type CrawlResultItem = {
+  title: string
+  markdown: string
+  description: string
+  source_url: string
+}
+
 export type FileItem = {
   fileID: string
   file: CustomFile

From eff80c8d27a3d6b12c141bb69bd477c7aa6e4dee Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 18:20:29 +0800
Subject: [PATCH 17/31] feat: scrawl loading

---
 .../icons/assets/public/other/row-struct.svg  |  5 ++
 .../icons/src/public/other/RowStruct.json     | 56 ++++++++++++++++++
 .../base/icons/src/public/other/RowStruct.tsx | 16 +++++
 .../base/icons/src/public/other/index.ts      |  1 +
 .../website/firecrawl/base/options-wrap.tsx   |  6 +-
 .../website/firecrawl/base/url-input.tsx      |  9 ++-
 .../create/website/firecrawl/crawling.tsx     | 37 ++++++++++++
 .../create/website/firecrawl/index.tsx        | 59 +++++++++++++------
 web/i18n/en-US/dataset-creation.ts            |  1 +
 web/i18n/zh-Hans/dataset-creation.ts          |  2 +-
 10 files changed, 168 insertions(+), 24 deletions(-)
 create mode 100644 web/app/components/base/icons/assets/public/other/row-struct.svg
 create mode 100644 web/app/components/base/icons/src/public/other/RowStruct.json
 create mode 100644 web/app/components/base/icons/src/public/other/RowStruct.tsx
 create mode 100644 web/app/components/datasets/create/website/firecrawl/crawling.tsx

diff --git a/web/app/components/base/icons/assets/public/other/row-struct.svg b/web/app/components/base/icons/assets/public/other/row-struct.svg
new file mode 100644
index 00000000000000..ba275ffeec9079
--- /dev/null
+++ b/web/app/components/base/icons/assets/public/other/row-struct.svg
@@ -0,0 +1,5 @@
+<svg width="624" height="48" viewBox="0 0 624 48" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="8" y="7" width="16" height="16" rx="5" fill="#F2F4F7"/>
+<rect x="32" y="10" width="233" height="10" rx="3" fill="#EAECF0"/>
+<rect x="32" y="31" width="345" height="6" rx="3" fill="#F2F4F7"/>
+</svg>
diff --git a/web/app/components/base/icons/src/public/other/RowStruct.json b/web/app/components/base/icons/src/public/other/RowStruct.json
new file mode 100644
index 00000000000000..0d1ef43f4f5d5d
--- /dev/null
+++ b/web/app/components/base/icons/src/public/other/RowStruct.json
@@ -0,0 +1,56 @@
+{
+	"icon": {
+		"type": "element",
+		"isRootNode": true,
+		"name": "svg",
+		"attributes": {
+			"width": "624",
+			"height": "48",
+			"viewBox": "0 0 624 48",
+			"fill": "none",
+			"xmlns": "http://www.w3.org/2000/svg"
+		},
+		"children": [
+			{
+				"type": "element",
+				"name": "rect",
+				"attributes": {
+					"x": "8",
+					"y": "7",
+					"width": "16",
+					"height": "16",
+					"rx": "5",
+					"fill": "#F2F4F7"
+				},
+				"children": []
+			},
+			{
+				"type": "element",
+				"name": "rect",
+				"attributes": {
+					"x": "32",
+					"y": "10",
+					"width": "233",
+					"height": "10",
+					"rx": "3",
+					"fill": "#EAECF0"
+				},
+				"children": []
+			},
+			{
+				"type": "element",
+				"name": "rect",
+				"attributes": {
+					"x": "32",
+					"y": "31",
+					"width": "345",
+					"height": "6",
+					"rx": "3",
+					"fill": "#F2F4F7"
+				},
+				"children": []
+			}
+		]
+	},
+	"name": "RowStruct"
+}
\ No newline at end of file
diff --git a/web/app/components/base/icons/src/public/other/RowStruct.tsx b/web/app/components/base/icons/src/public/other/RowStruct.tsx
new file mode 100644
index 00000000000000..ef5ab8c62df79a
--- /dev/null
+++ b/web/app/components/base/icons/src/public/other/RowStruct.tsx
@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './RowStruct.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'RowStruct'
+
+export default Icon
diff --git a/web/app/components/base/icons/src/public/other/index.ts b/web/app/components/base/icons/src/public/other/index.ts
index adf723edb7db55..257ba59b0d0131 100644
--- a/web/app/components/base/icons/src/public/other/index.ts
+++ b/web/app/components/base/icons/src/public/other/index.ts
@@ -1,2 +1,3 @@
 export { default as Icon3Dots } from './Icon3Dots'
 export { default as DefaultToolIcon } from './DefaultToolIcon'
+export { default as RowStruct } from './RowStruct'
diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index 8a27012ebb5227..af427249aa8291 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -12,6 +12,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 type Props = {
   className?: string
   children: React.ReactNode
+  isFilledFull?: boolean
   errorMsg?: string
 }
 
@@ -19,6 +20,7 @@ const OptionsWrap: FC<Props> = ({
   className = '',
   children,
   errorMsg,
+  isFilledFull = false,
 }) => {
   const { t } = useTranslation()
 
@@ -38,11 +40,11 @@ const OptionsWrap: FC<Props> = ({
         <ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
       </div>
       {!fold && (
-        <div>
+        <div className={cn(isFilledFull && 'mt-3 relative left-[-12px] w-[calc(100%_+_24px)] rounded-b-xl')}>
           {!errorMsg
             ? children
             : (
-              <ErrorMessage className='mt-3 relative left-[-12px] w-[calc(100%_+_24px)] rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={errorMsg} />
+              <ErrorMessage title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={errorMsg} />
             )}
         </div>
       )}
diff --git a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
index 1b572eb68e6f41..90acc769be6ce8 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/url-input.tsx
@@ -8,10 +8,12 @@ import Button from '@/app/components/base/button'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 type Props = {
+  isRunning: boolean
   onRun: (url: string) => void
 }
 
 const UrlInput: FC<Props> = ({
+  isRunning,
   onRun,
 }) => {
   const { t } = useTranslation()
@@ -20,8 +22,10 @@ const UrlInput: FC<Props> = ({
     setUrl(url as string)
   }, [])
   const handleOnRun = useCallback(() => {
+    if (isRunning)
+      return
     onRun(url)
-  }, [onRun, url])
+  }, [isRunning, onRun, url])
 
   return (
     <div className='flex items-center justify-between'>
@@ -34,8 +38,9 @@ const UrlInput: FC<Props> = ({
         type='primary'
         onClick={handleOnRun}
         className='ml-2 !h-8 text-[13px] font-medium'
+        loading={isRunning}
       >
-        {t(`${I18N_PREFIX}.run`)}
+        {!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
       </Button>
     </div>
   )
diff --git a/web/app/components/datasets/create/website/firecrawl/crawling.tsx b/web/app/components/datasets/create/website/firecrawl/crawling.tsx
new file mode 100644
index 00000000000000..97b2b01d2eeb61
--- /dev/null
+++ b/web/app/components/datasets/create/website/firecrawl/crawling.tsx
@@ -0,0 +1,37 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import cn from 'classnames'
+import { useTranslation } from 'react-i18next'
+import { RowStruct } from '@/app/components/base/icons/src/public/other'
+
+type Props = {
+  className?: string
+  crawledNum: number
+  totalNum: number
+}
+
+const Crawling: FC<Props> = ({
+  className = '',
+  crawledNum,
+  totalNum,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className={cn(className, 'border-t border-gray-200')}>
+      <div className='flex items-center h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
+        {t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
+      </div>
+
+      <div className='p-2'>
+        {['', '', '', ''].map((item, index) => (
+          <div className='py-[5px]' key={index}>
+            <RowStruct />
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+export default React.memo(Crawling)
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index fe640f1d25bea9..234ecc4e4b629e 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -9,9 +9,11 @@ import OptionsWrap from './base/options-wrap'
 import Options from './options'
 import mockCrawlResult from './mock-crawl-result'
 import CrawledResult from './crawled-result'
+import Crawling from './crawling'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
+import { sleep } from '@/utils'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -30,8 +32,15 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
   max_depth: 2,
 }
 
+enum Step {
+  init = 'init',
+  running = 'running',
+  finished = 'finished',
+}
+
 const FireCrawl: FC<Props> = () => {
   const { t } = useTranslation()
+  const [step, setStep] = useState<Step>(Step.running)
 
   const { setShowAccountSettingModal } = useModalContext()
   const handleSetting = useCallback(() => {
@@ -70,14 +79,15 @@ const FireCrawl: FC<Props> = () => {
     }
   }, [crawlOptions, t])
 
-  const [isCrawlFinished, setIsCrawlFinished] = useState(true)
+  const isInit = step === Step.init
+  const isCrawlFinished = step === Step.finished
+  const isRunning = step === Step.running
   const [crawlResult, setCrawlResult] = useState<CrawlResultItem[]>(mockCrawlResult)
   const [checkedCrawlResult, setCheckedCrawlResult] = useState<CrawlResultItem[]>([])
 
   const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
-  const showCrawlError = isCrawlFinished && !!crawlErrorMsg
-  const handleRun = useCallback((url: string) => {
-    setIsCrawlFinished(false)
+  const showCrawlError = step === Step.finished && !!crawlErrorMsg
+  const handleRun = useCallback(async (url: string) => {
     const { isValid, errorMsg } = checkValid(url)
     if (!isValid) {
       Toast.notify({
@@ -86,28 +96,39 @@ const FireCrawl: FC<Props> = () => {
       })
       return
     }
+    setStep(Step.running)
     // TODO: crawl
-    setIsCrawlFinished(true)
+    await sleep(2000)
+    setCrawlResult(mockCrawlResult) // TODO:
+
+    setStep(Step.finished)
     setCrawlErrorMsg('')
   }, [checkValid])
 
   return (
     <div>
       <Header onSetting={handleSetting} />
-      <div className={cn(!showCrawlError ? 'pb-4' : 'pb-0', 'mt-2 p-3 rounded-xl border border-gray-200')}>
-        <UrlInput onRun={handleRun} />
-        <OptionsWrap className='mt-3' errorMsg={isCrawlFinished ? crawlErrorMsg : ''} >
-          {!isCrawlFinished
-            ? (
-              <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />
-            )
-            : (
-              <CrawledResult
-                list={crawlResult}
-                checkedList={checkedCrawlResult}
-                onSelectedChange={setCheckedCrawlResult}
-              />
-            )}
+      <div className={cn(isInit ? 'pb-4' : 'pb-0', 'mt-2 p-3 rounded-xl border border-gray-200')}>
+        <UrlInput onRun={handleRun} isRunning={isRunning} />
+        <OptionsWrap
+          className={cn('mt-3')}
+          isFilledFull={!isInit}
+          errorMsg={isCrawlFinished ? crawlErrorMsg : ''}
+        >
+          {isInit && <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />}
+          {isRunning
+            && <Crawling
+              className='mt-2'
+              crawledNum={8}
+              totalNum={10}
+            />}
+          {isCrawlFinished && (
+            <CrawledResult
+              list={crawlResult}
+              checkedList={checkedCrawlResult}
+              onSelectedChange={setCheckedCrawlResult}
+            />
+          )}
         </OptionsWrap>
       </div>
     </div>
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index b059b6f0d1e885..87b1e57862f812 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -66,6 +66,7 @@ const translation = {
       includeOnlyPaths: 'Include only paths',
       extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
       exceptionErrorTitle: 'An exception occurred while running Firecrawl job:',
+      totalPageScraped: 'Total pages scraped:',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index 23df0a9d537732..a0be0f0cc100a2 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -66,7 +66,7 @@ const translation = {
       includeOnlyPaths: '仅包含路径',
       extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）',
       exceptionErrorTitle: '运行 Firecrawl 时发生异常:',
-
+      totalPageScraped: '抓取页面总数:',
     },
   },
   stepTwo: {

From 0448c49c7ac8a8a3f89f0ea71242d05466904cd8 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Wed, 5 Jun 2024 18:33:40 +0800
Subject: [PATCH 18/31] feat: result

---
 .../firecrawl/base/checkbox-with-label.tsx    |  4 +++-
 .../website/firecrawl/base/options-wrap.tsx   |  2 +-
 .../website/firecrawl/crawled-result.tsx      | 22 ++++++++++++++-----
 .../create/website/firecrawl/index.tsx        |  3 +--
 web/i18n/en-US/dataset-creation.ts            |  3 +++
 web/i18n/zh-Hans/dataset-creation.ts          |  3 +++
 6 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx
index f25e45724512e7..ed5d2efd511711 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/checkbox-with-label.tsx
@@ -9,6 +9,7 @@ type Props = {
   isChecked: boolean
   onChange: (isChecked: boolean) => void
   label: string
+  labelClassName?: string
 }
 
 const CheckboxWithLabel: FC<Props> = ({
@@ -16,11 +17,12 @@ const CheckboxWithLabel: FC<Props> = ({
   isChecked,
   onChange,
   label,
+  labelClassName,
 }) => {
   return (
     <label className={cn(className, 'flex items-center h-7 space-x-2')}>
       <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
-      <div className='text-sm font-normal text-gray-800'>{label}</div>
+      <div className={cn(labelClassName, 'text-sm font-normal text-gray-800')}>{label}</div>
     </label>
   )
 }
diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index af427249aa8291..61c89fc10745df 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -44,7 +44,7 @@ const OptionsWrap: FC<Props> = ({
           {!errorMsg
             ? children
             : (
-              <ErrorMessage title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={errorMsg} />
+              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={errorMsg} />
             )}
         </div>
       )}
diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
index 9df4b2b521679c..48cca13b93ad3b 100644
--- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
@@ -1,10 +1,13 @@
 'use client'
 import type { FC } from 'react'
 import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
 import CheckboxWithLabel from './base/checkbox-with-label'
 import CrawledResultItem from './crawled-result-item'
 import type { CrawlResultItem } from '@/models/datasets'
 
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
 type Props = {
   list: CrawlResultItem[]
   checkedList: CrawlResultItem[]
@@ -16,6 +19,8 @@ const CrawledResult: FC<Props> = ({
   checkedList,
   onSelectedChange,
 }) => {
+  const { t } = useTranslation()
+
   const isCheckAll = checkedList.length === list.length
 
   const handleCheckedAll = useCallback(() => {
@@ -37,12 +42,19 @@ const CrawledResult: FC<Props> = ({
   }, [checkedList, onSelectedChange])
 
   return (
-    <div>
-      <div className='flex items-center justify-between'>
-        <CheckboxWithLabel isChecked={isCheckAll} onChange={handleCheckedAll} label={isCheckAll ? 'Reset All' : 'Select All'} />
-        <div className=''>Scraped 10 pages in total within 12.4 seconds</div>
+    <div className='border-t border-gray-200'>
+      <div className='flex items-center justify-between h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
+        <CheckboxWithLabel
+          isChecked={isCheckAll}
+          onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
+          labelClassName='!font-medium'
+        />
+        <div>{t(`${I18N_PREFIX}.scrapTimeInfo`, {
+          total: list.length,
+          time: '12.4 seconds',
+        })}</div>
       </div>
-      <div>
+      <div className='p-2'>
         {list.map(item => (
           <CrawledResultItem
             key={item.source_url}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 234ecc4e4b629e..7c27d96803a84e 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -40,7 +40,7 @@ enum Step {
 
 const FireCrawl: FC<Props> = () => {
   const { t } = useTranslation()
-  const [step, setStep] = useState<Step>(Step.running)
+  const [step, setStep] = useState<Step>(Step.finished)
 
   const { setShowAccountSettingModal } = useModalContext()
   const handleSetting = useCallback(() => {
@@ -86,7 +86,6 @@ const FireCrawl: FC<Props> = () => {
   const [checkedCrawlResult, setCheckedCrawlResult] = useState<CrawlResultItem[]>([])
 
   const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
-  const showCrawlError = step === Step.finished && !!crawlErrorMsg
   const handleRun = useCallback(async (url: string) => {
     const { isValid, errorMsg } = checkValid(url)
     if (!isValid) {
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 87b1e57862f812..50b8669b4b25ce 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -67,6 +67,9 @@ const translation = {
       extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
       exceptionErrorTitle: 'An exception occurred while running Firecrawl job:',
       totalPageScraped: 'Total pages scraped:',
+      selectAll: 'Select All',
+      resetAll: 'Reset All',
+      scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index a0be0f0cc100a2..e801c44261ce5d 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -67,6 +67,9 @@ const translation = {
       extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）',
       exceptionErrorTitle: '运行 Firecrawl 时发生异常:',
       totalPageScraped: '抓取页面总数:',
+      selectAll: '全选',
+      resetAll: '重置全部',
+      scrapTimeInfo: '总共在 {{time}} 内抓取了 {{total}} 个页面',
     },
   },
   stepTwo: {

From c8294ac6e55a51a0a9972fbb3fba3a77022834f5 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 11:10:31 +0800
Subject: [PATCH 19/31] feat: crawl result

---
 .../website/firecrawl/crawled-result-item.tsx    | 16 ++++++++++++----
 .../create/website/firecrawl/crawled-result.tsx  | 11 ++++++++++-
 web/i18n/en-US/dataset-creation.ts               |  1 +
 web/i18n/zh-Hans/dataset-creation.ts             |  1 +
 4 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx
index 6e3735186f19a6..1730314b477924 100644
--- a/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/crawled-result-item.tsx
@@ -1,31 +1,39 @@
 'use client'
 import type { FC } from 'react'
 import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
 import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
 import Checkbox from '@/app/components/base/checkbox'
 
 type Props = {
   payload: CrawlResultItemType
   isChecked: boolean
+  isPreview: boolean
   onCheckChange: (checked: boolean) => void
+  onPreview: () => void
 }
 
 const CrawledResultItem: FC<Props> = ({
+  isPreview,
   payload,
   isChecked,
   onCheckChange,
+  onPreview,
 }) => {
+  const { t } = useTranslation()
+
   const handleCheckChange = useCallback(() => {
     onCheckChange(!isChecked)
   }, [isChecked, onCheckChange])
   return (
-    <div className='rounded-md px-2 py-[5px] hover:bg-gray-100'>
+    <div className={cn(isPreview ? 'border-[#D1E0FF] bg-primary-50 shadow-xs' : 'group hover:bg-gray-100', 'rounded-md px-2 py-[5px] cursor-pointer border border-transparent')}>
       <div className='flex items-center h-5'>
-        <Checkbox className='shrink-0 self-start ' checked={isChecked} onCheck={handleCheckChange} />
+        <Checkbox className='group-hover:border-2 group-hover:border-primary-600 mr-2 shrink-0' checked={isChecked} onCheck={handleCheckChange} />
         <div className='grow w-0 truncate text-sm font-medium text-gray-700' title={payload.title}>{payload.title}</div>
-        <div className='px-2 text-xs font-medium text-gray-500 uppercase'>Preview</div>
+        <div onClick={onPreview} className='hidden group-hover:flex items-center h-6 px-2 text-xs rounded-md font-medium text-gray-500 uppercase hover:bg-gray-50'>{t('datasetCreation.stepOne.website.preview')}</div>
       </div>
-      <div className='mt-2 truncate pl-6' title={payload.source_url}>{payload.source_url}</div>
+      <div className='mt-0.5 truncate pl-6 leading-[18px] text-xs font-normal text-gray-500' title={payload.source_url}>{payload.source_url}</div>
     </div>
   )
 }
diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
index 48cca13b93ad3b..5e92ce45a1bc1a 100644
--- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
@@ -41,6 +41,13 @@ const CrawledResult: FC<Props> = ({
     }
   }, [checkedList, onSelectedChange])
 
+  const [previewIndex, setPreviewIndex] = React.useState<number>(-1)
+  const handlePreview = useCallback((index: number) => {
+    return () => {
+      setPreviewIndex(index)
+    }
+  }, [])
+
   return (
     <div className='border-t border-gray-200'>
       <div className='flex items-center justify-between h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
@@ -55,9 +62,11 @@ const CrawledResult: FC<Props> = ({
         })}</div>
       </div>
       <div className='p-2'>
-        {list.map(item => (
+        {list.map((item, index) => (
           <CrawledResultItem
             key={item.source_url}
+            isPreview={index === previewIndex}
+            onPreview={handlePreview(index)}
             payload={item}
             isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
             onCheckChange={handleItemCheckChange(item)}
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 50b8669b4b25ce..baa32587d92582 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -70,6 +70,7 @@ const translation = {
       selectAll: 'Select All',
       resetAll: 'Reset All',
       scrapTimeInfo: 'Scraped {{total}} pages in total within {{time}}',
+      preview: 'Preview',
     },
   },
   stepTwo: {
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index e801c44261ce5d..c8866fbd3b2d1b 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -70,6 +70,7 @@ const translation = {
       selectAll: '全选',
       resetAll: '重置全部',
       scrapTimeInfo: '总共在 {{time}} 内抓取了 {{total}} 个页面',
+      preview: '预览',
     },
   },
   stepTwo: {

From 74c24ccceddc06015c6d6b0beb025e88620ed299 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 14:49:22 +0800
Subject: [PATCH 20/31] feat: handle preview

---
 .../datasets/create/step-one/index.tsx        | 21 +++++++++-
 .../website/firecrawl/crawled-result.tsx      |  5 ++-
 .../create/website/firecrawl/index.tsx        |  8 +++-
 .../datasets/create/website/index.tsx         |  9 ++--
 .../datasets/create/website/preview.tsx       | 41 +++++++++++++++++++
 5 files changed, 76 insertions(+), 8 deletions(-)
 create mode 100644 web/app/components/datasets/create/website/preview.tsx

diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx
index 7d46d5eb5d0bc1..dc1fd24ae2f5db 100644
--- a/web/app/components/datasets/create/step-one/index.tsx
+++ b/web/app/components/datasets/create/step-one/index.tsx
@@ -7,8 +7,9 @@ import FileUploader from '../file-uploader'
 import NotionPagePreview from '../notion-page-preview'
 import EmptyDatasetCreationModal from '../empty-dataset-creation-modal'
 import Website from '../website'
+import WebsitePreview from '../website/preview'
 import s from './index.module.css'
-import type { FileItem } from '@/models/datasets'
+import type { CrawlResultItem, FileItem } from '@/models/datasets'
 import type { NotionPage } from '@/models/common'
 import { DataSourceType } from '@/models/datasets'
 import Button from '@/app/components/base/button'
@@ -66,6 +67,7 @@ const StepOne = ({
   const [showModal, setShowModal] = useState(false)
   const [currentFile, setCurrentFile] = useState<File | undefined>()
   const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
+  const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
   const { t } = useTranslation()
 
   const modalShowHandle = () => setShowModal(true)
@@ -86,6 +88,10 @@ const StepOne = ({
     setCurrentNotionPage(undefined)
   }
 
+  const hideWebsitePreview = () => {
+    setCurrentWebsite(undefined)
+  }
+
   const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type)
 
   const { plan, enableBilling } = useProviderContext()
@@ -202,7 +208,17 @@ const StepOne = ({
             </>
           )}
           {dataSourceType === DataSourceType.WEB && (
-            <Website />
+            <>
+              <div className='mb-8 w-[640px]'>
+                <Website onPreview={setCurrentWebsite} />
+              </div>
+              {isShowVectorSpaceFull && (
+                <div className='max-w-[640px] mb-4'>
+                  <VectorSpaceFull />
+                </div>
+              )}
+              <Button disabled={isShowVectorSpaceFull || !notionPages.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
+            </>
           )}
           {!datasetId && (
             <>
@@ -215,6 +231,7 @@ const StepOne = ({
       </div>
       {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
       {currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
+      {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
     </div>
   )
 }
diff --git a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
index 5e92ce45a1bc1a..bf451e2eaa8a71 100644
--- a/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/crawled-result.tsx
@@ -12,12 +12,14 @@ type Props = {
   list: CrawlResultItem[]
   checkedList: CrawlResultItem[]
   onSelectedChange: (selected: CrawlResultItem[]) => void
+  onPreview: (payload: CrawlResultItem) => void
 }
 
 const CrawledResult: FC<Props> = ({
   list,
   checkedList,
   onSelectedChange,
+  onPreview,
 }) => {
   const { t } = useTranslation()
 
@@ -45,8 +47,9 @@ const CrawledResult: FC<Props> = ({
   const handlePreview = useCallback((index: number) => {
     return () => {
       setPreviewIndex(index)
+      onPreview(list[index])
     }
-  }, [])
+  }, [list, onPreview])
 
   return (
     <div className='border-t border-gray-200'>
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 7c27d96803a84e..1007551978d09e 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -21,8 +21,9 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 // const testCrawlErrorMsg = 'Firecrawl currently does not support social media scraping due to policy restrictions. We are actively working on building support for it.'
 
 type Props = {
-
+  onPreview: (payload: CrawlResultItem) => void
 }
+
 const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
   crawl_sub_pages: true,
   only_main_content: true,
@@ -38,7 +39,9 @@ enum Step {
   finished = 'finished',
 }
 
-const FireCrawl: FC<Props> = () => {
+const FireCrawl: FC<Props> = ({
+  onPreview,
+}) => {
   const { t } = useTranslation()
   const [step, setStep] = useState<Step>(Step.finished)
 
@@ -126,6 +129,7 @@ const FireCrawl: FC<Props> = () => {
               list={crawlResult}
               checkedList={checkedCrawlResult}
               onSelectedChange={setCheckedCrawlResult}
+              onPreview={onPreview}
             />
           )}
         </OptionsWrap>
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
index b85d986f798418..d3f94e314995ca 100644
--- a/web/app/components/datasets/create/website/index.tsx
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -4,12 +4,15 @@ import React, { useCallback, useState } from 'react'
 import NoData from './no-data'
 import Firecrawl from './firecrawl'
 import { useModalContext } from '@/context/modal-context'
+import type { CrawlResultItem } from '@/models/datasets'
 
 type Props = {
-
+  onPreview: (payload: CrawlResultItem) => void
 }
 
-const WebsitePreview: FC<Props> = () => {
+const WebsitePreview: FC<Props> = ({
+  onPreview,
+}) => {
   const { setShowAccountSettingModal } = useModalContext()
   const [isLoaded, setIsLoaded] = useState(false)
   const [isConfigured, setIsConfigured] = useState(true)
@@ -29,7 +32,7 @@ const WebsitePreview: FC<Props> = () => {
     <div>
       {isConfigured
         ? (
-          <Firecrawl />
+          <Firecrawl onPreview={onPreview} />
         )
         : (
           <NoData onConfig={handleOnConfig} />
diff --git a/web/app/components/datasets/create/website/preview.tsx b/web/app/components/datasets/create/website/preview.tsx
new file mode 100644
index 00000000000000..322ce43b174d5b
--- /dev/null
+++ b/web/app/components/datasets/create/website/preview.tsx
@@ -0,0 +1,41 @@
+'use client'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
+import { XMarkIcon } from '@heroicons/react/20/solid'
+import s from '../file-preview/index.module.css'
+import type { CrawlResultItem } from '@/models/datasets'
+
+type IProps = {
+  payload: CrawlResultItem
+  hidePreview: () => void
+}
+
+const WebsitePreview = ({
+  payload,
+  hidePreview,
+}: IProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className={cn(s.filePreview)}>
+      <div className={cn(s.previewHeader)}>
+        <div className={cn(s.title)}>
+          <span>{t('datasetCreation.stepOne.pagePreview')}</span>
+          <div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
+            <XMarkIcon className='h-4 w-4'></XMarkIcon>
+          </div>
+        </div>
+        <div className='leading-5 text-sm font-medium text-gray-900 break-words'>
+          {payload.title}
+        </div>
+        <div className='truncate leading-[18px] text-xs font-normal text-gray-500' title={payload.source_url}>{payload.source_url}</div>
+      </div>
+      <div className={cn(s.previewContent)}>
+        <div className={cn(s.fileContent)}>{payload.markdown}</div>
+      </div>
+    </div>
+  )
+}
+
+export default WebsitePreview

From efecdccf35beed86353287216fabd118796924c5 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 15:01:58 +0800
Subject: [PATCH 21/31] feat: support login by given mail (#4991)

---
 web/app/layout.tsx            | 3 ++-
 web/app/signin/normalForm.tsx | 8 +++++---
 web/config/index.ts           | 2 ++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/web/app/layout.tsx b/web/app/layout.tsx
index ca665fb360a366..fedf66045adac1 100644
--- a/web/app/layout.tsx
+++ b/web/app/layout.tsx
@@ -39,11 +39,12 @@ const LocaleLayout = ({
         data-api-prefix={process.env.NEXT_PUBLIC_API_PREFIX}
         data-pubic-api-prefix={process.env.NEXT_PUBLIC_PUBLIC_API_PREFIX}
         data-public-edition={process.env.NEXT_PUBLIC_EDITION}
+        data-public-support-mail-login={process.env.NEXT_PUBLIC_SUPPORT_MAIL_LOGIN}
         data-public-sentry-dsn={process.env.NEXT_PUBLIC_SENTRY_DSN}
         data-public-maintenance-notice={process.env.NEXT_PUBLIC_MAINTENANCE_NOTICE}
         data-public-site-about={process.env.NEXT_PUBLIC_SITE_ABOUT}
       >
-        <Topbar/>
+        <Topbar />
         <BrowerInitor>
           <SentryInitor>
             <I18nServer>{children}</I18nServer>
diff --git a/web/app/signin/normalForm.tsx b/web/app/signin/normalForm.tsx
index aba658b3ce2298..f6abf0d6c0f383 100644
--- a/web/app/signin/normalForm.tsx
+++ b/web/app/signin/normalForm.tsx
@@ -7,7 +7,7 @@ import useSWR from 'swr'
 import Link from 'next/link'
 import Toast from '../components/base/toast'
 import style from './page.module.css'
-import { IS_CE_EDITION, apiPrefix } from '@/config'
+import { IS_CE_EDITION, SUPPORT_MAIL_LOGIN, apiPrefix } from '@/config'
 import Button from '@/app/components/base/button'
 import { login, oauth } from '@/service/common'
 import { getPurifyHref } from '@/utils'
@@ -62,6 +62,8 @@ function reducer(state: IState, action: IAction) {
 
 const NormalForm = () => {
   const { t } = useTranslation()
+  const useEmailLogin = IS_CE_EDITION || SUPPORT_MAIL_LOGIN
+
   const router = useRouter()
 
   const [state, dispatch] = useReducer(reducer, {
@@ -150,7 +152,7 @@ const NormalForm = () => {
 
       <div className="w-full mx-auto mt-8">
         <div className="bg-white ">
-          {!IS_CE_EDITION && (
+          {!useEmailLogin && (
             <div className="flex flex-col gap-3 mt-6">
               <div className='w-full'>
                 <a href={getPurifyHref(`${apiPrefix}/oauth/login/github`)}>
@@ -194,7 +196,7 @@ const NormalForm = () => {
           )}
 
           {
-            IS_CE_EDITION && <>
+            useEmailLogin && <>
               {/* <div className="relative mt-6">
                 <div className="absolute inset-0 flex items-center" aria-hidden="true">
                   <div className="w-full border-t border-gray-300" />
diff --git a/web/config/index.ts b/web/config/index.ts
index 933c479b7c9e6e..16f458d58023fd 100644
--- a/web/config/index.ts
+++ b/web/config/index.ts
@@ -33,6 +33,8 @@ export const PUBLIC_API_PREFIX: string = publicApiPrefix
 const EDITION = process.env.NEXT_PUBLIC_EDITION || globalThis.document?.body?.getAttribute('data-public-edition') || 'SELF_HOSTED'
 export const IS_CE_EDITION = EDITION === 'SELF_HOSTED'
 
+export const SUPPORT_MAIL_LOGIN = !!(process.env.NEXT_PUBLIC_SUPPORT_MAIL_LOGIN || globalThis.document?.body?.getAttribute('data-public-support-mail-login'))
+
 export const TONE_LIST = [
   {
     id: 1,

From 1d3dc21c4ebb37348e4dcc759b9d6be2344bde5d Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 15:15:09 +0800
Subject: [PATCH 22/31] feat: finish step one

---
 web/app/components/datasets/create/index.tsx         |  6 +++++-
 .../components/datasets/create/step-one/index.tsx    | 12 ++++++++++--
 .../datasets/create/website/firecrawl/index.tsx      |  7 +++++--
 web/app/components/datasets/create/website/index.tsx | 10 +++++++++-
 4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
index 83d74b55baefd6..0a29252a871379 100644
--- a/web/app/components/datasets/create/index.tsx
+++ b/web/app/components/datasets/create/index.tsx
@@ -8,7 +8,7 @@ import StepOne from './step-one'
 import StepTwo from './step-two'
 import StepThree from './step-three'
 import { DataSourceType } from '@/models/datasets'
-import type { DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
+import type { CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
 import { fetchDataSource } from '@/service/common'
 import { fetchDatasetDetail } from '@/service/datasets'
 import type { NotionPage } from '@/models/common'
@@ -36,6 +36,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
     setNotionPages(value)
   }
 
+  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
+  console.log(websitePages)
   const updateFileList = (preparedFiles: FileItem[]) => {
     setFiles(preparedFiles)
   }
@@ -121,6 +123,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           notionPages={notionPages}
           updateNotionPages={updateNotionPages}
           onStepChange={nextStep}
+          websitePages={websitePages}
+          updateWebsitePages={setWebsitePages}
         />}
         {(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
           hasSetAPIKEY={!!embeddingsDefaultModel}
diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx
index dc1fd24ae2f5db..857cf88f589935 100644
--- a/web/app/components/datasets/create/step-one/index.tsx
+++ b/web/app/components/datasets/create/step-one/index.tsx
@@ -31,6 +31,8 @@ type IStepOneProps = {
   updateNotionPages: (value: NotionPage[]) => void
   onStepChange: () => void
   changeType: (type: DataSourceType) => void
+  websitePages?: CrawlResultItem[]
+  updateWebsitePages: (value: CrawlResultItem[]) => void
 }
 
 type NotionConnectorProps = {
@@ -62,6 +64,8 @@ const StepOne = ({
   updateFile,
   notionPages = [],
   updateNotionPages,
+  websitePages = [],
+  updateWebsitePages,
 }: IStepOneProps) => {
   const { dataset } = useDatasetDetailContext()
   const [showModal, setShowModal] = useState(false)
@@ -210,14 +214,18 @@ const StepOne = ({
           {dataSourceType === DataSourceType.WEB && (
             <>
               <div className='mb-8 w-[640px]'>
-                <Website onPreview={setCurrentWebsite} />
+                <Website
+                  onPreview={setCurrentWebsite}
+                  checkedCrawlResult={websitePages}
+                  onCheckedCrawlResultChange={updateWebsitePages}
+                />
               </div>
               {isShowVectorSpaceFull && (
                 <div className='max-w-[640px] mb-4'>
                   <VectorSpaceFull />
                 </div>
               )}
-              <Button disabled={isShowVectorSpaceFull || !notionPages.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
+              <Button disabled={isShowVectorSpaceFull || !websitePages.length} className={s.submitButton} type='primary' onClick={onStepChange}>{t('datasetCreation.stepOne.button')}</Button>
             </>
           )}
           {!datasetId && (
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 1007551978d09e..80e36392483a80 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -22,6 +22,8 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 type Props = {
   onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
 }
 
 const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
@@ -41,6 +43,8 @@ enum Step {
 
 const FireCrawl: FC<Props> = ({
   onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
 }) => {
   const { t } = useTranslation()
   const [step, setStep] = useState<Step>(Step.finished)
@@ -86,7 +90,6 @@ const FireCrawl: FC<Props> = ({
   const isCrawlFinished = step === Step.finished
   const isRunning = step === Step.running
   const [crawlResult, setCrawlResult] = useState<CrawlResultItem[]>(mockCrawlResult)
-  const [checkedCrawlResult, setCheckedCrawlResult] = useState<CrawlResultItem[]>([])
 
   const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
   const handleRun = useCallback(async (url: string) => {
@@ -128,7 +131,7 @@ const FireCrawl: FC<Props> = ({
             <CrawledResult
               list={crawlResult}
               checkedList={checkedCrawlResult}
-              onSelectedChange={setCheckedCrawlResult}
+              onSelectedChange={onCheckedCrawlResultChange}
               onPreview={onPreview}
             />
           )}
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
index d3f94e314995ca..799bcb76ecf1ae 100644
--- a/web/app/components/datasets/create/website/index.tsx
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -8,10 +8,14 @@ import type { CrawlResultItem } from '@/models/datasets'
 
 type Props = {
   onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
 }
 
 const WebsitePreview: FC<Props> = ({
   onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
 }) => {
   const { setShowAccountSettingModal } = useModalContext()
   const [isLoaded, setIsLoaded] = useState(false)
@@ -32,7 +36,11 @@ const WebsitePreview: FC<Props> = ({
     <div>
       {isConfigured
         ? (
-          <Firecrawl onPreview={onPreview} />
+          <Firecrawl
+            onPreview={onPreview}
+            checkedCrawlResult={checkedCrawlResult}
+            onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+          />
         )
         : (
           <NoData onConfig={handleOnConfig} />

From 042bc4f7b34275f63b9e700ef27b602c31d1ff0a Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 15:53:06 +0800
Subject: [PATCH 23/31] feat: step two

---
 web/app/components/datasets/create/index.tsx  |  1 +
 .../datasets/create/step-two/index.module.css |  1 +
 .../datasets/create/step-two/index.tsx        | 45 ++++++++++++++++++-
 web/i18n/en-US/dataset-creation.ts            |  2 +
 web/i18n/zh-Hans/dataset-creation.ts          |  2 +
 web/models/datasets.ts                        |  7 ++-
 6 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
index 0a29252a871379..1d7d3c6b0e9a9e 100644
--- a/web/app/components/datasets/create/index.tsx
+++ b/web/app/components/datasets/create/index.tsx
@@ -134,6 +134,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           dataSourceType={dataSourceType}
           files={fileList.map(file => file.file)}
           notionPages={notionPages}
+          websitePages={websitePages}
           onStepChange={changeStep}
           updateIndexingTypeCache={updateIndexingTypeCache}
           updateResultCache={updateResultCache}
diff --git a/web/app/components/datasets/create/step-two/index.module.css b/web/app/components/datasets/create/step-two/index.module.css
index b5089b5fa327b4..24a62c8e3c9054 100644
--- a/web/app/components/datasets/create/step-two/index.module.css
+++ b/web/app/components/datasets/create/step-two/index.module.css
@@ -323,6 +323,7 @@
 }
 
 .sourceContent {
+  width: 0;
   flex: 1 1 auto;
 }
 
diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx
index 90b798025ed2f3..35930b77f99666 100644
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@@ -12,7 +12,7 @@ import RetrievalMethodInfo from '../../common/retrieval-method-info'
 import PreviewItem, { PreviewType } from './preview-item'
 import LanguageSelect from './language-select'
 import s from './index.module.css'
-import type { CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
+import type { CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, IndexingEstimateResponse, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets'
 import {
   createDocument,
   createFirstDocument,
@@ -44,6 +44,7 @@ import TooltipPlus from '@/app/components/base/tooltip-plus'
 import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { LanguagesSupported } from '@/i18n/language'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel'
 
 type ValueOf<T> = T[keyof T]
 type StepTwoProps = {
@@ -56,6 +57,7 @@ type StepTwoProps = {
   dataSourceType: DataSourceType
   files: CustomFile[]
   notionPages?: NotionPage[]
+  websitePages?: CrawlResultItem[]
   onStepChange?: (delta: number) => void
   updateIndexingTypeCache?: (type: string) => void
   updateResultCache?: (res: createDocumentResponse) => void
@@ -82,6 +84,7 @@ const StepTwo = ({
   dataSourceType,
   files,
   notionPages = [],
+  websitePages = [],
   onStepChange,
   updateIndexingTypeCache,
   updateResultCache,
@@ -242,6 +245,14 @@ const StepTwo = ({
     }) as NotionInfo[]
   }
 
+  const getWebsiteInfo = () => {
+    return {
+      provider: 'firecrawl',
+      job_id: 'xxx',
+      urls: websitePages.map(page => page.source_url),
+    }
+  }
+
   const getFileIndexingEstimateParams = (docForm: DocForm): IndexingEstimateParams | undefined => {
     if (dataSourceType === DataSourceType.FILE) {
       return {
@@ -271,6 +282,19 @@ const StepTwo = ({
         dataset_id: datasetId as string,
       }
     }
+    if (dataSourceType === DataSourceType.WEB) {
+      return {
+        info_list: {
+          data_source_type: dataSourceType,
+          website_info_list: getWebsiteInfo(),
+        },
+        indexing_technique: getIndexing_technique() as string,
+        process_rule: getProcessRule(),
+        doc_form: docForm,
+        doc_language: docLanguage,
+        dataset_id: datasetId as string,
+      }
+    }
   }
   const {
     modelList: rerankModelList,
@@ -335,6 +359,9 @@ const StepTwo = ({
       }
       if (dataSourceType === DataSourceType.NOTION)
         params.data_source.info_list.notion_info_list = getNotionInfo()
+
+      if (dataSourceType === DataSourceType.WEB)
+        params.data_source.info_list.website_info_list = getWebsiteInfo()
     }
     return params
   }
@@ -819,6 +846,22 @@ const StepTwo = ({
                     </div>
                   </>
                 )}
+                {dataSourceType === DataSourceType.WEB && (
+                  <>
+                    <div className='mb-2 text-xs font-medium text-gray-500'>{t('datasetCreation.stepTwo.websiteSource')}</div>
+                    <div className='flex items-center text-sm leading-6 font-medium text-gray-800'>
+                      <Globe01 className='shrink-0 mr-1' />
+                      <span className='grow w-0 truncate'>{websitePages[0].source_url}</span>
+                      {websitePages.length > 1 && (
+                        <span className={s.sourceCount}>
+                          <span>{t('datasetCreation.stepTwo.other')}</span>
+                          <span>{websitePages.length - 1}</span>
+                          <span>{t('datasetCreation.stepTwo.webpageUnit')}</span>
+                        </span>
+                      )}
+                    </div>
+                  </>
+                )}
               </div>
               <div className={s.divider} />
               <div className={s.segmentCount}>
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index baa32587d92582..c539319a0d01f7 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -108,9 +108,11 @@ const translation = {
     calculating: 'Calculating...',
     fileSource: 'Preprocess documents',
     notionSource: 'Preprocess pages',
+    websiteSource: 'Preprocess website',
     other: 'and other ',
     fileUnit: ' files',
     notionUnit: ' pages',
+    webpageUnit: ' pages',
     previousStep: 'Previous step',
     nextStep: 'Save & Process',
     save: 'Save & Process',
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index c8866fbd3b2d1b..15de59b3e5750c 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -108,9 +108,11 @@ const translation = {
     calculating: '计算中...',
     fileSource: '预处理文档',
     notionSource: '预处理页面',
+    websiteSource: '预处理页面',
     other: '和其他 ',
     fileUnit: ' 个文件',
     notionUnit: ' 个页面',
+    webpageUnit: ' 个页面',
     previousStep: '上一步',
     nextStep: '保存并处理',
     save: '保存并处理',
diff --git a/web/models/datasets.ts b/web/models/datasets.ts
index 1edb63748faadd..47b786c7c9b8f8 100644
--- a/web/models/datasets.ts
+++ b/web/models/datasets.ts
@@ -5,7 +5,7 @@ import type { Tag } from '@/app/components/base/tag-management/constant'
 export enum DataSourceType {
   FILE = 'upload_file',
   NOTION = 'notion_import',
-  WEB = 'web_import',
+  WEB = 'website',
 }
 
 export type DataSet = {
@@ -229,6 +229,11 @@ export type DataSource = {
     file_info_list?: {
       file_ids: string[]
     }
+    website_info_list?: {
+      provider: string
+      job_id: string
+      urls: string[]
+    }
   }
 }
 

From dcf988231499b3fcc13af1c97c09832a45b0148b Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 16:25:12 +0800
Subject: [PATCH 24/31] feat: list page

---
 web/app/components/datasets/create/index.tsx   |  1 -
 web/app/components/datasets/documents/list.tsx | 10 ++++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
index cc21618e13aa79..30ee370be54753 100644
--- a/web/app/components/datasets/create/index.tsx
+++ b/web/app/components/datasets/create/index.tsx
@@ -37,7 +37,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   }
 
   const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
-  console.log(websitePages)
   const updateFileList = (preparedFiles: FileItem[]) => {
     setFiles(preparedFiles)
   }
diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx
index 4566287fb81aca..611fdd3241ad05 100644
--- a/web/app/components/datasets/documents/list.tsx
+++ b/web/app/components/datasets/documents/list.tsx
@@ -13,6 +13,7 @@ import cn from 'classnames'
 import dayjs from 'dayjs'
 import { Edit03 } from '../../base/icons/src/vender/solid/general'
 import TooltipPlus from '../../base/tooltip-plus'
+import { Globe01 } from '../../base/icons/src/vender/line/mapsAndTravel'
 import s from './style.module.css'
 import RenameModal from './rename-modal'
 import Switch from '@/app/components/base/switch'
@@ -418,10 +419,10 @@ const DocumentList: FC<IDocumentListProps> = ({ embeddingAvailable, documents =
               <td>
                 <div className='group flex items-center justify-between'>
                   <span className={s.tdValue}>
-                    {
-                      doc?.data_source_type === DataSourceType.NOTION
-                        ? <NotionIcon className='inline-flex -mt-[3px] mr-1.5 align-middle' type='page' src={doc.data_source_info.notion_page_icon} />
-                        : <div className={cn(s[`${doc?.data_source_info?.upload_file?.extension ?? fileType}Icon`], s.commonIcon, 'mr-1.5')}></div>
+                    {doc?.data_source_type === DataSourceType.NOTION && <NotionIcon className='inline-flex -mt-[3px] mr-1.5 align-middle' type='page' src={doc.data_source_info.notion_page_icon} />
+                    }
+                    {doc?.data_source_type === DataSourceType.FILE && <div className={cn(s[`${doc?.data_source_info?.upload_file?.extension ?? fileType}Icon`], s.commonIcon, 'mr-1.5')}></div>}
+                    {doc?.data_source_type === DataSourceType.WEB && <Globe01 className='inline-flex -mt-[3px] mr-1.5 align-middle' />
                     }
                     {
                       doc.name
@@ -450,6 +451,7 @@ const DocumentList: FC<IDocumentListProps> = ({ embeddingAvailable, documents =
               </td>
               <td>
                 {
+                  // TODO: Maybe add website
                   (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) && doc?.data_source_type === DataSourceType.NOTION)
                     ? <ProgressBar percent={doc.percent || 0} />
                     : <StatusItem status={doc.display_status} />

From c9dda6ead92e3b2876b5ff0dddc4936e29e92de5 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 17:21:29 +0800
Subject: [PATCH 25/31] feat: config firecrawl main

---
 .../create/website/firecrawl/base/field.tsx   |   4 +-
 .../config-firecrawl-modal.tsx                | 139 ++++++++++++++++++
 .../data-source-website/index.tsx             |  40 ++---
 web/models/common.ts                          |   6 +
 web/service/datasets.ts                       |   4 +
 5 files changed, 175 insertions(+), 18 deletions(-)
 create mode 100644 web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx

diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
index 12eaefa9b85f03..9cf978d0af191d 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx
@@ -7,6 +7,7 @@ import Input from './input'
 type Props = {
   className?: string
   label: string
+  labelClassName?: string
   value: string | number
   onChange: (value: string | number) => void
   isRequired?: boolean
@@ -17,6 +18,7 @@ type Props = {
 const Field: FC<Props> = ({
   className,
   label,
+  labelClassName,
   value,
   onChange,
   isRequired = false,
@@ -26,7 +28,7 @@ const Field: FC<Props> = ({
   return (
     <div className={cn(className)}>
       <div className='flex py-[7px]'>
-        <div className='flex items-center h-[18px] text-[13px] font-medium text-gray-900'>{label} </div>
+        <div className={cn(labelClassName, 'flex items-center h-[18px] text-[13px] font-medium text-gray-900')}>{label} </div>
         {isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
       </div>
       <Input
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx
new file mode 100644
index 00000000000000..e8d23fb3eae798
--- /dev/null
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx
@@ -0,0 +1,139 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+} from '@/app/components/base/portal-to-follow-elem'
+import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
+import Button from '@/app/components/base/button'
+import type { FirecrawlConfig } from '@/models/common'
+import Field from '@/app/components/datasets/create/website/firecrawl/base/field'
+import Toast from '@/app/components/base/toast'
+import { createFirecrawlApiKey } from '@/service/datasets'
+import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
+type Props = {
+  onCancel: () => void
+  onSaved: () => void
+}
+
+const DEFAULT_BASE_URL = 'https://api.firecrawl.dev'
+
+const ConfigFirecrawlModal: FC<Props> = ({
+  onCancel,
+  onSaved,
+}) => {
+  const { t } = useTranslation()
+  const [config, setConfig] = useState<FirecrawlConfig>({
+    api_key: '',
+    base_url: '',
+  })
+
+  const handleConfigChange = useCallback((key: string) => {
+    return (value: string | number) => {
+      setConfig(prev => ({ ...prev, [key]: value as string }))
+    }
+  }, [])
+
+  const handleSave = useCallback(async () => {
+    if (!config.api_key) {
+      const errorMsg = t('common.errorMsg.fieldRequired', {
+        field: 'API Key',
+      })
+      Toast.notify({
+        type: 'error',
+        message: errorMsg,
+      })
+      return
+    }
+    const postData = {
+      category: 'website',
+      provider: 'firecrawl',
+      credentials: {
+        auth_type: 'bearer',
+        config: {
+          api_key: config.api_key,
+          base_url: config.base_url || DEFAULT_BASE_URL,
+        },
+      },
+    }
+    await createFirecrawlApiKey(postData)
+    Toast.notify({
+      type: 'success',
+      message: t('common.operation.saveSuccess'),
+    })
+    onSaved()
+  }, [config.api_key, config.base_url, onSaved, t])
+
+  return (
+    <PortalToFollowElem open>
+      <PortalToFollowElemContent className='w-full h-full z-[60]'>
+        <div className='fixed inset-0 flex items-center justify-center bg-black/[.25]'>
+          <div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-white shadow-xl rounded-2xl overflow-y-auto'>
+            <div className='px-8 pt-8'>
+              <div className='flex justify-between items-center mb-4'>
+                <div className='text-xl font-semibold text-gray-900'>{t('common.modelProvider.editConfig')}</div>
+              </div>
+
+              <div className='space-y-4'>
+                <Field
+                  label='Base URL'
+                  labelClassName='!text-sm'
+                  value={config.base_url}
+                  onChange={handleConfigChange('base_url')}
+                  placeholder={DEFAULT_BASE_URL}
+                />
+                <Field
+                  label='API Key'
+                  labelClassName='!text-sm'
+                  isRequired
+                  value={config.api_key}
+                  onChange={handleConfigChange('api_key')}
+                  placeholder={t('common.modelProvider.apiKey')!}
+                />
+              </div>
+              <div className='my-8 flex justify-between items-center h-8'>
+                <a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-[#155EEF]' target='_blank' href='https://www.firecrawl.dev/account'>
+                  <span>Get your API key from firecrawl.dev</span>
+                  <LinkExternal02 className='w-3 h-3' />
+                </a>
+                <div className='flex'>
+                  <Button
+                    className='mr-2 h-9 text-sm font-medium text-gray-700'
+                    onClick={onCancel}
+                  >
+                    {t('common.operation.cancel')}
+                  </Button>
+                  <Button
+                    className='h-9 text-sm font-medium'
+                    type='primary'
+                    onClick={handleSave}
+                  >
+                    {t('common.operation.save')}
+                  </Button>
+                </div>
+
+              </div>
+            </div>
+            <div className='border-t-[0.5px] border-t-black/5'>
+              <div className='flex justify-center items-center py-3 bg-gray-50 text-xs text-gray-500'>
+                <Lock01 className='mr-1 w-3 h-3 text-gray-500' />
+                {t('common.modelProvider.encrypted.front')}
+                <a
+                  className='text-primary-600 mx-1'
+                  target='_blank' rel='noopener noreferrer'
+                  href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
+                >
+                  PKCS1_OAEP
+                </a>
+                {t('common.modelProvider.encrypted.back')}
+              </div>
+            </div>
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+export default React.memo(ConfigFirecrawlModal)
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
index 489af9842cbc03..d3da70ce96cde6 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
@@ -5,6 +5,7 @@ import { useBoolean } from 'ahooks'
 import cn from 'classnames'
 import Panel from '../panel'
 import { DataSourceType } from '../panel/types'
+import ConfigFirecrawlModal from './config-firecrawl-modal'
 import { fetchWebsiteDataSource } from '@/service/common'
 import type {
   DataSourceWebsiteItem,
@@ -18,7 +19,7 @@ import {
 
 type Props = {}
 
-const isUseMock = true
+const isUseMock = false
 const mockList: DataSourceWebsiteItem[] = [
   {
     id: '1',
@@ -51,29 +52,34 @@ const DataSourceWebsite: FC<Props> = () => {
   const [isShowConfig, {
     setTrue: showConfig,
     setFalse: hideConfig,
-  }] = useBoolean(false)
+  }] = useBoolean(true)
 
   const handleRemove = useCallback(() => {
 
   }, [])
 
   return (
-    <Panel
-      type={DataSourceType.website}
-      isConfigured={list.length > 0}
-      onConfigure={showConfig}
-      readonly={!isCurrentWorkspaceManager}
-      configuredList={list.map(item => ({
-        id: item.id,
-        logo: ({ className }: { className: string }) => (
-          <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
-        ),
-        name: 'FireCrawl',
-        isActive: true,
-      }))}
-      onRemove={handleRemove}
+    <>
+      <Panel
+        type={DataSourceType.website}
+        isConfigured={list.length > 0}
+        onConfigure={showConfig}
+        readonly={!isCurrentWorkspaceManager}
+        configuredList={list.map(item => ({
+          id: item.id,
+          logo: ({ className }: { className: string }) => (
+            <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
+          ),
+          name: 'FireCrawl',
+          isActive: true,
+        }))}
+        onRemove={handleRemove}
+      />
+      {isShowConfig && (
+        <ConfigFirecrawlModal onSaved={hideConfig} onCancel={hideConfig} />
+      )}
+    </>
 
-    />
   )
 }
 export default React.memo(DataSourceWebsite)
diff --git a/web/models/common.ts b/web/models/common.ts
index a8322e2478206f..ec10ff2b88c4ae 100644
--- a/web/models/common.ts
+++ b/web/models/common.ts
@@ -185,6 +185,12 @@ export type WebsiteCredentials = {
     api_key: string
   }
 }
+
+export type FirecrawlConfig = {
+  api_key: string
+  base_url: string
+}
+
 export type DataSourceWebsiteItem = {
   id: string
   category: DataSourceCategory.website
diff --git a/web/service/datasets.ts b/web/service/datasets.ts
index 302b16f6f55567..200785b02db01c 100644
--- a/web/service/datasets.ts
+++ b/web/service/datasets.ts
@@ -227,6 +227,10 @@ export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> =
   return get<{ api_base_url: string }>(url)
 }
 
+export const createFirecrawlApiKey: Fetcher<CommonResponse, Record<string, any>> = (body) => {
+  return post<CommonResponse>('api-key-auth/data-source/binding', { body })
+}
+
 type FileTypesRes = {
   allowed_extensions: string[]
 }

From a1e73bb2b183a123fbda5d2b4603bffc2b1cf4bf Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 17:49:14 +0800
Subject: [PATCH 26/31] chore: config i18n

---
 .../create/website/firecrawl/index.tsx        |  2 --
 .../config-firecrawl-modal.tsx                | 27 ++++++++++++++-----
 .../data-source-website/index.tsx             |  2 +-
 web/i18n/en-US/dataset-creation.ts            |  6 +++++
 web/i18n/zh-Hans/dataset-creation.ts          |  6 +++++
 5 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 80e36392483a80..8bee207976dc0c 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -18,8 +18,6 @@ import { sleep } from '@/utils'
 const ERROR_I18N_PREFIX = 'common.errorMsg'
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
-// const testCrawlErrorMsg = 'Firecrawl currently does not support social media scraping due to policy restrictions. We are actively working on building support for it.'
-
 type Props = {
   onPreview: (payload: CrawlResultItem) => void
   checkedCrawlResult: CrawlResultItem[]
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx
index e8d23fb3eae798..f2f71d0b1d2975 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx
@@ -18,6 +18,8 @@ type Props = {
   onSaved: () => void
 }
 
+const I18N_PREFIX = 'datasetCreation.firecrawl'
+
 const DEFAULT_BASE_URL = 'https://api.firecrawl.dev'
 
 const ConfigFirecrawlModal: FC<Props> = ({
@@ -37,10 +39,21 @@ const ConfigFirecrawlModal: FC<Props> = ({
   }, [])
 
   const handleSave = useCallback(async () => {
-    if (!config.api_key) {
-      const errorMsg = t('common.errorMsg.fieldRequired', {
-        field: 'API Key',
-      })
+    let errorMsg = ''
+    if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
+      errorMsg = t('common.errorMsg.urlError')
+    if (!errorMsg) {
+      if (!config.api_key) {
+        errorMsg = t('common.errorMsg.fieldRequired', {
+          field: 'API Key',
+        })
+      }
+      else if (!config.api_key.startsWith('fc-')) {
+        errorMsg = t(`${I18N_PREFIX}.apiKeyFormatError`)
+      }
+    }
+
+    if (errorMsg) {
       Toast.notify({
         type: 'error',
         message: errorMsg,
@@ -73,7 +86,7 @@ const ConfigFirecrawlModal: FC<Props> = ({
           <div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-white shadow-xl rounded-2xl overflow-y-auto'>
             <div className='px-8 pt-8'>
               <div className='flex justify-between items-center mb-4'>
-                <div className='text-xl font-semibold text-gray-900'>{t('common.modelProvider.editConfig')}</div>
+                <div className='text-xl font-semibold text-gray-900'>{t(`${I18N_PREFIX}.configFirecrawl`)}</div>
               </div>
 
               <div className='space-y-4'>
@@ -90,12 +103,12 @@ const ConfigFirecrawlModal: FC<Props> = ({
                   isRequired
                   value={config.api_key}
                   onChange={handleConfigChange('api_key')}
-                  placeholder={t('common.modelProvider.apiKey')!}
+                  placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
                 />
               </div>
               <div className='my-8 flex justify-between items-center h-8'>
                 <a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-[#155EEF]' target='_blank' href='https://www.firecrawl.dev/account'>
-                  <span>Get your API key from firecrawl.dev</span>
+                  <span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
                   <LinkExternal02 className='w-3 h-3' />
                 </a>
                 <div className='flex'>
diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
index d3da70ce96cde6..e56d19eece3870 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
@@ -52,7 +52,7 @@ const DataSourceWebsite: FC<Props> = () => {
   const [isShowConfig, {
     setTrue: showConfig,
     setFalse: hideConfig,
-  }] = useBoolean(true)
+  }] = useBoolean(false)
 
   const handleRemove = useCallback(() => {
 
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 8e63a466a78ba3..466b0afc3ad16c 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -11,6 +11,12 @@ const translation = {
   error: {
     unavailable: 'This Knowledge is not available',
   },
+  firecrawl: {
+    configFirecrawl: 'Configure 🔥Firecrawl',
+    apiKeyPlaceholder: 'API key from firecrawl.dev, starting with "fc-"',
+    apiKeyFormatError: 'API key should start with "fc-"',
+    getApiKeyLinkText: 'Get your API key from firecrawl.dev',
+  },
   stepOne: {
     filePreview: 'File Preview',
     pagePreview: 'Page Preview',
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index 15de59b3e5750c..015d3b61c8fd29 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -11,6 +11,12 @@ const translation = {
   error: {
     unavailable: '该知识库不可用',
   },
+  firecrawl: {
+    configFirecrawl: '配置 🔥Firecrawl',
+    apiKeyPlaceholder: '从 firecrawl.dev 获取 API Key，以 "fc-" 开头',
+    apiKeyFormatError: 'API Key 应以 "fc-" 开头',
+    getApiKeyLinkText: '从 firecrawl.dev 获取您的 API Key',
+  },
   stepOne: {
     filePreview: '文件预览',
     pagePreview: '页面预览',

From 980ce12976cb88820b6ec0b3cc3ecca8a990d633 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Thu, 6 Jun 2024 18:19:06 +0800
Subject: [PATCH 27/31] feat: fetch and remove

---
 .../data-source-website/index.tsx             | 43 +++++++------------
 web/models/common.ts                          |  1 +
 web/service/datasets.ts                       |  8 ++++
 3 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
index e56d19eece3870..4992598296c321 100644
--- a/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
+++ b/web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx
@@ -1,50 +1,34 @@
 'use client'
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
 import { useBoolean } from 'ahooks'
 import cn from 'classnames'
 import Panel from '../panel'
 import { DataSourceType } from '../panel/types'
 import ConfigFirecrawlModal from './config-firecrawl-modal'
-import { fetchWebsiteDataSource } from '@/service/common'
+import { fetchFirecrawlApiKey, removeFirecrawlApiKey } from '@/service/datasets'
+
 import type {
   DataSourceWebsiteItem,
 } from '@/models/common'
 import { useAppContext } from '@/context/app-context'
 
 import {
-  DataSourceCategory,
   WebsiteProvider,
 } from '@/models/common'
+import Toast from '@/app/components/base/toast'
 
 type Props = {}
 
-const isUseMock = false
-const mockList: DataSourceWebsiteItem[] = [
-  {
-    id: '1',
-    category: DataSourceCategory.website,
-    provider: WebsiteProvider.fireCrawl,
-    credentials: {
-      auth_type: 'bearer',
-      config: {
-        base_url: 'https://xxx',
-        api_key: '123456',
-      },
-    },
-    created_at: 1627584000,
-    updated_at: 1627584000,
-  },
-]
-
 const DataSourceWebsite: FC<Props> = () => {
+  const { t } = useTranslation()
   const { isCurrentWorkspaceManager } = useAppContext()
-  const [list, setList] = useState<DataSourceWebsiteItem[]>(isUseMock ? mockList : [])
+  const [list, setList] = useState<DataSourceWebsiteItem[]>([])
   useEffect(() => {
     (async () => {
-      const { data } = await fetchWebsiteDataSource()
-      const list = data.settings.filter(item => item.provider === WebsiteProvider.fireCrawl)
-
+      const res = await fetchFirecrawlApiKey() as any
+      const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled)
       setList(list)
     })()
   }, [])
@@ -54,9 +38,14 @@ const DataSourceWebsite: FC<Props> = () => {
     setFalse: hideConfig,
   }] = useBoolean(false)
 
-  const handleRemove = useCallback(() => {
-
-  }, [])
+  const handleRemove = useCallback(async () => {
+    await removeFirecrawlApiKey(list[0].id)
+    setList([])
+    Toast.notify({
+      type: 'success',
+      message: t('common.operation.saveSuccess'),
+    })
+  }, [list, t])
 
   return (
     <>
diff --git a/web/models/common.ts b/web/models/common.ts
index ec10ff2b88c4ae..da52b0c7f75f45 100644
--- a/web/models/common.ts
+++ b/web/models/common.ts
@@ -196,6 +196,7 @@ export type DataSourceWebsiteItem = {
   category: DataSourceCategory.website
   provider: WebsiteProvider
   credentials: WebsiteCredentials
+  disabled: boolean
   created_at: number
   updated_at: number
 }
diff --git a/web/service/datasets.ts b/web/service/datasets.ts
index 200785b02db01c..b620f2133ecfa2 100644
--- a/web/service/datasets.ts
+++ b/web/service/datasets.ts
@@ -227,10 +227,18 @@ export const fetchDatasetApiBaseUrl: Fetcher<{ api_base_url: string }, string> =
   return get<{ api_base_url: string }>(url)
 }
 
+export const fetchFirecrawlApiKey = () => {
+  return get<CommonResponse>('api-key-auth/data-source')
+}
+
 export const createFirecrawlApiKey: Fetcher<CommonResponse, Record<string, any>> = (body) => {
   return post<CommonResponse>('api-key-auth/data-source/binding', { body })
 }
 
+export const removeFirecrawlApiKey: Fetcher<CommonResponse, string> = (id: string) => {
+  return del<CommonResponse>(`api-key-auth/data-source/${id}`)
+}
+
 type FileTypesRes = {
   allowed_extensions: string[]
 }

From 637546aac70515184a41fea6d139b1dbf9551348 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 7 Jun 2024 11:30:32 +0800
Subject: [PATCH 28/31] feat:  crawl api

---
 .../create/website/firecrawl/index.tsx        | 39 ++++++++++++++++---
 web/service/datasets.ts                       | 17 ++++++++
 2 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 8bee207976dc0c..db5c4592bda613 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -13,6 +13,7 @@ import Crawling from './crawling'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import Toast from '@/app/components/base/toast'
+import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
 import { sleep } from '@/utils'
 
 const ERROR_I18N_PREFIX = 'common.errorMsg'
@@ -45,8 +46,7 @@ const FireCrawl: FC<Props> = ({
   onCheckedCrawlResultChange,
 }) => {
   const { t } = useTranslation()
-  const [step, setStep] = useState<Step>(Step.finished)
-
+  const [step, setStep] = useState<Step>(Step.init)
   const { setShowAccountSettingModal } = useModalContext()
   const handleSetting = useCallback(() => {
     setShowAccountSettingModal({
@@ -90,6 +90,25 @@ const FireCrawl: FC<Props> = ({
   const [crawlResult, setCrawlResult] = useState<CrawlResultItem[]>(mockCrawlResult)
 
   const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
+
+  const waitForCrawlFinished = useCallback(async (jobId: string) => {
+    const res = await checkFirecrawlTaskStatus(jobId) as any
+    if (res.status === 'completed') {
+      return {
+        isError: false,
+        data: res.data,
+      }
+    }
+    if (res.status === 'error') {
+      return {
+        isError: true,
+        errorMessage: res.errorMessage, // TODO: wait for the api structure
+      }
+    }
+    await sleep(2500)
+    return await waitForCrawlFinished(jobId)
+  }, [])
+
   const handleRun = useCallback(async (url: string) => {
     const { isValid, errorMsg } = checkValid(url)
     if (!isValid) {
@@ -100,13 +119,21 @@ const FireCrawl: FC<Props> = ({
       return
     }
     setStep(Step.running)
-    // TODO: crawl
-    await sleep(2000)
-    setCrawlResult(mockCrawlResult) // TODO:
+    const res = await createFirecrawlTask({
+      url,
+      options: crawlOptions,
+    }) as any
+    const jobId = res.job_id
+    const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
+    if (isError) {
+      setCrawlErrorMsg(errorMessage)
+      setCrawlResult(data)
+    }
 
     setStep(Step.finished)
+    setCrawlResult(data)
     setCrawlErrorMsg('')
-  }, [checkValid])
+  }, [checkValid, crawlOptions, waitForCrawlFinished])
 
   return (
     <div>
diff --git a/web/service/datasets.ts b/web/service/datasets.ts
index b620f2133ecfa2..91397756048a19 100644
--- a/web/service/datasets.ts
+++ b/web/service/datasets.ts
@@ -239,6 +239,23 @@ export const removeFirecrawlApiKey: Fetcher<CommonResponse, string> = (id: strin
   return del<CommonResponse>(`api-key-auth/data-source/${id}`)
 }
 
+export const createFirecrawlTask: Fetcher<CommonResponse, Record<string, any>> = (body) => {
+  return post<CommonResponse>('website/crawl', {
+    body: {
+      ...body,
+      provider: 'firecrawl',
+    },
+  })
+}
+
+export const checkFirecrawlTaskStatus: Fetcher<CommonResponse, string> = (jobId: string) => {
+  return get<CommonResponse>(`website/crawl/status/${jobId}`, {
+    params: {
+      provider: 'firecrawl',
+    },
+  })
+}
+
 type FileTypesRes = {
   allowed_extensions: string[]
 }

From a5f2b4a6e7d7a90c42715d664b0eaef6656afec8 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 7 Jun 2024 14:40:20 +0800
Subject: [PATCH 29/31] feat: firecrawel init datasets

---
 web/app/components/datasets/create/index.tsx  |  3 ++
 .../datasets/create/step-one/index.tsx        |  3 ++
 .../datasets/create/step-two/index.tsx        |  4 ++-
 .../create/website/firecrawl/index.tsx        |  5 ++-
 .../datasets/create/website/index.tsx         | 33 ++++++++++++++-----
 web/service/common.ts                         |  5 ---
 6 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx
index 30ee370be54753..21acb4a70bddc7 100644
--- a/web/app/components/datasets/create/index.tsx
+++ b/web/app/components/datasets/create/index.tsx
@@ -40,6 +40,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const updateFileList = (preparedFiles: FileItem[]) => {
     setFiles(preparedFiles)
   }
+  const [fireCrawlJobId, setFireCrawlJobId] = useState('')
 
   const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
     const targetIndex = list.findIndex(file => file.fileID === fileItem.fileID)
@@ -124,6 +125,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           onStepChange={nextStep}
           websitePages={websitePages}
           updateWebsitePages={setWebsitePages}
+          onFireCrawlJobIdChange={setFireCrawlJobId}
         />}
         {(step === 2 && (!datasetId || (datasetId && !!detail))) && <StepTwo
           isAPIKeySet={!!embeddingsDefaultModel}
@@ -134,6 +136,7 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           files={fileList.map(file => file.file)}
           notionPages={notionPages}
           websitePages={websitePages}
+          fireCrawlJobId={fireCrawlJobId}
           onStepChange={changeStep}
           updateIndexingTypeCache={updateIndexingTypeCache}
           updateResultCache={updateResultCache}
diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx
index 857cf88f589935..c013bc94a50d8c 100644
--- a/web/app/components/datasets/create/step-one/index.tsx
+++ b/web/app/components/datasets/create/step-one/index.tsx
@@ -33,6 +33,7 @@ type IStepOneProps = {
   changeType: (type: DataSourceType) => void
   websitePages?: CrawlResultItem[]
   updateWebsitePages: (value: CrawlResultItem[]) => void
+  onFireCrawlJobIdChange: (jobId: string) => void
 }
 
 type NotionConnectorProps = {
@@ -66,6 +67,7 @@ const StepOne = ({
   updateNotionPages,
   websitePages = [],
   updateWebsitePages,
+  onFireCrawlJobIdChange,
 }: IStepOneProps) => {
   const { dataset } = useDatasetDetailContext()
   const [showModal, setShowModal] = useState(false)
@@ -218,6 +220,7 @@ const StepOne = ({
                   onPreview={setCurrentWebsite}
                   checkedCrawlResult={websitePages}
                   onCheckedCrawlResultChange={updateWebsitePages}
+                  onJobIdChange={onFireCrawlJobIdChange}
                 />
               </div>
               {isShowVectorSpaceFull && (
diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx
index a0d085fd104edf..0ffc9da2a6be09 100644
--- a/web/app/components/datasets/create/step-two/index.tsx
+++ b/web/app/components/datasets/create/step-two/index.tsx
@@ -58,6 +58,7 @@ type StepTwoProps = {
   files: CustomFile[]
   notionPages?: NotionPage[]
   websitePages?: CrawlResultItem[]
+  fireCrawlJobId?: string
   onStepChange?: (delta: number) => void
   updateIndexingTypeCache?: (type: string) => void
   updateResultCache?: (res: createDocumentResponse) => void
@@ -85,6 +86,7 @@ const StepTwo = ({
   files,
   notionPages = [],
   websitePages = [],
+  fireCrawlJobId = '',
   onStepChange,
   updateIndexingTypeCache,
   updateResultCache,
@@ -248,7 +250,7 @@ const StepTwo = ({
   const getWebsiteInfo = () => {
     return {
       provider: 'firecrawl',
-      job_id: 'xxx',
+      job_id: fireCrawlJobId,
       urls: websitePages.map(page => page.source_url),
     }
   }
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index db5c4592bda613..808c444d701018 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -23,6 +23,7 @@ type Props = {
   onPreview: (payload: CrawlResultItem) => void
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onJobIdChange: (jobId: string) => void
 }
 
 const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
@@ -44,6 +45,7 @@ const FireCrawl: FC<Props> = ({
   onPreview,
   checkedCrawlResult,
   onCheckedCrawlResultChange,
+  onJobIdChange,
 }) => {
   const { t } = useTranslation()
   const [step, setStep] = useState<Step>(Step.init)
@@ -124,6 +126,7 @@ const FireCrawl: FC<Props> = ({
       options: crawlOptions,
     }) as any
     const jobId = res.job_id
+    onJobIdChange(jobId)
     const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
     if (isError) {
       setCrawlErrorMsg(errorMessage)
@@ -133,7 +136,7 @@ const FireCrawl: FC<Props> = ({
     setStep(Step.finished)
     setCrawlResult(data)
     setCrawlErrorMsg('')
-  }, [checkValid, crawlOptions, waitForCrawlFinished])
+  }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished])
 
   return (
     <div>
diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx
index 799bcb76ecf1ae..9230a999fe5573 100644
--- a/web/app/components/datasets/create/website/index.tsx
+++ b/web/app/components/datasets/create/website/index.tsx
@@ -1,45 +1,60 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback, useState } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
 import NoData from './no-data'
 import Firecrawl from './firecrawl'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlResultItem } from '@/models/datasets'
+import { fetchFirecrawlApiKey } from '@/service/datasets'
+import { type DataSourceWebsiteItem, WebsiteProvider } from '@/models/common'
 
 type Props = {
   onPreview: (payload: CrawlResultItem) => void
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onJobIdChange: (jobId: string) => void
 }
 
-const WebsitePreview: FC<Props> = ({
+const Website: FC<Props> = ({
   onPreview,
   checkedCrawlResult,
   onCheckedCrawlResultChange,
+  onJobIdChange,
 }) => {
   const { setShowAccountSettingModal } = useModalContext()
   const [isLoaded, setIsLoaded] = useState(false)
-  const [isConfigured, setIsConfigured] = useState(true)
+  const [isSetFirecrawlApiKey, setIsSetFirecrawlApiKey] = useState(false)
+  const checkSetApiKey = useCallback(async () => {
+    const res = await fetchFirecrawlApiKey() as any
+    const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled)
+    setIsSetFirecrawlApiKey(list.length > 0)
+  }, [])
 
+  useEffect(() => {
+    checkSetApiKey().then(() => {
+      setIsLoaded(true)
+    })
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
   const handleOnConfig = useCallback(() => {
     setShowAccountSettingModal({
       payload: 'data-source',
+      onCancelCallback: checkSetApiKey,
     })
-  }, [setShowAccountSettingModal])
-
-  // TODO: on Hide account setting modal
+  }, [checkSetApiKey, setShowAccountSettingModal])
 
-  if (isLoaded)
+  if (!isLoaded)
     return null
 
   return (
     <div>
-      {isConfigured
+      {isSetFirecrawlApiKey
         ? (
           <Firecrawl
             onPreview={onPreview}
             checkedCrawlResult={checkedCrawlResult}
             onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+            onJobIdChange={onJobIdChange}
           />
         )
         : (
@@ -48,4 +63,4 @@ const WebsitePreview: FC<Props> = ({
     </div>
   )
 }
-export default React.memo(WebsitePreview)
+export default React.memo(Website)
diff --git a/web/service/common.ts b/web/service/common.ts
index fa0b2ffc5ffa16..a68aeb2256391b 100644
--- a/web/service/common.ts
+++ b/web/service/common.ts
@@ -6,7 +6,6 @@ import type {
   CodeBasedExtension,
   CommonResponse,
   DataSourceNotion,
-  DataSourceWebsite,
   FileUploadConfigResponse,
   ICurrentWorkspace,
   IWorkspace,
@@ -146,10 +145,6 @@ export const updateDataSourceNotionAction: Fetcher<CommonResponse, { url: string
   return patch<CommonResponse>(url)
 }
 
-export const fetchWebsiteDataSource = () => {
-  return get<{ data: DataSourceWebsite }>('/api-key-auth/data-source')
-}
-
 export const fetchPluginProviders: Fetcher<PluginProvider[] | null, string> = (url) => {
   return get<PluginProvider[] | null>(url)
 }

From 2e3b9001fc956e44535c363a82d90007ed949674 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 7 Jun 2024 14:56:55 +0800
Subject: [PATCH 30/31] feat: handle error show

---
 .../create/website/firecrawl/base/error-message.tsx |  6 ++++--
 .../create/website/firecrawl/base/options-wrap.tsx  |  8 ++++----
 .../datasets/create/website/firecrawl/index.tsx     | 13 +++++++------
 web/i18n/en-US/dataset-creation.ts                  |  2 +-
 web/i18n/zh-Hans/dataset-creation.ts                |  2 +-
 5 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx b/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx
index 6e78286fe69a7d..3af234e09f5180 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/error-message.tsx
@@ -7,7 +7,7 @@ import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/aler
 type Props = {
   className?: string
   title: string
-  errorMsg: string
+  errorMsg?: string
 }
 
 const ErrorMessage: FC<Props> = ({
@@ -21,7 +21,9 @@ const ErrorMessage: FC<Props> = ({
         <AlertTriangle className='mr-2 w-4 h-4 text-[#F79009]' />
         <div className='text-sm font-medium text-[#DC6803]'>{title}</div>
       </div>
-      <div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
+      {errorMsg && (
+        <div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
+      )}
     </div>
   )
 }
diff --git a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
index 61c89fc10745df..aef5097c75d46b 100644
--- a/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/base/options-wrap.tsx
@@ -13,13 +13,13 @@ type Props = {
   className?: string
   children: React.ReactNode
   isFilledFull?: boolean
-  errorMsg?: string
+  hasError?: boolean
 }
 
 const OptionsWrap: FC<Props> = ({
   className = '',
   children,
-  errorMsg,
+  hasError,
   isFilledFull = false,
 }) => {
   const { t } = useTranslation()
@@ -41,10 +41,10 @@ const OptionsWrap: FC<Props> = ({
       </div>
       {!fold && (
         <div className={cn(isFilledFull && 'mt-3 relative left-[-12px] w-[calc(100%_+_24px)] rounded-b-xl')}>
-          {!errorMsg
+          {!hasError
             ? children
             : (
-              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={errorMsg} />
+              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} />
             )}
         </div>
       )}
diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 808c444d701018..106ef12cd6b4f8 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -91,7 +91,7 @@ const FireCrawl: FC<Props> = ({
   const isRunning = step === Step.running
   const [crawlResult, setCrawlResult] = useState<CrawlResultItem[]>(mockCrawlResult)
 
-  const [crawlErrorMsg, setCrawlErrorMsg] = useState('')
+  const [crawlHasError, setCrawlHasError] = useState(false)
 
   const waitForCrawlFinished = useCallback(async (jobId: string) => {
     const res = await checkFirecrawlTaskStatus(jobId) as any
@@ -102,9 +102,9 @@ const FireCrawl: FC<Props> = ({
       }
     }
     if (res.status === 'error') {
+      // can't get the error message from the firecrawl api
       return {
         isError: true,
-        errorMessage: res.errorMessage, // TODO: wait for the api structure
       }
     }
     await sleep(2500)
@@ -120,6 +120,7 @@ const FireCrawl: FC<Props> = ({
       })
       return
     }
+    setCrawlHasError(false)
     setStep(Step.running)
     const res = await createFirecrawlTask({
       url,
@@ -127,15 +128,15 @@ const FireCrawl: FC<Props> = ({
     }) as any
     const jobId = res.job_id
     onJobIdChange(jobId)
-    const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
+    const { isError, data } = await waitForCrawlFinished(jobId)
     if (isError) {
-      setCrawlErrorMsg(errorMessage)
+      setCrawlHasError(true)
       setCrawlResult(data)
     }
 
     setStep(Step.finished)
     setCrawlResult(data)
-    setCrawlErrorMsg('')
+    setCrawlHasError(false)
   }, [checkValid, crawlOptions, onJobIdChange, waitForCrawlFinished])
 
   return (
@@ -146,7 +147,7 @@ const FireCrawl: FC<Props> = ({
         <OptionsWrap
           className={cn('mt-3')}
           isFilledFull={!isInit}
-          errorMsg={isCrawlFinished ? crawlErrorMsg : ''}
+          hasError={isCrawlFinished && crawlHasError}
         >
           {isInit && <Options className='mt-2' payload={crawlOptions} onChange={setCrawlOptions} />}
           {isRunning
diff --git a/web/i18n/en-US/dataset-creation.ts b/web/i18n/en-US/dataset-creation.ts
index 466b0afc3ad16c..caff36ccb7c59e 100644
--- a/web/i18n/en-US/dataset-creation.ts
+++ b/web/i18n/en-US/dataset-creation.ts
@@ -71,7 +71,7 @@ const translation = {
       excludePaths: 'Exclude paths',
       includeOnlyPaths: 'Include only paths',
       extractOnlyMainContent: 'Extract only main content (no headers, navs, footers, etc.)',
-      exceptionErrorTitle: 'An exception occurred while running Firecrawl job:',
+      exceptionErrorTitle: 'An exception occurred while running Firecrawl job',
       totalPageScraped: 'Total pages scraped:',
       selectAll: 'Select All',
       resetAll: 'Reset All',
diff --git a/web/i18n/zh-Hans/dataset-creation.ts b/web/i18n/zh-Hans/dataset-creation.ts
index 015d3b61c8fd29..46143ecc4cc5a6 100644
--- a/web/i18n/zh-Hans/dataset-creation.ts
+++ b/web/i18n/zh-Hans/dataset-creation.ts
@@ -71,7 +71,7 @@ const translation = {
       excludePaths: '排除路径',
       includeOnlyPaths: '仅包含路径',
       extractOnlyMainContent: '仅提取主要内容（无标题、导航、页脚等）',
-      exceptionErrorTitle: '运行 Firecrawl 时发生异常:',
+      exceptionErrorTitle: '运行 Firecrawl 时发生异常',
       totalPageScraped: '抓取页面总数:',
       selectAll: '全选',
       resetAll: '重置全部',

From cd01ff7c03272bee5fe6f39e8ee0a0a4283655a0 Mon Sep 17 00:00:00 2001
From: Joel <iamjoel007@gmail.com>
Date: Fri, 7 Jun 2024 15:24:14 +0800
Subject: [PATCH 31/31] feat: handle prossed info

---
 .../create/website/firecrawl/index.tsx        | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/web/app/components/datasets/create/website/firecrawl/index.tsx b/web/app/components/datasets/create/website/firecrawl/index.tsx
index 106ef12cd6b4f8..5150d4bd5e9502 100644
--- a/web/app/components/datasets/create/website/firecrawl/index.tsx
+++ b/web/app/components/datasets/create/website/firecrawl/index.tsx
@@ -7,7 +7,6 @@ import Header from './header'
 import UrlInput from './base/url-input'
 import OptionsWrap from './base/options-wrap'
 import Options from './options'
-import mockCrawlResult from './mock-crawl-result'
 import CrawledResult from './crawled-result'
 import Crawling from './crawling'
 import { useModalContext } from '@/context/modal-context'
@@ -89,8 +88,11 @@ const FireCrawl: FC<Props> = ({
   const isInit = step === Step.init
   const isCrawlFinished = step === Step.finished
   const isRunning = step === Step.running
-  const [crawlResult, setCrawlResult] = useState<CrawlResultItem[]>(mockCrawlResult)
-
+  const [crawlResult, setCrawlResult] = useState<{
+    current: number
+    total: number
+    data: CrawlResultItem[]
+  } | undefined>(undefined)
   const [crawlHasError, setCrawlHasError] = useState(false)
 
   const waitForCrawlFinished = useCallback(async (jobId: string) => {
@@ -98,18 +100,24 @@ const FireCrawl: FC<Props> = ({
     if (res.status === 'completed') {
       return {
         isError: false,
-        data: res.data,
+        data: {
+          ...res,
+          total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
+        },
       }
     }
     if (res.status === 'error') {
       // can't get the error message from the firecrawl api
       return {
         isError: true,
+        data: {
+          data: [],
+        },
       }
     }
     await sleep(2500)
     return await waitForCrawlFinished(jobId)
-  }, [])
+  }, [crawlOptions.limit])
 
   const handleRun = useCallback(async (url: string) => {
     const { isValid, errorMsg } = checkValid(url)
@@ -153,12 +161,12 @@ const FireCrawl: FC<Props> = ({
           {isRunning
             && <Crawling
               className='mt-2'
-              crawledNum={8}
-              totalNum={10}
+              crawledNum={crawlResult?.current || 0}
+              totalNum={crawlResult?.total || parseFloat(crawlOptions.limit as string) || 0}
             />}
           {isCrawlFinished && (
             <CrawledResult
-              list={crawlResult}
+              list={crawlResult?.data || []}
               checkedList={checkedCrawlResult}
               onSelectedChange={onCheckedCrawlResultChange}
               onPreview={onPreview}