Add agent component for web crawler (#2878)

### What problem does this PR solve?

Add agent component for  web crawler

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

web/src/pages/flow/constant.tsx

@@ -4,6 +4,7 @@ import { ReactComponent as baiduFanyiIcon } from '@/assets/svg/baidu-fanyi.svg';
 import { ReactComponent as BaiduIcon } from '@/assets/svg/baidu.svg';
 import { ReactComponent as BingIcon } from '@/assets/svg/bing.svg';
 import { ReactComponent as ConcentratorIcon } from '@/assets/svg/concentrator.svg';
+import { ReactComponent as CrawlerIcon } from '@/assets/svg/crawler.svg';
 import { ReactComponent as DeepLIcon } from '@/assets/svg/deepl.svg';
 import { ReactComponent as DuckIcon } from '@/assets/svg/duck.svg';
 import { ReactComponent as ExeSqlIcon } from '@/assets/svg/exesql.svg';
@@ -73,6 +74,7 @@ export enum Operator {
   Concentrator = 'Concentrator',
   TuShare = 'TuShare',
   Note = 'Note',
+  Crawler = 'Crawler',
 }
 
 export const CommonOperatorList = Object.values(Operator).filter(
@@ -110,6 +112,7 @@ export const operatorIconMap = {
   [Operator.Concentrator]: ConcentratorIcon,
   [Operator.TuShare]: TuShareIcon,
   [Operator.Note]: NoteIcon,
+  [Operator.Crawler]: CrawlerIcon,
 };
 
 export const operatorMap: Record<
@@ -233,6 +236,9 @@ export const operatorMap: Record<
   },
   [Operator.TuShare]: { backgroundColor: '#f8cfa0' },
   [Operator.Note]: { backgroundColor: '#f8cfa0' },
+  [Operator.Crawler]: {
+    backgroundColor: '#dee0e2',
+  },
 };
 
 export const componentMenuList = [
@@ -323,6 +329,9 @@ export const componentMenuList = [
   {
     name: Operator.TuShare,
   },
+  {
+    name: Operator.Crawler,
+  },
 ];
 
 export const initialRetrievalValues = {
@@ -572,6 +581,7 @@ export const RestrictedUpstreamMap = {
   [Operator.Jin10]: [Operator.Begin],
   [Operator.Concentrator]: [Operator.Begin],
   [Operator.TuShare]: [Operator.Begin],
+  [Operator.Crawler]: [Operator.Begin],
 };
 
 export const NodeMap = {
@@ -605,6 +615,7 @@ export const NodeMap = {
   [Operator.Jin10]: 'ragNode',
   [Operator.TuShare]: 'ragNode',
   [Operator.Note]: 'noteNode',
+  [Operator.Crawler]: 'ragNode',
 };
 
 export const LanguageOptions = [
@@ -2791,3 +2802,4 @@ export const TuShareSrcOptions = [
   'fenghuang',
   'jinrongjie',
 ];
+export const CrawlerResultOptions = ['markdown', 'html', 'content'];

web/src/pages/flow/flow-drawer/index.tsx

@@ -12,6 +12,7 @@ import BaiduForm from '../form/baidu-form';
 import BeginForm from '../form/begin-form';
 import BingForm from '../form/bing-form';
 import CategorizeForm from '../form/categorize-form';
+import CrawlerForm from '../form/crawler-form';
 import DeepLForm from '../form/deepl-form';
 import DuckDuckGoForm from '../form/duckduckgo-form';
 import ExeSQLForm from '../form/exesql-form';
@@ -70,6 +71,7 @@ const FormMap = {
   [Operator.YahooFinance]: YahooFinanceForm,
   [Operator.Jin10]: Jin10Form,
   [Operator.TuShare]: TuShareForm,
+  [Operator.Crawler]: CrawlerForm,
 };
 
 const EmptyContent = () => <div>empty</div>;

web/src/pages/flow/form/crawler-form/index.tsx

@@ -0,0 +1,37 @@
+import { useTranslate } from '@/hooks/common-hooks';
+import { Form, Input, Select } from 'antd';
+import { useMemo } from 'react';
+import { CrawlerResultOptions } from '../../constant';
+import { IOperatorForm } from '../../interface';
+const CrawlerForm = ({ onValuesChange, form }: IOperatorForm) => {
+  const { t } = useTranslate('flow');
+  const crawlerResultOptions = useMemo(() => {
+    return CrawlerResultOptions.map((x) => ({
+      value: x,
+      label: t(`crawlerResultOptions.${x}`),
+    }));
+  }, [t]);
+  return (
+    <Form
+      name="basic"
+      labelCol={{ span: 6 }}
+      wrapperCol={{ span: 18 }}
+      autoComplete="off"
+      form={form}
+      onValuesChange={onValuesChange}
+    >
+      <Form.Item label={t('proxy')} name={'proxy'}>
+        <Input placeholder="like: http://127.0.0.1:8888"></Input>
+      </Form.Item>
+      <Form.Item
+        label={t('extractType')}
+        name={'extract_type'}
+        initialValue="markdown"
+      >
+        <Select options={crawlerResultOptions}></Select>
+      </Form.Item>
+    </Form>
+  );
+};
+
+export default CrawlerForm;