dale_yxc/ragflow_python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: select the corresponding parsing method according to the file type and after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. and remove ChunkMethodModal from knowledge-file (#158)

Browse Source 
* feat: select the corresponding parsing method according to the file type

* feat: after the document is successfully uploaded, use the ChunkMethodModal  to select the parsing method.

* feat: add pdf types to ParserListMap

* feat: remove ChunkMethodModal from knowledge-file
This commit is contained in:
balibabu
2024-03-27 17:56:34 +08:00
committed by GitHub
parent bf2e3d7fc1
commit 44541a8c33
14 changed files with 340 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
web/src/pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx
View File

@@ -1,278 +0,0 @@
import MaxTokenNumber from '@/components/max-token-number';
import { IModalManagerChildrenProps } from '@/components/modal-manager';
import { IKnowledgeFileParserConfig } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import {
MinusCircleOutlined,
PlusOutlined,
QuestionCircleOutlined,
} from '@ant-design/icons';
import {
Button,
Divider,
Form,
InputNumber,
Modal,
Space,
Switch,
Tag,
Tooltip,
} from 'antd';
import omit from 'lodash/omit';
import React, { useEffect, useMemo } from 'react';
import { useFetchParserListOnMount } from './hooks';
import styles from './index.less';
const { CheckableTag } = Tag;
interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
loading: boolean;
onOk: (
parserId: string,
parserConfig: IChangeParserConfigRequestBody,
) => void;
showModal?(): void;
parserId: string;
parserConfig: IKnowledgeFileParserConfig;
documentType: string;
}
const hidePagesChunkMethods = ['qa', 'table', 'picture', 'resume', 'one'];
const ChunkMethodModal: React.FC<IProps> = ({
parserId,
onOk,
hideModal,
visible,
documentType,
parserConfig,
}) => {
const { parserList, handleChange, selectedTag } =
useFetchParserListOnMount(parserId);
const [form] = Form.useForm();
const handleOk = async () => {
const values = await form.validateFields();
const parser_config = {
...values.parser_config,
pages: values.pages?.map((x: any) => [x.from, x.to]) ?? [],
};
onOk(selectedTag, parser_config);
};
const showPages = useMemo(() => {
return (
documentType === 'pdf' &&
hidePagesChunkMethods.every((x) => x !== selectedTag)
);
}, [documentType, selectedTag]);
const showOne = useMemo(() => {
return showPages || selectedTag === 'one';
}, [showPages, selectedTag]);
const afterClose = () => {
form.resetFields();
};
useEffect(() => {
if (visible) {
const pages =
parserConfig.pages?.map((x) => ({ from: x[0], to: x[1] })) ?? [];
form.setFieldsValue({
pages: pages.length > 0 ? pages : [{ from: 1, to: 1024 }],
parser_config: omit(parserConfig, 'pages'),
});
}
}, [form, parserConfig, visible]);
return (
<Modal
title="Chunk Method"
open={visible}
onOk={handleOk}
onCancel={hideModal}
afterClose={afterClose}
>
<Space size={[0, 8]} wrap>
<div className={styles.tags}>
{parserList.map((x) => {
return (
<CheckableTag
key={x.value}
checked={selectedTag === x.value}
onChange={(checked) => {
handleChange(x.value, checked);
}}
>
{x.label}
</CheckableTag>
);
})}
</div>
</Space>
<Divider></Divider>
{
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
{showPages && (
<>
<Space>
<p>Page Ranges:</p>
<Tooltip
title={
'page ranges: Define the page ranges that need to be parsed. The pages that not included in these ranges will be ignored.'
}
>
<QuestionCircleOutlined
className={styles.questionIcon}
></QuestionCircleOutlined>
</Tooltip>
</Space>
<Form.List name="pages">
{(fields, { add, remove }) => (
<>
{fields.map(({ key, name, ...restField }) => (
<Space
key={key}
style={{
display: 'flex',
}}
align="baseline"
>
<Form.Item
{...restField}
name={[name, 'from']}
dependencies={name > 0 ? [name - 1, 'to'] : []}
rules={[
{
required: true,
message: 'Missing start page number',
},
({ getFieldValue }) => ({
validator(_, value) {
if (
name === 0 ||
!value ||
getFieldValue(['pages', name - 1, 'to']) <
value
) {
return Promise.resolve();
}
return Promise.reject(
new Error(
'The current value must be greater than the previous to!',
),
);
},
}),
]}
>
<InputNumber
placeholder="from"
min={0}
precision={0}
className={styles.pageInputNumber}
/>
</Form.Item>
<Form.Item
{...restField}
name={[name, 'to']}
dependencies={[name, 'from']}
rules={[
{
required: true,
message: 'Missing end page number(excluded)',
},
({ getFieldValue }) => ({
validator(_, value) {
if (
!value ||
getFieldValue(['pages', name, 'from']) < value
) {
return Promise.resolve();
}
return Promise.reject(
new Error(
'The current value must be greater than to!',
),
);
},
}),
]}
>
<InputNumber
placeholder="to"
min={0}
precision={0}
className={styles.pageInputNumber}
/>
</Form.Item>
{name > 0 && (
<MinusCircleOutlined onClick={() => remove(name)} />
)}
</Space>
))}
<Form.Item>
<Button
type="dashed"
onClick={() => add()}
block
icon={<PlusOutlined />}
>
Add page
</Button>
</Form.Item>
</>
)}
</Form.List>
</>
)}
{showOne && (
<Form.Item
name={['parser_config', 'layout_recognize']}
label="Layout recognize"
initialValue={true}
valuePropName="checked"
tooltip={
'Use visual models for layout analysis to better identify document structure, find where the titles, text blocks, images, and tables are. Without this feature, only the plain text of the PDF can be obtained.'
}
>
<Switch />
</Form.Item>
)}
{showPages && (
<Form.Item
noStyle
dependencies={[['parser_config', 'layout_recognize']]}
>
{({ getFieldValue }) =>
getFieldValue(['parser_config', 'layout_recognize']) && (
<Form.Item
name={['parser_config', 'task_page_size']}
label="Task page size"
tooltip={`If using layout recognize, the PDF file will be split into groups of successive. Layout analysis will be performed parallelly between groups to increase the processing speed.
The 'Task page size' determines the size of groups. The larger the page size is, the lower the chance of splitting continuous text between pages into different chunks.`}
initialValue={12}
rules={[
{
required: true,
message: 'Please input your task page size!',
},
]}
>
<InputNumber min={1} max={128} />
</Form.Item>
)
}
</Form.Item>
)}
{selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
</Form>
}
</Modal>
);
};
export default ChunkMethodModal;

23
web/src/pages/add-knowledge/components/knowledge-file/hooks.ts
View File

@@ -7,10 +7,7 @@ import {
} from '@/hooks/documentHooks';
import { useGetKnowledgeSearchParams } from '@/hooks/routeHook';
import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { Pagination } from '@/interfaces/common';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
@@ -243,21 +240,3 @@ export const useChangeDocumentParser = (documentId: string) => {
showChangeParserModal,
};
};
export const useFetchParserListOnMount = (parserId: string) => {
const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList();
useFetchTenantInfo();
useEffect(() => {
setSelectedTag(parserId);
}, [parserId]);
const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
setSelectedTag(nextSelectedTag);
};
return { parserList, handleChange, selectedTag };
};

5
web/src/pages/add-knowledge/components/knowledge-file/index.tsx
View File

@@ -23,7 +23,6 @@ import {
} from 'antd';
import type { ColumnsType } from 'antd/es/table';
import { useMemo } from 'react';
import ChunkMethodModal from './chunk-method-modal';
import CreateFileModal from './create-file-modal';
import {
useChangeDocumentParser,
@@ -39,6 +38,8 @@ import ParsingActionCell from './parsing-action-cell';
import ParsingStatusCell from './parsing-status-cell';
import RenameModal from './rename-modal';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { getExtension } from '@/utils/documentUtils';
import styles from './index.less';
const KnowledgeFile = () => {
@@ -227,7 +228,7 @@ const KnowledgeFile = () => {
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentType={currentRecord.type}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}

13
web/src/pages/add-knowledge/components/knowledge-file/model.ts
View File

@@ -209,6 +209,19 @@ const model: DvaModel<KFModelState> = {
console.warn(error);
}
},
*upload_document({ payload = {} }, { call, put }) {
const formData = new FormData();
formData.append('file', payload.file);
formData.append('kb_id', payload.kb_id);
const { data } = yield call(kbService.document_upload, formData);
if (data.retcode === 0) {
yield put({
type: 'getKfList',
payload: { kb_id: payload.kb_id },
});
}
return data;
},
},
};
export default model;

42
web/src/pages/add-knowledge/components/knowledge-file/upload.tsx
View File

@@ -1,42 +0,0 @@
import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
import uploadService from '@/services/uploadService';
import type { UploadProps } from 'antd';
import React from 'react';
import { Link } from 'umi';
interface PropsType {
kb_id: string;
getKfList: () => void;
}
type UploadRequestOption = Parameters<
NonNullable<UploadProps['customRequest']>
>[0];
const FileUpload: React.FC<PropsType> = ({ kb_id, getKfList }) => {
const knowledgeBaseId = useKnowledgeBaseId();
const createRequest: (props: UploadRequestOption) => void = async function ({
file,
onSuccess,
onError,
}) {
const { retcode, data } = await uploadService.uploadFile(file, kb_id);
if (retcode === 0) {
onSuccess && onSuccess(data, file);
} else {
onError && onError(data);
}
getKfList && getKfList();
};
const uploadProps: UploadProps = {
customRequest: createRequest,
showUploadList: false,
};
return (
// <Upload {...uploadProps}>
<Link to={`/knowledge/dataset/upload?id=${knowledgeBaseId}`}>导入文件</Link>
// </Upload>
);
};
export default FileUpload;