update document sdk (#2445)

### What problem does this PR solve?


### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
This commit is contained in:
JobSmithManipulation
2024-09-18 11:08:19 +08:00
committed by GitHub
parent e7dd487779
commit 62cb5f1bac
5 changed files with 348 additions and 58 deletions

View File

@@ -3,32 +3,48 @@ from .base import Base
class Chunk(Base):
def __init__(self, rag, res_dict):
# 初始化类的属性
self.id = ""
self.content_with_weight = ""
self.content_ltks = []
self.content_sm_ltks = []
self.important_kwd = []
self.important_tks = []
self.content = ""
self.important_keywords = []
self.create_time = ""
self.create_timestamp_flt = 0.0
self.kb_id = None
self.docnm_kwd = ""
self.doc_id = ""
self.q_vec = []
self.knowledgebase_id = None
self.document_name = ""
self.document_id = ""
self.status = "1"
for k, v in res_dict.items():
if hasattr(self, k):
setattr(self, k, v)
for k in list(res_dict.keys()):
if k not in self.__dict__:
res_dict.pop(k)
super().__init__(rag, res_dict)
def delete(self) -> bool:
"""
Delete the chunk in the document.
"""
res = self.rm('/doc/chunk/rm',
{"doc_id": [self.id],""})
res = self.post('/doc/chunk/rm',
{"doc_id": self.document_id, 'chunk_ids': [self.id]})
res = res.json()
if res.get("retmsg") == "success":
return True
raise Exception(res["retmsg"])
raise Exception(res["retmsg"])
def save(self) -> bool:
"""
Save the document details to the server.
"""
res = self.post('/doc/chunk/set',
{"chunk_id": self.id,
"kb_id": self.knowledgebase_id,
"name": self.document_name,
"content_with_weight": self.content,
"important_kwd": self.important_keywords,
"create_time": self.create_time,
"create_timestamp_flt": self.create_timestamp_flt,
"doc_id": self.document_id,
"status": self.status,
})
res = res.json()
if res.get("retmsg") == "success":
return True
raise Exception(res["retmsg"])

View File

@@ -9,15 +9,15 @@ class Document(Base):
self.id = ""
self.name = ""
self.thumbnail = None
self.kb_id = None
self.knowledgebase_id = None
self.parser_method = ""
self.parser_config = {"pages": [[1, 1000000]]}
self.source_type = "local"
self.type = ""
self.created_by = ""
self.size = 0
self.token_num = 0
self.chunk_num = 0
self.token_count = 0
self.chunk_count = 0
self.progress = 0.0
self.progress_msg = ""
self.process_begin_at = None
@@ -34,10 +34,10 @@ class Document(Base):
Save the document details to the server.
"""
res = self.post('/doc/save',
{"id": self.id, "name": self.name, "thumbnail": self.thumbnail, "kb_id": self.kb_id,
{"id": self.id, "name": self.name, "thumbnail": self.thumbnail, "kb_id": self.knowledgebase_id,
"parser_id": self.parser_method, "parser_config": self.parser_config.to_json(),
"source_type": self.source_type, "type": self.type, "created_by": self.created_by,
"size": self.size, "token_num": self.token_num, "chunk_num": self.chunk_num,
"size": self.size, "token_num": self.token_count, "chunk_num": self.chunk_count,
"progress": self.progress, "progress_msg": self.progress_msg,
"process_begin_at": self.process_begin_at, "process_duation": self.process_duration
})
@@ -177,8 +177,10 @@ class Document(Base):
if res.status_code == 200:
res_data = res.json()
if res_data.get("retmsg") == "success":
chunks = res_data["data"]["chunks"]
self.chunks = chunks # Store the chunks in the document instance
chunks=[]
for chunk_data in res_data["data"].get("chunks", []):
chunk=Chunk(self.rag,chunk_data)
chunks.append(chunk)
return chunks
else:
raise Exception(f"Error fetching chunks: {res_data.get('retmsg')}")
@@ -187,10 +189,9 @@ class Document(Base):
def add_chunk(self, content: str):
res = self.post('/doc/chunk/create', {"doc_id": self.id, "content_with_weight":content})
# 假设返回的 response 包含 chunk 的信息
if res.status_code == 200:
chunk_data = res.json()
return Chunk(self.rag,chunk_data) # 假设有一个 Chunk 类来处理 chunk 对象
res_data = res.json().get("data")
chunk_data = res_data.get("chunk")
return Chunk(self.rag,chunk_data)
else:
raise Exception(f"Failed to add chunk: {res.status_code} {res.text}")