dale_yxc/ragflow_python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix the bug causing garbled text (#3640)

Browse Source 
### What problem does this PR solve?

Fix the bug causing garbled text #3613

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

Co-authored-by: liuhua <10215101452@stu.ecun.edu.cn>
This commit is contained in:
liuhua
2024-11-26 12:06:56 +08:00
committed by GitHub
parent f6c3d7ccf6
commit 5c59651bda
3 changed files with 13 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/apps/kb_app.py
View File

@@ -162,9 +162,9 @@ def rm():
message="Database error (Document removal)!") message="Database error (Document removal)!")
f2d = File2DocumentService.get_by_document_id(doc.id) f2d = File2DocumentService.get_by_document_id(doc.id)
FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id]) FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id])
FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
File2DocumentService.delete_by_document_id(doc.id) File2DocumentService.delete_by_document_id(doc.id)
FileService.filter_delete(
[File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
if not KnowledgebaseService.delete_by_id(req["kb_id"]): if not KnowledgebaseService.delete_by_id(req["kb_id"]):
return get_data_error_result( return get_data_error_result(
message="Database error (Knowledgebase removal)!") message="Database error (Knowledgebase removal)!")

2
api/apps/sdk/dataset.py
View File

@@ -252,9 +252,9 @@ def delete(tenant_id):
File.id == f2d[0].file_id, File.id == f2d[0].file_id,
] ]
) )
File2DocumentService.delete_by_document_id(doc.id)
FileService.filter_delete( FileService.filter_delete(
[File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name]) [File.source_type == FileSource.KNOWLEDGEBASE, File.type == "folder", File.name == kbs[0].name])
File2DocumentService.delete_by_document_id(doc.id)
if not KnowledgebaseService.delete_by_id(id): if not KnowledgebaseService.delete_by_id(id):
return get_error_data_result(message="Delete dataset error.(Database error)") return get_error_data_result(message="Delete dataset error.(Database error)")
return get_result(code=settings.RetCode.SUCCESS) return get_result(code=settings.RetCode.SUCCESS)

11
rag/nlp/__init__.py
View File

@@ -28,6 +28,8 @@ from cn2an import cn2an
from PIL import Image from PIL import Image
import json import json
import chardet
all_codecs = [ all_codecs = [
'utf-8', 'gb2312', 'gbk', 'utf_16', 'ascii', 'big5', 'big5hkscs', 'utf-8', 'gb2312', 'gbk', 'utf_16', 'ascii', 'big5', 'big5hkscs',
'cp037', 'cp273', 'cp424', 'cp437', 'cp037', 'cp273', 'cp424', 'cp437',
@@ -43,12 +45,17 @@ all_codecs = [
'iso8859_14', 'iso8859_15', 'iso8859_16', 'johab', 'koi8_r', 'koi8_t', 'koi8_u', 'iso8859_14', 'iso8859_15', 'iso8859_16', 'johab', 'koi8_r', 'koi8_t', 'koi8_u',
'kz1048', 'mac_cyrillic', 'mac_greek', 'mac_iceland', 'mac_latin2', 'mac_roman', 'kz1048', 'mac_cyrillic', 'mac_greek', 'mac_iceland', 'mac_latin2', 'mac_roman',
'mac_turkish', 'ptcp154', 'shift_jis', 'shift_jis_2004', 'shift_jisx0213', 'mac_turkish', 'ptcp154', 'shift_jis', 'shift_jis_2004', 'shift_jisx0213',
'utf_32', 'utf_32_be', 'utf_32_le''utf_16_be', 'utf_16_le', 'utf_7' 'utf_32', 'utf_32_be', 'utf_32_le', 'utf_16_be', 'utf_16_le', 'utf_7', 'windows-1250', 'windows-1251',
'windows-1252', 'windows-1253', 'windows-1254', 'windows-1255', 'windows-1256',
'windows-1257', 'windows-1258', 'latin-2'
] ]
def find_codec(blob): def find_codec(blob):
global all_codecs detected = chardet.detect(blob[:1024])
if detected['confidence'] > 0.5:
return detected['encoding']
for c in all_codecs: for c in all_codecs:
try: try:
blob[:1024].decode(c) blob[:1024].decode(c)