Use consistent log file names, introduced initLogger (#3403)

### What problem does this PR solve? Use consistent log file names, introduced initLogger ### Type of change - [ ] Bug Fix (non-breaking change which fixes an issue) - [ ] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [x] Refactoring - [ ] Performance Improvement - [ ] Other (please describe):
2024-11-14 17:13:48 +08:00
parent ab4384e011
commit 30f6421760
75 changed files with 396 additions and 402 deletions
--- a/api/db/db_models.py
+++ b/api/db/db_models.py
@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import inspect
 import os
 import sys
@@ -32,7 +33,6 @@ from playhouse.pool import PooledMySQLDatabase, PooledPostgresqlDatabase
 from api.db import SerializedType, ParserType
 from api.settings import DATABASE, SECRET_KEY, DATABASE_TYPE
 from api import utils
-from api.utils.log_utils import logger

 def singleton(cls, *args, **kw):
    instances = {}
@@ -285,7 +285,7 @@ class BaseDataBase:
        database_config = DATABASE.copy()
        db_name = database_config.pop("name")
        self.database_connection = PooledDatabase[DATABASE_TYPE.upper()].value(db_name, **database_config)
-        logger.info('init database on cluster mode successfully')
+        logging.info('init database on cluster mode successfully')

 class PostgresDatabaseLock:
    def __init__(self, lock_name, timeout=10, db=None):
@@ -393,7 +393,7 @@ def close_connection():
        if DB:
            DB.close_stale(age=30)
    except Exception as e:
-        logger.exception(e)
+        logging.exception(e)


 class DataBaseModel(BaseModel):
@@ -409,15 +409,15 @@ def init_database_tables(alter_fields=[]):
    for name, obj in members:
        if obj != DataBaseModel and issubclass(obj, DataBaseModel):
            table_objs.append(obj)
-            logger.info(f"start create table {obj.__name__}")
+            logging.debug(f"start create table {obj.__name__}")
            try:
                obj.create_table()
-                logger.info(f"create table success: {obj.__name__}")
+                logging.debug(f"create table success: {obj.__name__}")
            except Exception as e:
-                logger.exception(e)
+                logging.exception(e)
                create_failed_list.append(obj.__name__)
    if create_failed_list:
-        logger.info(f"create tables failed: {create_failed_list}")
+        logging.error(f"create tables failed: {create_failed_list}")
        raise Exception(f"create tables failed: {create_failed_list}")
    migrate_db()

--- a/api/db/init_data.py
+++ b/api/db/init_data.py
@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import base64
 import json
 import os
@@ -30,7 +31,6 @@ from api.db.services.llm_service import LLMFactoriesService, LLMService, TenantL
 from api.db.services.user_service import TenantService, UserTenantService
 from api.settings import CHAT_MDL, EMBEDDING_MDL, ASR_MDL, IMAGE2TEXT_MDL, PARSERS, LLM_FACTORY, API_KEY, LLM_BASE_URL
 from api.utils.file_utils import get_project_base_directory
-from api.utils.log_utils import logger


 def encode_to_base64(input_string):
@@ -70,26 +70,26 @@ def init_superuser():
             "api_key": API_KEY, "api_base": LLM_BASE_URL})

    if not UserService.save(**user_info):
-        logger.info("can't init admin.")
+        logging.error("can't init admin.")
        return
    TenantService.insert(**tenant)
    UserTenantService.insert(**usr_tenant)
    TenantLLMService.insert_many(tenant_llm)
-    logger.info(
-        "Super user initialized. email: admin@ragflow.io, password: admin. Changing the password after logining is strongly recomanded.")
+    logging.info(
+        "Super user initialized. email: admin@ragflow.io, password: admin. Changing the password after login is strongly recommended.")

    chat_mdl = LLMBundle(tenant["id"], LLMType.CHAT, tenant["llm_id"])
    msg = chat_mdl.chat(system="", history=[
                        {"role": "user", "content": "Hello!"}], gen_conf={})
    if msg.find("ERROR: ") == 0:
-        logger.error(
+        logging.error(
            "'{}' dosen't work. {}".format(
                tenant["llm_id"],
                msg))
    embd_mdl = LLMBundle(tenant["id"], LLMType.EMBEDDING, tenant["embd_id"])
    v, c = embd_mdl.encode(["Hello!"])
    if c == 0:
-        logger.error(
+        logging.error(
            "'{}' dosen't work!".format(
                tenant["embd_id"]))

@@ -172,7 +172,7 @@ def add_graph_templates():
            except:
                CanvasTemplateService.update_by_id(cnvs["id"], cnvs)
        except Exception:
-            logger.exception("Add graph templates error: ")
+            logging.exception("Add graph templates error: ")


 def init_web_data():
@@ -183,7 +183,7 @@ def init_web_data():
    #    init_superuser()

    add_graph_templates()
-    logger.info("init web data success:{}".format(time.time() - start_time))
+    logging.info("init web data success:{}".format(time.time() - start_time))


 if __name__ == '__main__':
--- a/api/db/services/dialog_service.py
+++ b/api/db/services/dialog_service.py
@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import binascii
 import os
 import json
@@ -31,7 +32,6 @@ from rag.app.resume import forbidden_select_fields4resume
 from rag.nlp.search import index_name
 from rag.utils import rmSpace, num_tokens_from_string, encoder
 from api.utils.file_utils import get_project_base_directory
-from api.utils.log_utils import logger


 class DialogService(CommonService):
@@ -178,7 +178,7 @@ def chat(dialog, messages, stream=True, **kwargs):
        tts_mdl = LLMBundle(dialog.tenant_id, LLMType.TTS)
    # try to use sql if field mapping is good to go
    if field_map:
-        logger.info("Use SQL to retrieval:{}".format(questions[-1]))
+        logging.debug("Use SQL to retrieval:{}".format(questions[-1]))
        ans = use_sql(questions[-1], field_map, dialog.tenant_id, chat_mdl, prompt_config.get("quote", True))
        if ans:
            yield ans
@@ -220,7 +220,7 @@ def chat(dialog, messages, stream=True, **kwargs):
                                        doc_ids=attachments,
                                        top=dialog.top_k, aggs=False, rerank_mdl=rerank_mdl)
    knowledges = [ck["content_with_weight"] for ck in kbinfos["chunks"]]
-    logger.info(
+    logging.debug(
        "{}->{}".format(" ".join(questions), "\n->".join(knowledges)))
    retrieval_tm = timer()

@@ -292,7 +292,7 @@ def chat(dialog, messages, stream=True, **kwargs):
        yield decorate_answer(answer)
    else:
        answer = chat_mdl.chat(prompt, msg[1:], gen_conf)
-        logger.info("User: {}|Assistant: {}".format(
+        logging.debug("User: {}|Assistant: {}".format(
            msg[-1]["content"], answer))
        res = decorate_answer(answer)
        res["audio_binary"] = tts(tts_mdl, answer)
@@ -320,7 +320,7 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
        nonlocal sys_prompt, user_promt, question, tried_times
        sql = chat_mdl.chat(sys_prompt, [{"role": "user", "content": user_promt}], {
            "temperature": 0.06})
-        logger.info(f"{question} ==> {user_promt} get SQL: {sql}")
+        logging.debug(f"{question} ==> {user_promt} get SQL: {sql}")
        sql = re.sub(r"[\r\n]+", " ", sql.lower())
        sql = re.sub(r".*select ", "select ", sql.lower())
        sql = re.sub(r" +", " ", sql)
@@ -340,7 +340,7 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
                    flds.append(k)
                sql = "select doc_id,docnm_kwd," + ",".join(flds) + sql[8:]

-        logger.info(f"{question} get SQL(refined): {sql}")
+        logging.debug(f"{question} get SQL(refined): {sql}")
        tried_times += 1
        return retrievaler.sql_retrieval(sql, format="json"), sql

@@ -369,9 +369,9 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
            question, sql, tbl["error"]
        )
        tbl, sql = get_table()
-        logger.info("TRY it again: {}".format(sql))
+        logging.debug("TRY it again: {}".format(sql))

-    logger.info("GET table: {}".format(tbl))
+    logging.debug("GET table: {}".format(tbl))
    if tbl.get("error") or len(tbl["rows"]) == 0:
        return None

@@ -401,7 +401,7 @@ def use_sql(question, field_map, tenant_id, chat_mdl, quota=True):
    rows = re.sub(r"T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|", "|", rows)

    if not docid_idx or not docnm_idx:
-        logger.warning("SQL missing field: " + sql)
+        logging.warning("SQL missing field: " + sql)
        return {
            "answer": "\n".join([clmns, line, rows]),
            "reference": {"chunks": [], "doc_aggs": []},
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import hashlib
 import json
 import random
@@ -39,7 +40,6 @@ from api.db.services.common_service import CommonService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db import StatusEnum
 from rag.utils.redis_conn import REDIS_CONN
-from api.utils.log_utils import logger


 class DocumentService(CommonService):
@@ -387,7 +387,7 @@ class DocumentService(CommonService):
                cls.update_by_id(d["id"], info)
            except Exception as e:
                if str(e).find("'0'") < 0:
-                    logger.exception("fetch task exception")
+                    logging.exception("fetch task exception")

    @classmethod
    @DB.connection_context()
@@ -544,7 +544,7 @@ def doc_upload_and_parse(conversation_id, file_objs, user_id):
                    "knowledge_graph_kwd": "mind_map"
                })
            except Exception as e:
-                logger.exception("Mind map generation error")
+                logging.exception("Mind map generation error")

        vects = embedding(doc_id, [c["content_with_weight"] for c in cks])
        assert len(cks) == len(vects)
--- a/api/db/services/file_service.py
+++ b/api/db/services/file_service.py
@@ -13,6 +13,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 import re
 import os
 from concurrent.futures import ThreadPoolExecutor
@@ -30,7 +31,6 @@ from api.db.services.file2document_service import File2DocumentService
 from api.utils import get_uuid
 from api.utils.file_utils import filename_type, thumbnail_img
 from rag.utils.storage_factory import STORAGE_IMPL
-from api.utils.log_utils import logger


 class FileService(CommonService):
@@ -276,7 +276,7 @@ class FileService(CommonService):
            return cls.model.delete().where((cls.model.tenant_id == user_id)
                                            & (cls.model.id == folder_id)).execute(),
        except Exception:
-            logger.exception("delete_folder_by_pf_id")
+            logging.exception("delete_folder_by_pf_id")
            raise RuntimeError("Database error (File retrieval)!")

    @classmethod
@@ -325,7 +325,7 @@ class FileService(CommonService):
        try:
            cls.filter_update((cls.model.id << file_ids, ), { 'parent_id': folder_id })
        except Exception:
-            logger.exception("move_file")
+            logging.exception("move_file")
            raise RuntimeError("Database error (File move)!")

    @classmethod
--- a/api/db/services/llm_service.py
+++ b/api/db/services/llm_service.py
@@ -13,13 +13,13 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
 from api.db.services.user_service import TenantService
 from rag.llm import EmbeddingModel, CvModel, ChatModel, RerankModel, Seq2txtModel, TTSModel
 from api.db import LLMType
 from api.db.db_models import DB
 from api.db.db_models import LLMFactories, LLM, TenantLLM
 from api.db.services.common_service import CommonService
-from api.utils.log_utils import logger


 class LLMFactoriesService(CommonService):
@@ -209,7 +209,7 @@ class LLMBundle(object):
        emd, used_tokens = self.mdl.encode(texts, batch_size)
        if not TenantLLMService.increase_usage(
                self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                "LLMBundle.encode can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
        return emd, used_tokens

@@ -217,7 +217,7 @@ class LLMBundle(object):
        emd, used_tokens = self.mdl.encode_queries(query)
        if not TenantLLMService.increase_usage(
                self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                "LLMBundle.encode_queries can't update token usage for {}/EMBEDDING used_tokens: {}".format(self.tenant_id, used_tokens))
        return emd, used_tokens

@@ -225,7 +225,7 @@ class LLMBundle(object):
        sim, used_tokens = self.mdl.similarity(query, texts)
        if not TenantLLMService.increase_usage(
                self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                "LLMBundle.similarity can't update token usage for {}/RERANK used_tokens: {}".format(self.tenant_id, used_tokens))
        return sim, used_tokens

@@ -233,7 +233,7 @@ class LLMBundle(object):
        txt, used_tokens = self.mdl.describe(image, max_tokens)
        if not TenantLLMService.increase_usage(
                self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                "LLMBundle.describe can't update token usage for {}/IMAGE2TEXT used_tokens: {}".format(self.tenant_id, used_tokens))
        return txt

@@ -241,7 +241,7 @@ class LLMBundle(object):
        txt, used_tokens = self.mdl.transcription(audio)
        if not TenantLLMService.increase_usage(
                self.tenant_id, self.llm_type, used_tokens):
-            logger.error(
+            logging.error(
                "LLMBundle.transcription can't update token usage for {}/SEQUENCE2TXT used_tokens: {}".format(self.tenant_id, used_tokens))
        return txt

@@ -250,7 +250,7 @@ class LLMBundle(object):
            if isinstance(chunk,int):
                if not TenantLLMService.increase_usage(
                    self.tenant_id, self.llm_type, chunk, self.llm_name):
-                        logger.error(
+                        logging.error(
                            "LLMBundle.tts can't update token usage for {}/TTS".format(self.tenant_id))
                return
            yield chunk     
@@ -259,7 +259,7 @@ class LLMBundle(object):
        txt, used_tokens = self.mdl.chat(system, history, gen_conf)
        if isinstance(txt, int) and not TenantLLMService.increase_usage(
                self.tenant_id, self.llm_type, used_tokens, self.llm_name):
-            logger.error(
+            logging.error(
                "LLMBundle.chat can't update token usage for {}/CHAT llm_name: {}, used_tokens: {}".format(self.tenant_id, self.llm_name, used_tokens))
        return txt

@@ -268,7 +268,7 @@ class LLMBundle(object):
            if isinstance(txt, int):
                if not TenantLLMService.increase_usage(
                        self.tenant_id, self.llm_type, txt, self.llm_name):
-                    logger.error(
+                    logging.error(
                        "LLMBundle.chat_streamly can't update token usage for {}/CHAT llm_name: {}, content: {}".format(self.tenant_id, self.llm_name, txt))
                return
            yield txt