dale_yxc/ragflow_python
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add llm API (#19)

Browse Source 
* add llm API

* refine llm API
This commit is contained in:
KevinHuSh
2023-12-28 13:50:13 +08:00
committed by GitHub
parent cdd956568d
commit d0db329fef
17 changed files with 349 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
python/util/__init__.py
View File

@@ -1,19 +1,24 @@
import re
def rmSpace(txt):
txt = re.sub(r"([^a-z0-9.,]) +([^ ])", r"\1\2", txt)
return re.sub(r"([^ ]) +([^a-z0-9.,])", r"\1\2", txt)
def findMaxDt(fnm):
m = "1970-01-01 00:00:00"
try:
with open(fnm, "r") as f:
while True:
l = f.readline()
if not l:break
if not l:
break
l = l.strip("\n")
if l == 'nan':continue
if l > m:m = l
if l == 'nan':
continue
if l > m:
m = l
except Exception as e:
print("WARNING: can't find "+ fnm)
print("WARNING: can't find " + fnm)
return m

26
python/util/config.py
View File

@@ -1,25 +1,31 @@
from configparser import ConfigParser
import os,inspect
from configparser import ConfigParser
import os
import inspect
CF = ConfigParser()
__fnm = os.path.join(os.path.dirname(__file__), '../conf/sys.cnf')
if not os.path.exists(__fnm):__fnm = os.path.join(os.path.dirname(__file__), '../../conf/sys.cnf')
assert os.path.exists(__fnm), f"【EXCEPTION】can't find {__fnm}." + os.path.dirname(__file__)
if not os.path.exists(__fnm): __fnm = "./sys.cnf"
if not os.path.exists(__fnm):
__fnm = os.path.join(os.path.dirname(__file__), '../../conf/sys.cnf')
assert os.path.exists(
__fnm), f"【EXCEPTION】can't find {__fnm}." + os.path.dirname(__file__)
if not os.path.exists(__fnm):
__fnm = "./sys.cnf"
CF.read(__fnm)
class Config:
def __init__(self, env):
self.env = env
if env == "spark":CF.read("./cv.cnf")
if env == "spark":
CF.read("./cv.cnf")
def get(self, key, default=None):
global CF
return os.environ.get(key.upper(), \
CF[self.env].get(key, default)
)
return os.environ.get(key.upper(),
CF[self.env].get(key, default)
)
def init(env):
return Config(env)

27
python/util/db_conn.py
View File

@@ -3,6 +3,7 @@ import time
from util import config
import pandas as pd
class Postgres(object):
def __init__(self, env, dbnm):
self.config = config.init(env)
@@ -13,36 +14,42 @@ class Postgres(object):
def __open__(self):
import psycopg2
try:
if self.conn:self.__close__()
if self.conn:
self.__close__()
del self.conn
except Exception as e:
pass
try:
self.conn = psycopg2.connect(f"dbname={self.dbnm} user={self.config.get('pgdb_usr')} password={self.config.get('pgdb_pwd')} host={self.config.get('pgdb_host')} port={self.config.get('pgdb_port')}")
self.conn = psycopg2.connect(f"""dbname={self.dbnm}
user={self.config.get('postgres_user')}
password={self.config.get('postgres_password')}
host={self.config.get('postgres_host')}
port={self.config.get('postgres_port')}""")
except Exception as e:
logging.error("Fail to connect %s "%self.config.get("pgdb_host") + str(e))
logging.error(
"Fail to connect %s " %
self.config.get("pgdb_host") + str(e))
def __close__(self):
try:
self.conn.close()
except Exception as e:
logging.error("Fail to close %s "%self.config.get("pgdb_host") + str(e))
logging.error(
"Fail to close %s " %
self.config.get("pgdb_host") + str(e))
def select(self, sql):
for _ in range(10):
try:
return pd.read_sql(sql, self.conn)
except Exception as e:
logging.error(f"Fail to exec {sql} "+str(e))
logging.error(f"Fail to exec {sql} " + str(e))
self.__open__()
time.sleep(1)
return pd.DataFrame()
def update(self, sql):
for _ in range(10):
try:
@@ -53,11 +60,11 @@ class Postgres(object):
cur.close()
return updated_rows
except Exception as e:
logging.error(f"Fail to exec {sql} "+str(e))
logging.error(f"Fail to exec {sql} " + str(e))
self.__open__()
time.sleep(1)
return 0
if __name__ == "__main__":
Postgres("infiniflow", "docgpt")

11
python/util/es_conn.py
View File

@@ -228,7 +228,8 @@ class HuEs:
return False
def search(self, q, idxnm=None, src=False, timeout="2s"):
if not isinstance(q, dict): q = Search().query(q).to_dict()
if not isinstance(q, dict):
q = Search().query(q).to_dict()
for i in range(3):
try:
res = self.es.search(index=(self.idxnm if not idxnm else idxnm),
@@ -274,9 +275,10 @@ class HuEs:
return False
def updateScriptByQuery(self, q, scripts, idxnm=None):
ubq = UpdateByQuery(index=self.idxnm if not idxnm else idxnm).using(self.es).query(q)
ubq = UpdateByQuery(
index=self.idxnm if not idxnm else idxnm).using(
self.es).query(q)
ubq = ubq.script(source=scripts)
ubq = ubq.params(refresh=True)
ubq = ubq.params(slices=5)
@@ -294,7 +296,6 @@ class HuEs:
return False
def deleteByQuery(self, query, idxnm=""):
for i in range(3):
try:
@@ -392,7 +393,7 @@ class HuEs:
return rr
def scrollIter(self, pagesize=100, scroll_time='2m', q={
"query": {"match_all": {}}, "sort": [{"updated_at": {"order": "desc"}}]}):
"query": {"match_all": {}}, "sort": [{"updated_at": {"order": "desc"}}]}):
for _ in range(100):
try:
page = self.es.search(

36
python/util/minio_conn.py
View File

@@ -4,6 +4,7 @@ from util import config
from minio import Minio
from io import BytesIO
class HuMinio(object):
def __init__(self, env):
self.config = config.init(env)
@@ -12,64 +13,62 @@ class HuMinio(object):
def __open__(self):
try:
if self.conn:self.__close__()
if self.conn:
self.__close__()
except Exception as e:
pass
try:
self.conn = Minio(self.config.get("minio_host"),
access_key=self.config.get("minio_usr"),
secret_key=self.config.get("minio_pwd"),
access_key=self.config.get("minio_user"),
secret_key=self.config.get("minio_password"),
secure=False
)
)
except Exception as e:
logging.error("Fail to connect %s "%self.config.get("minio_host") + str(e))
logging.error(
"Fail to connect %s " %
self.config.get("minio_host") + str(e))
def __close__(self):
del self.conn
self.conn = None
def put(self, bucket, fnm, binary):
for _ in range(10):
try:
if not self.conn.bucket_exists(bucket):
self.conn.make_bucket(bucket)
r = self.conn.put_object(bucket, fnm,
r = self.conn.put_object(bucket, fnm,
BytesIO(binary),
len(binary)
)
)
return r
except Exception as e:
logging.error(f"Fail put {bucket}/{fnm}: "+str(e))
logging.error(f"Fail put {bucket}/{fnm}: " + str(e))
self.__open__()
time.sleep(1)
def get(self, bucket, fnm):
for _ in range(10):
try:
r = self.conn.get_object(bucket, fnm)
return r.read()
except Exception as e:
logging.error(f"fail get {bucket}/{fnm}: "+str(e))
logging.error(f"fail get {bucket}/{fnm}: " + str(e))
self.__open__()
time.sleep(1)
return
return
def get_presigned_url(self, bucket, fnm, expires):
for _ in range(10):
try:
return self.conn.get_presigned_url("GET", bucket, fnm, expires)
except Exception as e:
logging.error(f"fail get {bucket}/{fnm}: "+str(e))
logging.error(f"fail get {bucket}/{fnm}: " + str(e))
self.__open__()
time.sleep(1)
return
return
if __name__ == "__main__":
@@ -78,9 +77,8 @@ if __name__ == "__main__":
from PIL import Image
img = Image.open(fnm)
buff = BytesIO()
img.save(buff, format='JPEG')
img.save(buff, format='JPEG')
print(conn.put("test", "11-408.jpg", buff.getvalue()))
bts = conn.get("test", "11-408.jpg")
img = Image.open(BytesIO(bts))
img.save("test.jpg")