add llm API (#19)

* add llm API

* refine llm API

python/llm/embedding_model.py

@@ -1,8 +1,11 @@
 from abc import ABC
+from openai import OpenAI
 from FlagEmbedding import FlagModel
 import torch
+import os
 import numpy as np
 
+
 class Base(ABC):
     def encode(self, texts: list, batch_size=32):
         raise NotImplementedError("Please implement encode method!")
@@ -22,11 +25,37 @@ class HuEmbedding(Base):
 
         """
         self.model = FlagModel("BAAI/bge-large-zh-v1.5",
-                              query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章：",
-                              use_fp16=torch.cuda.is_available())
+                               query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章：",
+                               use_fp16=torch.cuda.is_available())
 
     def encode(self, texts: list, batch_size=32):
         res = []
         for i in range(0, len(texts), batch_size):
-            res.extend(self.model.encode(texts[i:i+batch_size]).tolist())
+            res.extend(self.model.encode(texts[i:i + batch_size]).tolist())
         return np.array(res)
+
+
+class GptEmbed(Base):
+    def __init__(self):
+        self.client = OpenAI(api_key=os.envirement["OPENAI_API_KEY"])
+
+    def encode(self, texts: list, batch_size=32):
+        res = self.client.embeddings.create(input=texts,
+                                            model="text-embedding-ada-002")
+        return [d["embedding"] for d in res["data"]]
+
+
+class QWenEmbd(Base):
+    def encode(self, texts: list, batch_size=32, text_type="document"):
+        # export DASHSCOPE_API_KEY=YOUR_DASHSCOPE_API_KEY
+        import dashscope
+        from http import HTTPStatus
+        res = []
+        for txt in texts:
+            resp = dashscope.TextEmbedding.call(
+                model=dashscope.TextEmbedding.Models.text_embedding_v2,
+                input=txt[:2048],
+                text_type=text_type
+            )
+            res.append(resp["output"]["embeddings"][0]["embedding"])
+        return res