Add task moduel, and pipline the task and every parser (#49)

2024-01-31 19:57:45 +08:00
parent af3ef26977
commit 6224edcd1b
15 changed files with 369 additions and 237 deletions
--- a/rag/app/init.py
+++ b/rag/app/init.py
@@ -67,4 +67,6 @@ def tokenize(d, t, eng):
        d["content_ltks"] = " ".join([stemmer.stem(w) for w in word_tokenize(t)])
    else:
        d["content_ltks"] = huqie.qie(t)
-        d["content_sm_ltks"] = huqie.qieqie(d["content_ltks"])
+        d["content_sm_ltks"] = huqie.qieqie(d["content_ltks"])
+
+