Some document API refined. (#53)

Add naive chunking method to RAG

rag/app/book.py

@@ -1,10 +1,9 @@
 import copy
 import random
 import re
-from io import BytesIO
-from docx import Document
 import numpy as np
-from rag.app import bullets_category, BULLET_PATTERN, is_english, tokenize, remove_contents_table
+from rag.parser import bullets_category, BULLET_PATTERN, is_english, tokenize, remove_contents_table, \
+    hierarchical_merge, make_colon_as_title, naive_merge
 from rag.nlp import huqie
 from rag.parser.docx_parser import HuDocxParser
 from rag.parser.pdf_parser import HuParser
@@ -28,7 +27,6 @@ class Pdf(HuParser):
         self._table_transformer_job(zoomin)
         callback(0.68, "Table analysis finished")
         self._text_merge()
-        column_width = np.median([b["x1"] - b["x0"] for b in self.boxes])
         self._concat_downward(concat_between_pages=False)
         self._filter_forpages()
         self._merge_with_same_bullet()
@@ -37,10 +35,10 @@ class Pdf(HuParser):
 
         callback(0.8, "Text extraction finished")
 
-        return [(b["text"] + self._line_tag(b, zoomin), b.get("layoutno","")) for b in self.boxes]
+        return [(b["text"] + self._line_tag(b, zoomin), b.get("layoutno","")) for b in self.boxes], tbls
 
 
-def chunk(filename, binary=None, from_page=0, to_page=100000, callback=None):
+def chunk(filename, binary=None, from_page=0, to_page=100000, callback=None, **kwargs):
     doc = {
         "docnm_kwd": filename,
         "title_tks": huqie.qie(re.sub(r"\.[a-zA-Z]+$", "", filename))
@@ -52,8 +50,8 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, callback=None):
         callback(0.1, "Start to parse.")
         doc_parser = HuDocxParser()
         # TODO: table of contents need to be removed
-        sections, tbls = doc_parser(binary if binary else filename)
-        remove_contents_table(sections, eng = is_english(random.choices([t for t,_ in sections], k=200)))
+        sections, tbls = doc_parser(binary if binary else filename, from_page=from_page, to_page=to_page)
+        remove_contents_table(sections, eng=is_english(random.choices([t for t,_ in sections], k=200)))
         callback(0.8, "Finish parsing.")
     elif re.search(r"\.pdf$", filename, re.IGNORECASE):
         pdf_parser = Pdf()
@@ -75,54 +73,12 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, callback=None):
         callback(0.8, "Finish parsing.")
     else: raise NotImplementedError("file type not supported yet(docx, pdf, txt supported)")
 
-    bull = bullets_category([b["text"] for b in random.choices([t for t,_ in sections], k=100)])
-    projs = [len(BULLET_PATTERN[bull]) + 1] * len(sections)
-    levels = [[]] * len(BULLET_PATTERN[bull]) + 2
-    for i, (txt, layout) in enumerate(sections):
-        for j, p in enumerate(BULLET_PATTERN[bull]):
-            if re.match(p, txt.strip()):
-                projs[i] = j
-                levels[j].append(i)
-                break
-        else:
-            if re.search(r"(title|head)", layout):
-                projs[i] = BULLET_PATTERN[bull]
-                levels[BULLET_PATTERN[bull]].append(i)
-            else:
-                levels[BULLET_PATTERN[bull] + 1].append(i)
-    sections = [t for t,_ in sections]
-
-    def binary_search(arr, target):
-        if target > arr[-1]: return len(arr) - 1
-        if target > arr[0]: return -1
-        s, e = 0, len(arr)
-        while e - s > 1:
-            i = (e + s) // 2
-            if target > arr[i]:
-                s = i
-                continue
-            elif target < arr[i]:
-                e = i
-                continue
-            else:
-                assert False
-        return s
-
-    cks = []
-    readed = [False] * len(sections)
-    levels = levels[::-1]
-    for i, arr in enumerate(levels):
-        for j in arr:
-            if readed[j]: continue
-            readed[j] = True
-            cks.append([j])
-            if i + 1 == len(levels) - 1: continue
-            for ii in range(i + 1, len(levels)):
-                jj = binary_search(levels[ii], j)
-                if jj < 0: break
-                if jj > cks[-1][-1]: cks[-1].pop(-1)
-                cks[-1].append(levels[ii][jj])
+    make_colon_as_title(sections)
+    bull = bullets_category([t for t in random.choices([t for t,_ in sections], k=100)])
+    if bull >= 0: cks = hierarchical_merge(bull, sections, 3)
+    else: cks = naive_merge(sections, kwargs.get("chunk_token_num", 256), kwargs.get("delimer", "\n。；！？"))
 
+    sections = [t for t, _ in sections]
     # is it English
     eng = is_english(random.choices(sections, k=218))
 
@@ -138,11 +94,11 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, callback=None):
             tokenize(d, r, eng)
             d["image"] = img
             res.append(d)
+            print("TABLE", d["content_with_weight"])
     # wrap up to es documents
     for ck in cks:
-        print("\n-".join(ck[::-1]))
-        ck = "\n".join(ck[::-1])
         d = copy.deepcopy(doc)
+        ck = "\n".join(ck)
         if pdf_parser:
             d["image"] = pdf_parser.crop(ck)
             ck = pdf_parser.remove_tag(ck)
@@ -153,4 +109,6 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, callback=None):
 
 if __name__ == "__main__":
     import sys
-    chunk(sys.argv[1])
+    def dummy(a, b):
+        pass
+    chunk(sys.argv[1], from_page=1, to_page=10, callback=dummy)