RAG增强之路：增强PDF解析并结构化技术路线方案及思路 原创

from modelscope.outputs import OutputKeys
from modelscope.pipelines import pipeline
from modelscope.utils.constant import Tasks

p = pipeline(
    task=Tasks.document_segmentation,
    model='damo/nlp_bert_document-segmentation_chinese-base')

result = p(documents='移动端语音唤醒模型，检测关键词为“小云小云”。模型主体为4层FSMN结构，使用CTC训练准则，参数量750K，适用于移动端设备运行。模型输入为Fbank特征，输出为基于char建模的中文全集token预测，测试工具根据每一帧的预测数据进行后处理得到输入音频的实时检测结果。模型训练采用“basetrain + finetune”的模式，basetrain过程使用大量内部移动端数据，在此基础上，使用1万条设备端录制安静场景“小云小云”数据进行微调，得到最终面向业务的模型。后续用户可在basetrain模型基础上，使用其他关键词数据进行微调，得到新的语音唤醒模型，但暂时未开放模型finetune功能。')

print(result[OutputKeys.TEXT])
# 输出
'''
   移动端语音唤醒模型，检测关键词为“小云小云”。模型主体为4层FSMN结构，使用CTC训练准则，参数量750K，适用于移动端设备运行。模型输入为Fbank特征，输出为基于char建模的中文全集token预测，测试工具根据每一帧的预测数据进行后处理得到输入音频的实时检测结果。
   模型训练采用“basetrain + finetune”的模式，basetrain过程使用大量内部移动端数据，在此基础上，使用1万条设备端录制安静场景“小云小云”数据进行微调，得到最终面向业务的模型。后续用户可在basetrain模型基础上，使用其他关键词数据进行微调，得到新的语音唤醒模型，但暂时未开放模型finetune功能。
'''
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

import cv2
from paddleocr import PaddleOCR
paddleocr = PaddleOCR(lang='ch', show_log=False, enable_mkldnn=True) 
img = cv2.imread('1.jpg')  
result = paddleocr.ocr(img)
for i in range(len(result[0])):
    print(result[0][i][1][0])   # 输出识别结果
1.
2.
3.
4.
5.
6.
7.

import os
import cv2
from paddleocr import PPStructure,save_structure_res
table_engine = PPStructure(layout=False, show_log=True) 
save_folder = './output'
img_path = 'table.jpg'
img = cv2.imread(img_path)
result = table_engine(img)
save_structure_res(result, save_folder, os.path.basename(img_path).split('.')[0])
for line in result:
    line.pop('img')
    print(line)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.

from PIL import Image
from pix2tex.cli import LatexOCR

model = LatexOCR()
img = Image.open('1.jpg')
print(model(img))
1.
2.
3.
4.
5.
6.

import numpy as np

def xy_cut(bboxes, directinotallow="x"):
    result = []
    K = len(bboxes)
    indexes = range(K)
    if len(bboxes) <= 0:
        return result
    if direction == "x":
        # x first
        sorted_ids = sorted(indexes, key=lambda k: (bboxes[k][0], bboxes[k][1]))
        sorted_boxes = sorted(bboxes, key=lambda x: (x[0], x[1]))
        next_dir = "y"
    else:
        sorted_ids = sorted(indexes, key=lambda k: (bboxes[k][1], bboxes[k][0]))
        sorted_boxes = sorted(bboxes, key=lambda x: (x[1], x[0]))
        next_dir = "x"

    curr = 0
    np_bboxes = np.array(sorted_boxes)
    for idx in range(len(sorted_boxes)):
        if direction == "x":
            # a new seg path
            if idx != K - 1 and sorted_boxes[idx][2] < sorted_boxes[idx + 1][0]:
                rel_res = xy_cut(sorted_boxes[curr:idx + 1], next_dir)
                result += [sorted_ids[i + curr] for i in rel_res]
                curr = idx + 1
        else:
            # a new seg path
            if idx != K - 1 and sorted_boxes[idx][3] < sorted_boxes[idx + 1][1]:
                rel_res = xy_cut(sorted_boxes[curr:idx + 1], next_dir)
                result += [sorted_ids[i + curr] for i in rel_res]
                curr = idx + 1

    result += sorted_ids[curr:idx + 1]
    return result


def augment_xy_cut(bboxes,
                   directinotallow="x",
                   lambda_x=0.5,
                   lambda_y=0.5,
                   theta=5,
                   aug=False):
    if aug is True:
        for idx in range(len(bboxes)):
            vx = np.random.normal(loc=0, scale=1)
            vy = np.random.normal(loc=0, scale=1)
            if np.abs(vx) >= lambda_x:
                bboxes[idx][0] += round(theta * vx)
                bboxes[idx][2] += round(theta * vx)
            if np.abs(vy) >= lambda_y:
                bboxes[idx][1] += round(theta * vy)
                bboxes[idx][3] += round(theta * vy)
            bboxes[idx] = [max(0, i) for i in bboxes[idx]]
    res_idx = xy_cut(bboxes, directinotallow=direction)
    res_bboxes = [bboxes[idx] for idx in res_idx]
    return res_idx, res_bboxes


bboxes = [[58.54924774169922, 1379.6373291015625, 1112.8863525390625, 1640.0870361328125],
          [60.1091423034668, 483.88677978515625, 1117.4927978515625, 586.197021484375],
          [57.687435150146484, 1098.1053466796875, 387.9796142578125, 1216.916015625],
          [63.158992767333984, 311.2080993652344, 1116.2508544921875, 365.2145080566406],
          [138.85513305664062, 144.44039916992188, 845.18017578125, 198.04937744140625],
          [996.1032104492188, 1053.6279296875, 1126.1046142578125, 1071.3463134765625],
          [58.743492126464844, 634.3077392578125, 898.405029296875, 700.9544677734375],
          [61.35755920410156, 750.6771240234375, 1051.1060791015625, 850.3980712890625],
          [426.77691650390625, 70.69780731201172, 556.0884399414062, 109.58145141601562],
          [997.040283203125, 903.5933227539062, 1129.2984619140625, 921.10595703125],
          [59.40523910522461, 1335.1563720703125, 329.7382507324219, 1357.46533203125],
          [568.9025268554688, 14.365530967712402, 1087.898193359375, 32.60292434692383],
          [998.1250610351562, 752.936279296875, 1128.435546875, 770.4116821289062],
          [59.6968879699707, 947.9129638671875, 601.4513549804688, 999.4548950195312],
          [58.91489028930664, 1049.8773193359375, 487.3372497558594, 1072.2935791015625],
          [60.49456024169922, 902.8802490234375, 600.7571411132812, 1000.3502197265625],
          [60.188941955566406, 247.99755859375, 155.72970581054688, 272.1385192871094],
          [996.873291015625, 637.3861694335938, 1128.3558349609375, 655.1572875976562],
          [59.74936294555664, 1272.98828125, 154.8768310546875, 1295.870361328125],
          [58.835716247558594, 1050.5926513671875, 481.59027099609375, 1071.966796875],
          [60.60163116455078, 750.1132202148438, 376.1781921386719, 771.8764038085938],
          [57.982513427734375, 419.16058349609375, 155.35882568359375, 444.25115966796875],
          [1017.0194091796875, 1336.21826171875, 1128.002197265625, 1355.67724609375],
          [1019.8740844726562, 486.90814208984375, 1127.482421875, 504.61767578125]]

res_idx, res_bboxes = augment_xy_cut(bboxes, directinotallow="y")
print(res_idx)
# res_idx, res_bboxes = augment_xy_cut(bboxes, directinotallow="x")
# print(res_idx)

new_boxs = []
for i in res_idx:
    # print(i)

    new_boxs.append(bboxes[i])

print(new_boxs)
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.

import torch
from model import LayoutLMv3ForBboxClassification
from collections import defaultdict

CLS_TOKEN_ID = 0
UNK_TOKEN_ID = 3
EOS_TOKEN_ID = 2


def BboxesMasks(boxes):
    bbox = [[0, 0, 0, 0]] + boxes + [[0, 0, 0, 0]]
    input_ids = [CLS_TOKEN_ID] + [UNK_TOKEN_ID] * len(boxes) + [EOS_TOKEN_ID]
    attention_mask = [1] + [1] * len(boxes) + [1]
    return {
        "bbox": torch.tensor([bbox]),
        "attention_mask": torch.tensor([attention_mask]),
        "input_ids": torch.tensor([input_ids]),
    }


def decode(logits, length):
    logits = logits[1: length + 1, :length]
    orders = logits.argsort(descending=False).tolist()
    ret = [o.pop() for o in orders]
    while True:
        order_to_idxes = defaultdict(list)
        for idx, order in enumerate(ret):
            order_to_idxes[order].append(idx)
        order_to_idxes = {k: v for k, v in order_to_idxes.items() if len(v) > 1}
        if not order_to_idxes:
            break
        for order, idxes in order_to_idxes.items():
            idxes_to_logit = {}
            for idx in idxes:
                idxes_to_logit[idx] = logits[idx, order]
            idxes_to_logit = sorted(
                idxes_to_logit.items(), key=lambda x: x[1], reverse=True
            )
            for idx, _ in idxes_to_logit[1:]:
                ret[idx] = orders[idx].pop()
    return ret


def layoutreader(bboxes):
    inputs = BboxesMasks(bboxes)
    logits = model(**inputs).logits.cpu().squeeze(0)
    orders = decode(logits, len(bboxes))
    return orders


if __name__ == '__main__':
    bboxes = [[584, 0, 595, 1], [35, 120, 89, 133],
              [35, 140, 75, 152]]
    model_path = ""
    model = LayoutLMv3ForBboxClassification.from_pretrained()

    print(layoutreader(bboxes))
# [1, 2, 0]
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.