pdf_parse_by_model.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. import time
  2. # from anyio import Path
  3. from magic_pdf.libs.commons import fitz, get_delta_time, get_img_s3_client, get_docx_model_output
  4. import json
  5. import os
  6. import math
  7. from loguru import logger
  8. from magic_pdf.layout.bbox_sort import (
  9. prepare_bboxes_for_layout_split,
  10. )
  11. from magic_pdf.layout.layout_sort import LAYOUT_UNPROC, get_bboxes_layout, get_columns_cnt_of_layout, sort_text_block
  12. from magic_pdf.libs.drop_reason import DropReason
  13. from magic_pdf.libs.markdown_utils import escape_special_markdown_char
  14. from magic_pdf.libs.safe_filename import sanitize_filename
  15. from magic_pdf.libs.vis_utils import draw_bbox_on_page, draw_layout_bbox_on_page
  16. from magic_pdf.pre_proc.detect_images import parse_images
  17. from magic_pdf.pre_proc.detect_tables import parse_tables # 获取tables的bbox
  18. from magic_pdf.pre_proc.detect_equation import parse_equations # 获取equations的bbox
  19. from magic_pdf.pre_proc.detect_header import parse_headers # 获取headers的bbox
  20. from magic_pdf.pre_proc.detect_page_number import parse_pageNos # 获取pageNos的bbox
  21. from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model, parse_footnotes_by_rule # 获取footnotes的bbox
  22. from magic_pdf.pre_proc.detect_footer_by_model import parse_footers # 获取footers的bbox
  23. from magic_pdf.post_proc.detect_para import (
  24. ParaProcessPipeline,
  25. TitleDetectionException,
  26. TitleLevelException,
  27. ParaSplitException,
  28. ParaMergeException,
  29. DenseSingleLineBlockException,
  30. )
  31. from magic_pdf.pre_proc.main_text_font import get_main_text_font
  32. from magic_pdf.pre_proc.remove_colored_strip_bbox import remove_colored_strip_textblock
  33. from magic_pdf.pre_proc.remove_footer_header import remove_headder_footer_one_page
  34. '''
  35. from para.para_pipeline import ParaProcessPipeline
  36. from para.exceptions import (
  37. TitleDetectionException,
  38. TitleLevelException,
  39. ParaSplitException,
  40. ParaMergeException,
  41. DenseSingleLineBlockException,
  42. )
  43. '''
  44. from magic_pdf.libs.commons import read_file, join_path
  45. from magic_pdf.libs.pdf_image_tools import save_images_by_bboxes
  46. from magic_pdf.post_proc.remove_footnote import merge_footnote_blocks, remove_footnote_blocks
  47. from magic_pdf.pre_proc.citationmarker_remove import remove_citation_marker
  48. from magic_pdf.pre_proc.equations_replace import combine_chars_to_pymudict, remove_chars_in_text_blocks, replace_equations_in_textblock
  49. from magic_pdf.pre_proc.pdf_pre_filter import pdf_filter
  50. from magic_pdf.pre_proc.detect_footer_header_by_statistics import drop_footer_header
  51. from magic_pdf.pre_proc.construct_paras import construct_page_component
  52. from magic_pdf.pre_proc.fix_image import combine_images, fix_image_vertical, fix_seperated_image, include_img_title
  53. from magic_pdf.post_proc.pdf_post_filter import pdf_post_filter
  54. from magic_pdf.pre_proc.remove_rotate_bbox import get_side_boundry, remove_rotate_side_textblock, remove_side_blank_block
  55. from magic_pdf.pre_proc.resolve_bbox_conflict import check_text_block_horizontal_overlap, resolve_bbox_overlap_conflict
  56. from magic_pdf.pre_proc.fix_table import fix_table_text_block, fix_tables, include_table_title
  57. from magic_pdf.pre_proc.solve_line_alien import solve_inline_too_large_interval
  58. denseSingleLineBlockException_msg = DenseSingleLineBlockException().message
  59. titleDetectionException_msg = TitleDetectionException().message
  60. titleLevelException_msg = TitleLevelException().message
  61. paraSplitException_msg = ParaSplitException().message
  62. paraMergeException_msg = ParaMergeException().message
  63. def parse_pdf_by_model(
  64. s3_pdf_path,
  65. s3_pdf_profile,
  66. pdf_model_output,
  67. save_path,
  68. book_name,
  69. pdf_model_profile=None,
  70. image_s3_config=None,
  71. start_page_id=0,
  72. end_page_id=None,
  73. junk_img_bojids=[],
  74. debug_mode=False,
  75. ):
  76. pdf_bytes = read_file(s3_pdf_path, s3_pdf_profile)
  77. save_tmp_path = os.path.join(os.path.dirname(__file__), "../..", "tmp", "unittest")
  78. md_bookname_save_path = ""
  79. book_name = sanitize_filename(book_name)
  80. if debug_mode:
  81. save_path = join_path(save_tmp_path, "md")
  82. pdf_local_path = join_path(save_tmp_path, "download-pdfs", book_name)
  83. if not os.path.exists(os.path.dirname(pdf_local_path)):
  84. # 如果目录不存在,创建它
  85. os.makedirs(os.path.dirname(pdf_local_path))
  86. md_bookname_save_path = join_path(save_tmp_path, "md", book_name)
  87. if not os.path.exists(md_bookname_save_path):
  88. # 如果目录不存在,创建它
  89. os.makedirs(md_bookname_save_path)
  90. with open(pdf_local_path + ".pdf", "wb") as pdf_file:
  91. pdf_file.write(pdf_bytes)
  92. pdf_docs = fitz.open("pdf", pdf_bytes)
  93. pdf_info_dict = {}
  94. img_s3_client = get_img_s3_client(save_path, image_s3_config) # 更改函数名和参数,避免歧义
  95. # img_s3_client = "img_s3_client" #不创建这个对象,直接用字符串占位
  96. start_time = time.time()
  97. """通过统计pdf全篇文字,识别正文字体"""
  98. main_text_font = get_main_text_font(pdf_docs)
  99. end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
  100. for page_id in range(start_page_id, end_page_id + 1):
  101. page = pdf_docs[page_id]
  102. page_width = page.rect.width
  103. page_height = page.rect.height
  104. if debug_mode:
  105. time_now = time.time()
  106. logger.info(f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}")
  107. start_time = time_now
  108. """
  109. # 通过一个规则,过滤掉单页超过1500非junkimg的pdf
  110. # 对单页面非重复id的img数量做统计,如果当前页超过1500则直接return need_drop
  111. """
  112. page_imgs = page.get_images()
  113. img_counts = 0
  114. for img in page_imgs:
  115. img_bojid = img[0]
  116. if img_bojid in junk_img_bojids: # 判断这个图片在不在junklist中
  117. continue # 如果在junklist就不用管了,跳过
  118. else:
  119. recs = page.get_image_rects(img, transform=True)
  120. if recs: # 如果这张图在当前页面有展示
  121. img_counts += 1
  122. if img_counts >= 1500: # 如果去除了junkimg的影响,单页img仍然超过1500的话,就排除当前pdf
  123. logger.warning(
  124. f"page_id: {page_id}, img_counts: {img_counts}, drop this pdf: {book_name}, drop_reason: {DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}"
  125. )
  126. result = {"need_drop": True, "drop_reason": DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}
  127. if not debug_mode:
  128. return result
  129. """
  130. ==================================================================================================================================
  131. 首先获取基本的block数据,对pdf进行分解,获取图片、表格、公式、text的bbox
  132. """
  133. # 解析pdf原始文本block
  134. text_raw_blocks = page.get_text(
  135. "dict",
  136. flags=fitz.TEXTFLAGS_TEXT,
  137. )["blocks"]
  138. model_output_json = get_docx_model_output(pdf_model_output, pdf_model_profile, page_id)
  139. # 解析图片
  140. image_bboxes = parse_images(page_id, page, model_output_json, junk_img_bojids)
  141. image_bboxes = fix_image_vertical(image_bboxes, text_raw_blocks) # 修正图片的位置
  142. image_bboxes = fix_seperated_image(image_bboxes) # 合并有边重合的图片
  143. image_bboxes = include_img_title(text_raw_blocks, image_bboxes) # 向图片上方和下方寻找title,使用规则进行匹配,暂时只支持英文规则
  144. """此时image_bboxes中可能出现这种情况,水平并列的2个图片,下方分别有各自的子标题,2个子标题下方又有大标题(形如Figxxx),会出现2个图片的bbox都包含了这个大标题,这种情况需要把图片合并"""
  145. image_bboxes = combine_images(image_bboxes) # 合并图片
  146. # 解析表格并对table_bboxes进行位置的微调,防止表格周围的文字被截断
  147. table_bboxes = parse_tables(page_id, page, model_output_json)
  148. table_bboxes = fix_tables(page, table_bboxes, include_table_title=True, scan_line_num=2) # 修正
  149. table_bboxes = fix_table_text_block(text_raw_blocks, table_bboxes) # 修正与text block的关系,某些table修正与pymupdf获取到的table内textblock没有完全包含,因此要进行一次修正。
  150. #debug_show_bbox(pdf_docs, page_id, table_bboxes, [], [b['bbox'] for b in text_raw_blocks], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
  151. table_bboxes = include_table_title(text_raw_blocks, table_bboxes) # 向table上方和下方寻找title,使用规则进行匹配,暂时只支持英文规则
  152. # 解析公式
  153. equations_inline_bboxes, equations_interline_bboxes = parse_equations(page_id, page, model_output_json)
  154. """
  155. ==================================================================================================================================
  156. 进入预处理-1阶段
  157. -------------------
  158. # # 解析标题
  159. # title_bboxs = parse_titles(page_id, page, model_output_json)
  160. # # 评估Layout是否规整、简单
  161. # isSimpleLayout_flag, fullColumn_cnt, subColumn_cnt, curPage_loss = evaluate_pdf_layout(page_id, page, model_output_json)
  162. 接下来开始进行预处理过程
  163. """
  164. """去掉每页的页码、页眉、页脚"""
  165. page_no_bboxs = parse_pageNos(page_id, page, model_output_json)
  166. header_bboxs = parse_headers(page_id, page, model_output_json)
  167. footer_bboxs = parse_footers(page_id, page, model_output_json)
  168. image_bboxes, table_bboxes, remain_text_blocks, removed_hdr_foot_txt_block, removed_hdr_foot_img_block, removed_hdr_foot_table = remove_headder_footer_one_page(text_raw_blocks, image_bboxes, table_bboxes, header_bboxs, footer_bboxs, page_no_bboxs, page_width, page_height)
  169. """去除页面上半部分长条色块内的文本块"""
  170. remain_text_blocks, removed_colored_narrow_strip_background_text_block = remove_colored_strip_textblock(remain_text_blocks, page)
  171. #debug_show_bbox(pdf_docs, page_id, footnote_bboxes_by_model, [b['bbox'] for b in remain_text_blocks], header_bboxs, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
  172. """去掉旋转的文字:水印、垂直排列的文字"""
  173. remain_text_blocks, removed_non_horz_text_block = remove_rotate_side_textblock(
  174. remain_text_blocks, page_width, page_height
  175. ) # 去掉水印,非水平文字
  176. remain_text_blocks, removed_empty_side_block = remove_side_blank_block(remain_text_blocks, page_width, page_height) # 删除页面四周可能会留下的完全空白的textblock,这种block形成原因未知
  177. """出现在图片、表格上的文字块去掉,把层叠的图片单独分离出来,不参与layout的计算"""
  178. (
  179. image_bboxes,
  180. table_bboxes,
  181. equations_interline_bboxes,
  182. equations_inline_bboxes,
  183. remain_text_blocks,
  184. text_block_on_image_removed,
  185. images_overlap_backup,
  186. interline_eq_temp_text_block
  187. ) = resolve_bbox_overlap_conflict(
  188. image_bboxes, table_bboxes, equations_interline_bboxes, equations_inline_bboxes, remain_text_blocks
  189. )
  190. # """去掉footnote, 从文字和图片中"""
  191. # # 通过模型识别到的footnote
  192. # footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, md_bookname_save_path,
  193. # debug_mode=debug_mode)
  194. # # 通过规则识别到的footnote
  195. # footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id)
  196. """
  197. ==================================================================================================================================
  198. """
  199. if debug_mode: # debugmode截图到本地
  200. save_path = join_path(save_tmp_path, "md")
  201. # 把图、表、公式都进行截图,保存到存储上,返回图片路径作为内容
  202. image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info = save_images_by_bboxes(
  203. book_name,
  204. page_id,
  205. page,
  206. save_path,
  207. image_bboxes,
  208. images_overlap_backup,
  209. table_bboxes,
  210. equations_inline_bboxes,
  211. equations_interline_bboxes,
  212. # 传入img_s3_client
  213. img_s3_client,
  214. ) # 只要表格和图片的截图
  215. """"以下进入到公式替换环节 """
  216. char_level_text_blocks = page.get_text("rawdict", flags=fitz.TEXTFLAGS_TEXT)['blocks']
  217. remain_text_blocks = combine_chars_to_pymudict(remain_text_blocks, char_level_text_blocks)# 合并chars
  218. remain_text_blocks = replace_equations_in_textblock(remain_text_blocks, inline_eq_info, interline_eq_info)
  219. remain_text_blocks = remove_citation_marker(remain_text_blocks) # 公式替换之后去角标,防止公式无法替换成功。但是这样也会带来个问题就是把角标当公式。各有优劣。
  220. remain_text_blocks = remove_chars_in_text_blocks(remain_text_blocks) # 减少中间态数据体积
  221. #debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in inline_eq_info], [b['bbox'] for b in interline_eq_info], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 3)
  222. """去掉footnote, 从文字和图片中(先去角标再去footnote试试)"""
  223. # 通过模型识别到的footnote
  224. footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, md_bookname_save_path, debug_mode=debug_mode)
  225. # 通过规则识别到的footnote
  226. footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id, main_text_font)
  227. """进入pdf过滤器,去掉一些不合理的pdf"""
  228. is_good_pdf, err = pdf_filter(page, remain_text_blocks, table_bboxes, image_bboxes)
  229. if not is_good_pdf:
  230. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {err}")
  231. if not debug_mode:
  232. return err
  233. """
  234. ==================================================================================================================================
  235. 进行版面布局切分和过滤
  236. """
  237. """在切分之前,先检查一下bbox是否有左右重叠的情况,如果有,那么就认为这个pdf暂时没有能力处理好,这种左右重叠的情况大概率是由于pdf里的行间公式、表格没有被正确识别出来造成的 """
  238. is_text_block_horz_overlap = check_text_block_horizontal_overlap(remain_text_blocks, header_bboxs, footer_bboxs)
  239. if is_text_block_horz_overlap:
  240. # debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in remain_text_blocks], [], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 0)
  241. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.TEXT_BLCOK_HOR_OVERLAP}")
  242. result = {"need_drop": True, "drop_reason": DropReason.TEXT_BLCOK_HOR_OVERLAP}
  243. if not debug_mode:
  244. return result
  245. """统一格式化成一个数据结构用于计算layout"""
  246. page_y0 = 0 if len(header_bboxs) == 0 else max([b[3] for b in header_bboxs])
  247. page_y1 = page_height if len(footer_bboxs) == 0 else min([b[1] for b in footer_bboxs])
  248. left_x, right_x = get_side_boundry(removed_non_horz_text_block, page_width, page_height)
  249. page_boundry = [math.floor(left_x), page_y0 + 1, math.ceil(right_x), page_y1 - 1]
  250. # 返回的是一个数组,每个元素[x0, y0, x1, y1, block_content, idx_x, idx_y], 初始时候idx_x, idx_y都是None. 对于图片、公式来说,block_content是图片的地址, 对于段落来说,block_content是段落的内容
  251. all_bboxes = prepare_bboxes_for_layout_split(
  252. image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info, remain_text_blocks, page_boundry, page)
  253. #debug_show_bbox(pdf_docs, page_id, [], [], all_bboxes, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 1)
  254. """page_y0, page_y1能够过滤掉页眉和页脚,不会算作layout内"""
  255. layout_bboxes, layout_tree = get_bboxes_layout(all_bboxes, page_boundry, page_id)
  256. if len(remain_text_blocks)>0 and len(all_bboxes)>0 and len(layout_bboxes)==0:
  257. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}")
  258. result = {"need_drop": True, "drop_reason": DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}
  259. if not debug_mode:
  260. return result
  261. """以下去掉复杂的布局和超过2列的布局"""
  262. if any([lay["layout_label"] == LAYOUT_UNPROC for lay in layout_bboxes]): # 复杂的布局
  263. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.COMPLICATED_LAYOUT}")
  264. result = {"need_drop": True, "drop_reason": DropReason.COMPLICATED_LAYOUT}
  265. if not debug_mode:
  266. return result
  267. layout_column_width = get_columns_cnt_of_layout(layout_tree)
  268. if layout_column_width > 2: # 去掉超过2列的布局pdf
  269. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.TOO_MANY_LAYOUT_COLUMNS}")
  270. result = {
  271. "need_drop": True,
  272. "drop_reason": DropReason.TOO_MANY_LAYOUT_COLUMNS,
  273. "extra_info": {"column_cnt": layout_column_width},
  274. }
  275. if not debug_mode:
  276. return result
  277. """
  278. ==================================================================================================================================
  279. 构造出下游需要的数据结构
  280. """
  281. remain_text_blocks = remain_text_blocks + interline_eq_temp_text_block # 把计算layout时候临时删除的行间公式再放回去,防止行间公式替换的时候丢失。
  282. removed_text_blocks = []
  283. removed_text_blocks.extend(removed_hdr_foot_txt_block)
  284. # removed_text_blocks.extend(removed_footnote_text_block)
  285. removed_text_blocks.extend(text_block_on_image_removed)
  286. removed_text_blocks.extend(removed_non_horz_text_block)
  287. removed_text_blocks.extend(removed_colored_narrow_strip_background_text_block)
  288. removed_images = []
  289. # removed_images.extend(footnote_imgs)
  290. removed_images.extend(removed_hdr_foot_img_block)
  291. images_backup = []
  292. images_backup.extend(image_backup_info)
  293. remain_text_blocks = escape_special_markdown_char(remain_text_blocks) # 转义span里的text
  294. sorted_text_remain_text_block = sort_text_block(remain_text_blocks, layout_bboxes)
  295. footnote_bboxes_tmp = []
  296. footnote_bboxes_tmp.extend(footnote_bboxes_by_model)
  297. footnote_bboxes_tmp.extend(footnote_bboxes_by_rule)
  298. page_info = construct_page_component(
  299. page_id,
  300. image_info,
  301. table_info,
  302. sorted_text_remain_text_block,
  303. layout_bboxes,
  304. inline_eq_info,
  305. interline_eq_info,
  306. page.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)["blocks"],
  307. removed_text_blocks=removed_text_blocks,
  308. removed_image_blocks=removed_images,
  309. images_backup=images_backup,
  310. droped_table_block=[],
  311. table_backup=[],
  312. layout_tree=layout_tree,
  313. page_w=page.rect.width,
  314. page_h=page.rect.height,
  315. footnote_bboxes_tmp=footnote_bboxes_tmp
  316. )
  317. pdf_info_dict[f"page_{page_id}"] = page_info
  318. # end page for
  319. '''计算后处理阶段耗时'''
  320. start_time = time.time()
  321. """
  322. ==================================================================================================================================
  323. 去掉页眉和页脚,这里需要用到一定的统计量,所以放到最后
  324. 页眉和页脚主要从文本box和图片box中去除,位于页面的四周。
  325. 下面函数会直接修改pdf_info_dict,从文字块中、图片中删除属于页眉页脚的内容,删除内容做相对应记录
  326. """
  327. # 去页眉页脚
  328. header, footer = drop_footer_header(pdf_info_dict)
  329. """对单个layout内footnote和他下面的所有textbbox合并"""
  330. for page_key, page_info in pdf_info_dict.items():
  331. page_info = merge_footnote_blocks(page_info, main_text_font)
  332. page_info = remove_footnote_blocks(page_info)
  333. pdf_info_dict[page_key] = page_info
  334. """进入pdf后置过滤器,去掉一些不合理的pdf"""
  335. i = 0
  336. for page_info in pdf_info_dict.values():
  337. is_good_pdf, err = pdf_post_filter(page_info)
  338. if not is_good_pdf:
  339. logger.warning(f"page_id: {i}, drop this pdf: {book_name}, reason: {err}")
  340. if not debug_mode:
  341. return err
  342. i += 1
  343. if debug_mode:
  344. params_file_save_path = join_path(save_tmp_path, "md", book_name, "preproc_out.json")
  345. page_draw_rect_save_path = join_path(save_tmp_path, "md", book_name, "layout.pdf")
  346. # dir_path = os.path.dirname(page_draw_rect_save_path)
  347. # if not os.path.exists(dir_path):
  348. # # 如果目录不存在,创建它
  349. # os.makedirs(dir_path)
  350. with open(params_file_save_path, "w", encoding="utf-8") as f:
  351. json.dump(pdf_info_dict, f, ensure_ascii=False, indent=4)
  352. # 先检测本地 page_draw_rect_save_path 是否存在,如果存在则删除
  353. if os.path.exists(page_draw_rect_save_path):
  354. os.remove(page_draw_rect_save_path)
  355. # 绘制bbox和layout到pdf
  356. draw_bbox_on_page(pdf_docs, pdf_info_dict, page_draw_rect_save_path)
  357. draw_layout_bbox_on_page(pdf_docs, pdf_info_dict, header, footer, page_draw_rect_save_path)
  358. if debug_mode:
  359. # 打印后处理阶段耗时
  360. logger.info(f"post_processing_time: {get_delta_time(start_time)}")
  361. """
  362. ==================================================================================================================================
  363. 进入段落处理-2阶段
  364. """
  365. # 处理行内文字间距较大问题
  366. pdf_info_dict = solve_inline_too_large_interval(pdf_info_dict)
  367. start_time = time.time()
  368. para_process_pipeline = ParaProcessPipeline()
  369. def _deal_with_text_exception(error_info):
  370. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {error_info}")
  371. if error_info == denseSingleLineBlockException_msg:
  372. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.DENSE_SINGLE_LINE_BLOCK}")
  373. result = {"need_drop": True, "drop_reason": DropReason.DENSE_SINGLE_LINE_BLOCK}
  374. return result
  375. if error_info == titleDetectionException_msg:
  376. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.TITLE_DETECTION_FAILED}")
  377. result = {"need_drop": True, "drop_reason": DropReason.TITLE_DETECTION_FAILED}
  378. return result
  379. elif error_info == titleLevelException_msg:
  380. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.TITLE_LEVEL_FAILED}")
  381. result = {"need_drop": True, "drop_reason": DropReason.TITLE_LEVEL_FAILED}
  382. return result
  383. elif error_info == paraSplitException_msg:
  384. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.PARA_SPLIT_FAILED}")
  385. result = {"need_drop": True, "drop_reason": DropReason.PARA_SPLIT_FAILED}
  386. return result
  387. elif error_info == paraMergeException_msg:
  388. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.PARA_MERGE_FAILED}")
  389. result = {"need_drop": True, "drop_reason": DropReason.PARA_MERGE_FAILED}
  390. return result
  391. if debug_mode:
  392. input_pdf_file = f"{pdf_local_path}.pdf"
  393. output_dir = f"{save_path}/{book_name}"
  394. output_pdf_file = f"{output_dir}/pdf_annos.pdf"
  395. """
  396. Call the para_process_pipeline function to process the pdf_info_dict.
  397. Parameters:
  398. para_debug_mode: str or None
  399. If para_debug_mode is None, the para_process_pipeline will not keep any intermediate results.
  400. If para_debug_mode is "simple", the para_process_pipeline will only keep the annos on the pdf and the final results as a json file.
  401. If para_debug_mode is "full", the para_process_pipeline will keep all the intermediate results generated during each step.
  402. """
  403. pdf_info_dict, error_info = para_process_pipeline.para_process_pipeline(
  404. pdf_info_dict,
  405. para_debug_mode="simple",
  406. input_pdf_path=input_pdf_file,
  407. output_pdf_path=output_pdf_file,
  408. )
  409. # 打印段落处理阶段耗时
  410. logger.info(f"para_process_time: {get_delta_time(start_time)}")
  411. # debug的时候不return drop信息
  412. if error_info is not None:
  413. _deal_with_text_exception(error_info)
  414. return pdf_info_dict
  415. else:
  416. pdf_info_dict, error_info = para_process_pipeline.para_process_pipeline(pdf_info_dict)
  417. if error_info is not None:
  418. return _deal_with_text_exception(error_info)
  419. return pdf_info_dict