import re from loguru import logger from magic_pdf.libs.commons import join_path from magic_pdf.libs.language import detect_lang from magic_pdf.libs.MakeContentConfig import DropMode, MakeMode from magic_pdf.libs.markdown_utils import ocr_escape_special_markdown_char from magic_pdf.libs.ocr_content_type import BlockType, ContentType from magic_pdf.para.para_split_v3 import ListLineTag def __is_hyphen_at_line_end(line): """ Check if a line ends with one or more letters followed by a hyphen. Args: line (str): The line of text to check. Returns: bool: True if the line ends with one or more letters followed by a hyphen, False otherwise. """ # Use regex to check if the line ends with one or more letters followed by a hyphen return bool(re.search(r'[A-Za-z]+-\s*$', line)) def ocr_mk_mm_markdown_with_para_and_pagination(pdf_info_dict: list, img_buket_path): markdown_with_para_and_pagination = [] page_no = 0 for page_info in pdf_info_dict: paras_of_layout = page_info.get('para_blocks') if not paras_of_layout: continue page_markdown = ocr_mk_markdown_with_para_core_v2( paras_of_layout, 'mm', img_buket_path) markdown_with_para_and_pagination.append({ 'page_no': page_no, 'md_content': '\n\n'.join(page_markdown) }) page_no += 1 return markdown_with_para_and_pagination def ocr_mk_markdown_with_para_core_v2(paras_of_layout, mode, img_buket_path='', ): page_markdown = [] for para_block in paras_of_layout: para_text = '' para_type = para_block['type'] if para_type in [BlockType.Text, BlockType.List, BlockType.Index]: para_text = merge_para_with_text(para_block) elif para_type == BlockType.Title: para_text = f'# {merge_para_with_text(para_block)}' elif para_type == BlockType.InterlineEquation: para_text = merge_para_with_text(para_block) elif para_type == BlockType.Image: if mode == 'nlp': continue elif mode == 'mm': for block in para_block['blocks']: # 1st.拼image_body if block['type'] == BlockType.ImageBody: for line in block['lines']: for span in line['spans']: if span['type'] == ContentType.Image: if span.get('image_path', ''): para_text += f"\n![]({join_path(img_buket_path, span['image_path'])}) \n" for block in para_block['blocks']: # 2nd.拼image_caption if block['type'] == BlockType.ImageCaption: para_text += merge_para_with_text(block) + ' \n' for block in para_block['blocks']: # 3rd.拼image_footnote if block['type'] == BlockType.ImageFootnote: para_text += merge_para_with_text(block) + ' \n' elif para_type == BlockType.Table: if mode == 'nlp': continue elif mode == 'mm': for block in para_block['blocks']: # 1st.拼table_caption if block['type'] == BlockType.TableCaption: para_text += merge_para_with_text(block) + ' \n' for block in para_block['blocks']: # 2nd.拼table_body if block['type'] == BlockType.TableBody: for line in block['lines']: for span in line['spans']: if span['type'] == ContentType.Table: # if processed by table model if span.get('latex', ''): para_text += f"\n\n$\n {span['latex']}\n$\n\n" elif span.get('html', ''): para_text += f"\n\n{span['html']}\n\n" elif span.get('image_path', ''): para_text += f"\n![]({join_path(img_buket_path, span['image_path'])}) \n" for block in para_block['blocks']: # 3rd.拼table_footnote if block['type'] == BlockType.TableFootnote: para_text += merge_para_with_text(block) + ' \n' if para_text.strip() == '': continue else: page_markdown.append(para_text.strip() + ' ') return page_markdown def detect_language(text): en_pattern = r'[a-zA-Z]+' en_matches = re.findall(en_pattern, text) en_length = sum(len(match) for match in en_matches) if len(text) > 0: if en_length / len(text) >= 0.5: return 'en' else: return 'unknown' else: return 'empty' def merge_para_with_text(para_block): para_text = '' for i, line in enumerate(para_block['lines']): if i >= 1 and line.get(ListLineTag.IS_LIST_START_LINE, False): para_text += ' \n' line_text = '' line_lang = '' for span in line['spans']: span_type = span['type'] if span_type == ContentType.Text: line_text += span['content'].strip() if line_text != '': line_lang = detect_lang(line_text) for span in line['spans']: span_type = span['type'] content = '' if span_type == ContentType.Text: content = ocr_escape_special_markdown_char(span['content']) elif span_type == ContentType.InlineEquation: content = f" ${span['content']}$ " elif span_type == ContentType.InterlineEquation: content = f"\n$$\n{span['content']}\n$$\n" if content != '': langs = ['zh', 'ja', 'ko'] if line_lang in langs: # 遇到一些一个字一个span的文档,这种单字语言判断不准,需要用整行文本判断 para_text += content # 中文/日语/韩文语境下,content间不需要空格分隔 elif line_lang == 'en': # 如果是前一行带有-连字符,那么末尾不应该加空格 if __is_hyphen_at_line_end(content): para_text += content[:-1] else: para_text += content + ' ' else: para_text += content + ' ' # 西方文本语境下 content间需要空格分隔 return para_text def para_to_standard_format_v2(para_block, img_buket_path, page_idx, drop_reason=None): para_type = para_block['type'] para_content = {} if para_type in [BlockType.Text, BlockType.List, BlockType.Index]: para_content = { 'type': 'text', 'text': merge_para_with_text(para_block), } elif para_type == BlockType.Title: para_content = { 'type': 'text', 'text': merge_para_with_text(para_block), 'text_level': 1, } elif para_type == BlockType.InterlineEquation: para_content = { 'type': 'equation', 'text': merge_para_with_text(para_block), 'text_format': 'latex', } elif para_type == BlockType.Image: para_content = {'type': 'image', 'img_path': '', 'img_caption': [], 'img_footnote': []} for block in para_block['blocks']: if block['type'] == BlockType.ImageBody: for line in block['lines']: for span in line['spans']: if span['type'] == ContentType.Image: if span.get('image_path', ''): para_content['img_path'] = join_path(img_buket_path, span['image_path']) if block['type'] == BlockType.ImageCaption: para_content['img_caption'].append(merge_para_with_text(block)) if block['type'] == BlockType.ImageFootnote: para_content['img_footnote'].append(merge_para_with_text(block)) elif para_type == BlockType.Table: para_content = {'type': 'table', 'img_path': '', 'table_caption': [], 'table_footnote': []} for block in para_block['blocks']: if block['type'] == BlockType.TableBody: for line in block['lines']: for span in line['spans']: if span['type'] == ContentType.Table: if span.get('latex', ''): para_content['table_body'] = f"\n\n$\n {span['latex']}\n$\n\n" elif span.get('html', ''): para_content['table_body'] = f"\n\n{span['html']}\n\n" if span.get('image_path', ''): para_content['img_path'] = join_path(img_buket_path, span['image_path']) if block['type'] == BlockType.TableCaption: para_content['table_caption'].append(merge_para_with_text(block)) if block['type'] == BlockType.TableFootnote: para_content['table_footnote'].append(merge_para_with_text(block)) para_content['page_idx'] = page_idx if drop_reason is not None: para_content['drop_reason'] = drop_reason return para_content def union_make(pdf_info_dict: list, make_mode: str, drop_mode: str, img_buket_path: str = '', ): output_content = [] for page_info in pdf_info_dict: drop_reason_flag = False drop_reason = None if page_info.get('need_drop', False): drop_reason = page_info.get('drop_reason') if drop_mode == DropMode.NONE: pass elif drop_mode == DropMode.NONE_WITH_REASON: drop_reason_flag = True elif drop_mode == DropMode.WHOLE_PDF: raise Exception((f'drop_mode is {DropMode.WHOLE_PDF} ,' f'drop_reason is {drop_reason}')) elif drop_mode == DropMode.SINGLE_PAGE: logger.warning((f'drop_mode is {DropMode.SINGLE_PAGE} ,' f'drop_reason is {drop_reason}')) continue else: raise Exception('drop_mode can not be null') paras_of_layout = page_info.get('para_blocks') page_idx = page_info.get('page_idx') if not paras_of_layout: continue if make_mode == MakeMode.MM_MD: page_markdown = ocr_mk_markdown_with_para_core_v2( paras_of_layout, 'mm', img_buket_path) output_content.extend(page_markdown) elif make_mode == MakeMode.NLP_MD: page_markdown = ocr_mk_markdown_with_para_core_v2( paras_of_layout, 'nlp') output_content.extend(page_markdown) elif make_mode == MakeMode.STANDARD_FORMAT: for para_block in paras_of_layout: if drop_reason_flag: para_content = para_to_standard_format_v2( para_block, img_buket_path, page_idx) else: para_content = para_to_standard_format_v2( para_block, img_buket_path, page_idx) output_content.append(para_content) if make_mode in [MakeMode.MM_MD, MakeMode.NLP_MD]: return '\n\n'.join(output_content) elif make_mode == MakeMode.STANDARD_FORMAT: return output_content