MinerU/mineru/backend/pipeline/pipeline_analyze.py

import os
import time
import numpy as np
import torch
from pypdfium2 import PdfDocument

from mineru.backend.pipeline.model_init import MineruPipelineModel
from .model_json_to_middle_json import result_to_middle_json
from ...utils.pdf_classify import classify
from ...utils.pdf_image_tools import pdf_page_to_image




from loguru import logger

from ...utils.model_utils import get_vram, clean_memory
from magic_pdf.libs.config_reader import (get_device, get_formula_config,
                                          get_layout_config,
                                          get_local_models_dir,
                                          get_table_recog_config)



os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'  # 让mps可以fallback
os.environ['NO_ALBUMENTATIONS_UPDATE'] = '1'  # 禁止albumentations检查更新

class ModelSingleton:
    _instance = None
    _models = {}

    def __new__(cls, *args, **kwargs):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
        return cls._instance

    def get_model(
        self,
        lang=None,
        formula_enable=None,
        table_enable=None,
    ):
        key = (lang, formula_enable, table_enable)
        if key not in self._models:
            self._models[key] = custom_model_init(
                lang=lang,
                formula_enable=formula_enable,
                table_enable=table_enable,
            )
        return self._models[key]


def custom_model_init(
    lang=None,
    formula_enable=None,
    table_enable=None,
):
    model_init_start = time.time()
    # 从配置文件读取model-dir和device
    local_models_dir = get_local_models_dir()
    device = get_device()

    formula_config = get_formula_config()
    if formula_enable is not None:
        formula_config['enable'] = formula_enable

    table_config = get_table_recog_config()
    if table_enable is not None:
        table_config['enable'] = table_enable

    model_input = {
        'models_dir': local_models_dir,
        'device': device,
        'table_config': table_config,
        'formula_config': formula_config,
        'lang': lang,
    }

    custom_model = MineruPipelineModel(**model_input)

    model_init_cost = time.time() - model_init_start
    logger.info(f'model init cost: {model_init_cost}')

    return custom_model


def doc_analyze(
        pdf_bytes_list,
        lang_list,
        parse_method: str = 'auto',
        formula_enable=None,
        table_enable=None,
):
    """
    统一处理文档分析函数，根据输入参数类型决定处理单个数据集还是多个数据集

    Args:
        dataset_or_datasets: 单个Dataset对象或Dataset对象列表
        parse_method: 解析方法，'auto'/'ocr'/'txt'
        formula_enable: 是否启用公式识别
        table_enable: 是否启用表格识别

    Returns:
        单个dataset时返回单个model_json，多个dataset时返回model_json列表
    """
    MIN_BATCH_INFERENCE_SIZE = int(os.environ.get('MINERU_MIN_BATCH_INFERENCE_SIZE', 100))

    # 收集所有页面信息
    all_pages_info = []  # 存储(dataset_index, page_index, img, ocr, lang, width, height)

    for pdf_idx, pdf_bytes in enumerate(pdf_bytes_list):
        # 确定OCR设置
        _ocr = False
        if parse_method == 'auto':
            if classify(pdf_bytes) == 'ocr':
                _ocr = True
        elif parse_method == 'ocr':
            _ocr = True

        _lang = lang_list[pdf_idx]

        # 收集每个数据集中的页面
        pdf_doc = PdfDocument(pdf_bytes)
        for page_idx in range(len(pdf_doc)):
            page_data = pdf_doc[page_idx]
            img_dict = pdf_page_to_image(page_data)
            all_pages_info.append((
                pdf_idx, page_idx,
                img_dict['img_pil'], _ocr, _lang,
                img_dict['scale']
            ))

    # 准备批处理
    images_with_extra_info = [(info[2], info[3], info[4]) for info in all_pages_info]
    batch_size = MIN_BATCH_INFERENCE_SIZE
    batch_images = [
        images_with_extra_info[i:i + batch_size]
        for i in range(0, len(images_with_extra_info), batch_size)
    ]

    # 执行批处理
    results = []
    processed_images_count = 0
    for index, batch_image in enumerate(batch_images):
        processed_images_count += len(batch_image)
        logger.info(
            f'Batch {index + 1}/{len(batch_images)}: '
            f'{processed_images_count} pages/{len(images_with_extra_info)} pages'
        )
        batch_results = may_batch_image_analyze(batch_image, formula_enable, table_enable)
        results.extend(batch_results)

    # 构建返回结果

    # 多数据集模式：按数据集分组结果
    infer_results = [[] for _ in datasets]

    for i, page_info in enumerate(all_pages_info):
        pdf_idx, page_idx, pil_img, _, _ = page_info
        result = results[i]

        page_info_dict = {'page_no': page_idx, 'width': pil_img.get_width(), 'height': pil_img.get_height()}
        page_dict = {'layout_dets': result, 'page_info': page_info_dict}
        infer_results[pdf_idx].append(page_dict)

    middle_json_list = []
    for model_json in infer_results:
        middle_json = result_to_middle_json(model_json)
        middle_json_list.append(middle_json)

    return middle_json_list, infer_results


def may_batch_image_analyze(
        images_with_extra_info: list[(np.ndarray, bool, str)],
        formula_enable=None,
        table_enable=None):
    # os.environ['CUDA_VISIBLE_DEVICES'] = str(idx)

    from .batch_analyze import BatchAnalyze

    model_manager = ModelSingleton()

    batch_ratio = 1
    device = get_device()

    if str(device).startswith('npu'):
        import torch_npu
        if torch_npu.npu.is_available():
            torch.npu.set_compile_mode(jit_compile=False)

    if str(device).startswith('npu') or str(device).startswith('cuda'):
        vram = get_vram(device)
        if vram is not None:
            gpu_memory = int(os.getenv('VIRTUAL_VRAM_SIZE', round(vram)))
            if gpu_memory >= 16:
                batch_ratio = 16
            elif gpu_memory >= 12:
                batch_ratio = 8
            elif gpu_memory >= 8:
                batch_ratio = 4
            elif gpu_memory >= 6:
                batch_ratio = 2
            else:
                batch_ratio = 1
            logger.info(f'gpu_memory: {gpu_memory} GB, batch_ratio: {batch_ratio}')
        else:
            # Default batch_ratio when VRAM can't be determined
            batch_ratio = 1
            logger.info(f'Could not determine GPU memory, using default batch_ratio: {batch_ratio}')

    batch_model = BatchAnalyze(model_manager, batch_ratio, formula_enable, table_enable)
    results = batch_model(images_with_extra_info)

    clean_memory(get_device())

    return results