MinerU/mineru/backend/pipeline/pipeline_analyze.py

import os
import time
import numpy as np
import torch

from .model_init import MineruPipelineModel
from .config_reader import get_local_models_dir, get_device, get_formula_config, get_table_recog_config
from .model_json_to_middle_json import result_to_middle_json
from ...data.data_reader_writer import DataWriter
from ...utils.pdf_classify import classify
from ...utils.pdf_image_tools import load_images_from_pdf

from loguru import logger

from ...utils.model_utils import get_vram, clean_memory


os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'  # 让mps可以fallback
os.environ['NO_ALBUMENTATIONS_UPDATE'] = '1'  # 禁止albumentations检查更新

class ModelSingleton:
    _instance = None
    _models = {}

    def __new__(cls, *args, **kwargs):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
        return cls._instance

    def get_model(
        self,
        lang=None,
        formula_enable=None,
        table_enable=None,
    ):
        key = (lang, formula_enable, table_enable)
        if key not in self._models:
            self._models[key] = custom_model_init(
                lang=lang,
                formula_enable=formula_enable,
                table_enable=table_enable,
            )
        return self._models[key]


def custom_model_init(
    lang=None,
    formula_enable=None,
    table_enable=None,
):
    model_init_start = time.time()
    # 从配置文件读取model-dir和device
    local_models_dir = get_local_models_dir()
    device = get_device()

    formula_config = get_formula_config()
    if formula_enable is not None:
        formula_config['enable'] = formula_enable

    table_config = get_table_recog_config()
    if table_enable is not None:
        table_config['enable'] = table_enable

    model_input = {
        'models_dir': local_models_dir,
        'device': device,
        'table_config': table_config,
        'formula_config': formula_config,
        'lang': lang,
    }

    custom_model = MineruPipelineModel(**model_input)

    model_init_cost = time.time() - model_init_start
    logger.info(f'model init cost: {model_init_cost}')

    return custom_model


def doc_analyze(
        pdf_bytes_list,
        lang_list,
        image_writer: DataWriter | None,
        parse_method: str = 'auto',
        formula_enable=None,
        table_enable=None,
):
    """
    统一处理文档分析函数，根据输入参数类型决定处理单个数据集还是多个数据集

    Args:
        dataset_or_datasets: 单个Dataset对象或Dataset对象列表
        parse_method: 解析方法，'auto'/'ocr'/'txt'
        formula_enable: 是否启用公式识别
        table_enable: 是否启用表格识别

    Returns:
        单个dataset时返回单个model_json，多个dataset时返回model_json列表
    """
    MIN_BATCH_INFERENCE_SIZE = int(os.environ.get('MINERU_MIN_BATCH_INFERENCE_SIZE', 100))

    # 收集所有页面信息
    all_pages_info = []  # 存储(dataset_index, page_index, img, ocr, lang, width, height)

    all_image_lists = []
    all_pdf_docs = []
    ocr_enabled_list = []
    for pdf_idx, pdf_bytes in enumerate(pdf_bytes_list):
        # 确定OCR设置
        _ocr = False
        if parse_method == 'auto':
            if classify(pdf_bytes) == 'ocr':
                _ocr = True
        elif parse_method == 'ocr':
            _ocr = True

        ocr_enabled_list[pdf_idx] = _ocr
        _lang = lang_list[pdf_idx]

        # 收集每个数据集中的页面
        images_list, pdf_doc = load_images_from_pdf(pdf_bytes)
        all_image_lists.append(images_list)
        all_pdf_docs.append(pdf_doc)
        for page_idx in range(len(images_list)):
            img_dict = images_list[page_idx]
            all_pages_info.append((
                pdf_idx, page_idx,
                img_dict['img_pil'], _ocr, _lang,
            ))

    # 准备批处理
    images_with_extra_info = [(info[2], info[3], info[4]) for info in all_pages_info]
    batch_size = MIN_BATCH_INFERENCE_SIZE
    batch_images = [
        images_with_extra_info[i:i + batch_size]
        for i in range(0, len(images_with_extra_info), batch_size)
    ]

    # 执行批处理
    results = []
    processed_images_count = 0
    for index, batch_image in enumerate(batch_images):
        processed_images_count += len(batch_image)
        logger.info(
            f'Batch {index + 1}/{len(batch_images)}: '
            f'{processed_images_count} pages/{len(images_with_extra_info)} pages'
        )
        batch_results = may_batch_image_analyze(batch_image, formula_enable, table_enable)
        results.extend(batch_results)

    # 构建返回结果
    infer_results = []

    for i, page_info in enumerate(all_pages_info):
        pdf_idx, page_idx, pil_img, _, _ = page_info
        result = results[i]

        page_info_dict = {'page_no': page_idx, 'width': pil_img.width, 'height': pil_img.height}
        page_dict = {'layout_dets': result, 'page_info': page_info_dict}
        infer_results[pdf_idx][page_idx] = page_dict

    middle_json_list = []
    for pdf_idx, model_list in enumerate(infer_results):
        images_list = all_image_lists[pdf_idx]
        pdf_doc = all_pdf_docs[pdf_idx]
        _lang = lang_list[pdf_idx]
        _ocr = ocr_enabled_list[pdf_idx]
        middle_json = result_to_middle_json(model_list, images_list, pdf_doc, image_writer, _lang, _ocr)
        middle_json_list.append(middle_json)

    return middle_json_list, infer_results


def may_batch_image_analyze(
        images_with_extra_info: list[(np.ndarray, bool, str)],
        formula_enable=None,
        table_enable=None):
    # os.environ['CUDA_VISIBLE_DEVICES'] = str(idx)

    from .batch_analyze import BatchAnalyze

    model_manager = ModelSingleton()

    batch_ratio = 1
    device = get_device()

    if str(device).startswith('npu'):
        import torch_npu
        if torch_npu.npu.is_available():
            torch.npu.set_compile_mode(jit_compile=False)

    if str(device).startswith('npu') or str(device).startswith('cuda'):
        vram = get_vram(device)
        if vram is not None:
            gpu_memory = int(os.getenv('VIRTUAL_VRAM_SIZE', round(vram)))
            if gpu_memory >= 16:
                batch_ratio = 16
            elif gpu_memory >= 12:
                batch_ratio = 8
            elif gpu_memory >= 8:
                batch_ratio = 4
            elif gpu_memory >= 6:
                batch_ratio = 2
            else:
                batch_ratio = 1
            logger.info(f'gpu_memory: {gpu_memory} GB, batch_ratio: {batch_ratio}')
        else:
            # Default batch_ratio when VRAM can't be determined
            batch_ratio = 1
            logger.info(f'Could not determine GPU memory, using default batch_ratio: {batch_ratio}')

    batch_model = BatchAnalyze(model_manager, batch_ratio, formula_enable, table_enable)
    results = batch_model(images_with_extra_info)

    clean_memory(get_device())

    return results