Merge pull request #3183 from opendatalab/release-2.1.6

Release 2.1.6

README.md

@@ -43,6 +43,9 @@
 </div>
 
 # Changelog
+- 2025/07/26 2.1.6 Released
+  - Fixed table parsing issues in handwritten documents when using `vlm` backend
+  - Fixed visualization box position drift issue when document is rotated #3175
 - 2025/07/24 2.1.5 Released
   - `sglang` 0.4.9 version adaptation, synchronously upgrading the dockerfile base image to sglang 0.4.9.post3
 - 2025/07/23 2.1.4 Released

README_zh-CN.md

@@ -43,6 +43,9 @@
 </div>
 
 # 更新记录
+- 2025/07/26 2.1.6发布
+  - 修复`vlm`后端解析部分手写文档时的表格异常问题
+  - 修复文档旋转时可视化框位置漂移问题 #3175
 - 2025/07/24 2.1.5发布
   - `sglang` 0.4.9 版本适配，同步升级dockerfile基础镜像为sglang 0.4.9.post3
 - 2025/07/23 2.1.4发布

mineru/backend/pipeline/batch_analyze.py

@@ -256,9 +256,12 @@ class BatchAnalyze:
                 html_code, table_cell_bboxes, logic_points, elapse = table_model.predict(table_res_dict['table_img'])
                 # 判断是否返回正常
                 if html_code:
-                    expected_ending = html_code.strip().endswith('</html>') or html_code.strip().endswith('</table>')
-                    if expected_ending:
-                        table_res_dict['table_res']['html'] = html_code
+                    # 检查html_code是否包含'<table>'和'</table>'
+                    if '<table>' in html_code and '</table>' in html_code:
+                        # 选用<table>到</table>的内容，放入table_res_dict['table_res']['html']
+                        start_index = html_code.find('<table>')
+                        end_index = html_code.rfind('</table>') + len('</table>')
+                        table_res_dict['table_res']['html'] = html_code[start_index:end_index]
                     else:
                         logger.warning(
                             'table recognition processing fails, not found expected HTML table end'

mineru/backend/vlm/vlm_magic_model.py

@@ -5,7 +5,7 @@ from loguru import logger
 
 from mineru.utils.enum_class import ContentType, BlockType, SplitFlag
 from mineru.backend.vlm.vlm_middle_json_mkcontent import merge_para_with_text
-from mineru.utils.format_utils import convert_otsl_to_html
+from mineru.utils.format_utils import block_content_to_html
 from mineru.utils.magic_model_utils import reduct_overlap, tie_up_category_by_distance_v3
 
 
@@ -40,6 +40,10 @@ class MagicModel:
                 block_type = block_info[1].strip()
                 block_content = block_info[2].strip()
 
+                # 如果bbox是0,0,999,999，且type为text，按notes增加表格处理
+                if x1 == 0 and y1 == 0 and x2 == 999 and y2 == 999 and block_type == "text":
+                    block_content = block_content_to_html(block_content)
+
                 # print(f"坐标: {block_bbox}")
                 # print(f"类型: {block_type}")
                 # print(f"内容: {block_content}")
@@ -77,16 +81,7 @@ class MagicModel:
                     "type": span_type,
                 }
                 if span_type == ContentType.TABLE:
-                    if "<fcel>" in block_content or "<ecel>" in block_content:
-                        lines = block_content.split("\n\n")
-                        new_lines = []
-                        for line in lines:
-                            if "<fcel>" in line or "<ecel>" in line:
-                                line = convert_otsl_to_html(line)
-                            new_lines.append(line)
-                        span["html"] = "\n\n".join(new_lines)
-                    else:
-                        span["html"] = block_content
+                    span["html"] = block_content_to_html(block_content)
             elif span_type in [ContentType.INTERLINE_EQUATION]:
                 span = {
                     "bbox": block_bbox,

mineru/utils/draw_bbox.py

@@ -2,21 +2,64 @@ import json
 from io import BytesIO
 
 from loguru import logger
-from pypdf import PdfReader, PdfWriter
+from pypdf import PdfReader, PdfWriter, PageObject
 from reportlab.pdfgen import canvas
 
 from .enum_class import BlockType, ContentType
 
 
+def cal_canvas_rect(page, bbox):
+    """
+    Calculate the rectangle coordinates on the canvas based on the original PDF page and bounding box.
+
+    Args:
+        page: A PyPDF2 Page object representing a single page in the PDF.
+        bbox: [x0, y0, x1, y1] representing the bounding box coordinates.
+
+    Returns:
+        rect: [x0, y0, width, height] representing the rectangle coordinates on the canvas.
+    """
+    page_width, page_height = float(page.cropbox[2]), float(page.cropbox[3])
+    
+    actual_width = page_width    # The width of the final PDF display
+    actual_height = page_height  # The height of the final PDF display
+    
+    rotation = page.get("/Rotate", 0)
+    rotation = rotation % 360
+    
+    if rotation in [90, 270]:
+        # PDF is rotated 90 degrees or 270 degrees, and the width and height need to be swapped
+        actual_width, actual_height = actual_height, actual_width
+        
+    x0, y0, x1, y1 = bbox
+    rect_w = abs(x1 - x0)
+    rect_h = abs(y1 - y0)
+    
+    if 270 == rotation:
+        rect_w, rect_h = rect_h, rect_w
+        x0 = actual_height - y1
+        y0 = actual_width - x1
+    elif 180 == rotation:
+        x0 = page_width - x1
+        y0 = y0
+    elif 90 == rotation:
+        rect_w, rect_h = rect_h, rect_w
+        x0, y0 = y0, x0 
+    else:
+        # 0 == rotation:
+        x0 = x0
+        y0 = page_height - y1
+    
+    rect = [x0, y0, rect_w, rect_h]        
+    return rect
+
+
 def draw_bbox_without_number(i, bbox_list, page, c, rgb_config, fill_config):
     new_rgb = [float(color) / 255 for color in rgb_config]
     page_data = bbox_list[i]
-    page_width, page_height = page.cropbox[2], page.cropbox[3]
 
     for bbox in page_data:
-        width = bbox[2] - bbox[0]
-        height = bbox[3] - bbox[1]
-        rect = [bbox[0], page_height - bbox[3], width, height]  # Define the rectangle
+        rect = cal_canvas_rect(page, bbox)  # Define the rectangle  
 
         if fill_config:  # filled rectangle
             c.setFillColorRGB(new_rgb[0], new_rgb[1], new_rgb[2], 0.3)
@@ -35,10 +78,8 @@ def draw_bbox_with_number(i, bbox_list, page, c, rgb_config, fill_config, draw_b
 
     for j, bbox in enumerate(page_data):
         # 确保bbox的每个元素都是float
-        x0, y0, x1, y1 = map(float, bbox)
-        width = x1 - x0
-        height = y1 - y0
-        rect = [x0, page_height - y1, width, height]
+        rect = cal_canvas_rect(page, bbox)  # Define the rectangle  
+        
         if draw_bbox:
             if fill_config:
                 c.setFillColorRGB(*new_rgb, 0.3)
@@ -48,8 +89,23 @@ def draw_bbox_with_number(i, bbox_list, page, c, rgb_config, fill_config, draw_b
                 c.rect(rect[0], rect[1], rect[2], rect[3], stroke=1, fill=0)
         c.setFillColorRGB(*new_rgb, 1.0)
         c.setFontSize(size=10)
-        # 这里也要用float
-        c.drawString(x1 + 2, page_height - y0 - 10, str(j + 1))
+        
+        c.saveState()
+        rotation = page.get("/Rotate", 0)
+        rotation = rotation % 360
+    
+        if 0 == rotation:
+            c.translate(rect[0] + rect[2] + 2, rect[1] + rect[3] - 10)
+        elif 90 == rotation:
+            c.translate(rect[0] + 10, rect[1] + rect[3] + 2)
+        elif 180 == rotation:
+            c.translate(rect[0] - 2, rect[1] + 10)
+        elif 270 == rotation:
+            c.translate(rect[0] + rect[2] - 10, rect[1] - 2)
+            
+        c.rotate(rotation)
+        c.drawString(0, 0, str(j + 1))
+        c.restoreState()
 
     return c
 
@@ -185,6 +241,9 @@ def draw_layout_bbox(pdf_info, pdf_bytes, out_path, filename):
 
         # 添加检查确保overlay_pdf.pages不为空
         if len(overlay_pdf.pages) > 0:
+            new_page = PageObject(pdf=None)
+            new_page.update(page)
+            page = new_page
             page.merge_page(overlay_pdf.pages[0])
         else:
             # 记录日志并继续处理下一个页面
@@ -300,6 +359,9 @@ def draw_span_bbox(pdf_info, pdf_bytes, out_path, filename):
 
         # 添加检查确保overlay_pdf.pages不为空
         if len(overlay_pdf.pages) > 0:
+            new_page = PageObject(pdf=None)
+            new_page.update(page)
+            page = new_page
             page.merge_page(overlay_pdf.pages[0])
         else:
             # 记录日志并继续处理下一个页面

mineru/utils/format_utils.py

@@ -317,3 +317,26 @@ def convert_otsl_to_html(otsl_content: str):
     )
 
     return export_to_html(table_data)
+
+
+def block_content_to_html(block_content: str) -> str:
+    """
+    Converts block content containing OTSL (Open Table Structure Language) tags into HTML.
+
+    This function processes a block of text, splitting it into lines and converting any lines
+    containing OTSL table tags (e.g., <fcel>, <ecel>) into HTML tables. Lines without these
+    tags are left unchanged.
+
+    Parameters:
+        block_content (str): A string containing block content with potential OTSL tags.
+
+    Returns:
+        str: The processed block content with OTSL tags converted to HTML tables.
+    """
+    lines = block_content.split("\n\n")
+    new_lines = []
+    for line in lines:
+        if "<fcel>" in line or "<ecel>" in line:
+            line = convert_otsl_to_html(line)
+        new_lines.append(line)
+    return "\n\n".join(new_lines)