|
|
@@ -1,1533 +0,0 @@
|
|
|
-import sys
|
|
|
-import time
|
|
|
-import re
|
|
|
-import difflib
|
|
|
-import json
|
|
|
-import argparse
|
|
|
-from typing import Dict, List, Tuple
|
|
|
-import markdown
|
|
|
-from bs4 import BeautifulSoup
|
|
|
-from fuzzywuzzy import fuzz
|
|
|
-
|
|
|
-class OCRResultComparator:
|
|
|
- def __init__(self):
|
|
|
- self.differences = []
|
|
|
- self.paragraph_match_threshold = 80 # 段落相似度阈值, 大于80代表段落匹配，<100,表示存在差异，小于80代表段落不匹配
|
|
|
- self.content_similarity_threshold = 95 # 段落匹配，比较内容，大于95认为无差异
|
|
|
- self.max_paragraph_window = 6
|
|
|
- self.table_comparison_mode = 'standard' # 新增：表格比较模式
|
|
|
- self.header_similarity_threshold = 90 # 表头相似度阈值
|
|
|
-
|
|
|
- def normalize_text(self, text: str) -> str:
|
|
|
- """标准化文本：去除多余空格、回车等无效字符"""
|
|
|
- if not text:
|
|
|
- return ""
|
|
|
- # 去除多余的空白字符
|
|
|
- text = re.sub(r'\s+', ' ', text.strip())
|
|
|
- # 去除标点符号周围的空格
|
|
|
- text = re.sub(r'\s*([，。：；！？、])\s*', r'\1', text)
|
|
|
- return text
|
|
|
-
|
|
|
- def is_image_reference(self, text: str) -> bool:
|
|
|
- """判断是否为图片引用或描述"""
|
|
|
- image_keywords = [
|
|
|
- '图', '图片', '图像', 'image', 'figure', 'fig',
|
|
|
- '照片', '截图', '示意图', '流程图', '结构图'
|
|
|
- ]
|
|
|
- # 检查是否包含图片相关关键词
|
|
|
- for keyword in image_keywords:
|
|
|
- if keyword in text.lower():
|
|
|
- return True
|
|
|
-
|
|
|
- # 检查是否为Markdown图片语法
|
|
|
- if re.search(r'!\[.*?\]\(.*?\)', text):
|
|
|
- return True
|
|
|
-
|
|
|
- # 检查是否为HTML图片标签
|
|
|
- if re.search(r'<img[^>]*>', text, re.IGNORECASE):
|
|
|
- return True
|
|
|
-
|
|
|
- return False
|
|
|
-
|
|
|
- def extract_table_data(self, md_content: str) -> List[List[List[str]]]:
|
|
|
- """从Markdown中提取表格数据"""
|
|
|
- tables = []
|
|
|
-
|
|
|
- # 使用BeautifulSoup解析HTML表格
|
|
|
- soup = BeautifulSoup(md_content, 'html.parser')
|
|
|
- html_tables = soup.find_all('table')
|
|
|
-
|
|
|
- for table in html_tables:
|
|
|
- table_data = []
|
|
|
- rows = table.find_all('tr')
|
|
|
-
|
|
|
- for row in rows:
|
|
|
- cells = row.find_all(['td', 'th'])
|
|
|
- row_data = []
|
|
|
- for cell in cells:
|
|
|
- cell_text = self.normalize_text(cell.get_text())
|
|
|
- # 跳过图片内容
|
|
|
- if not self.is_image_reference(cell_text):
|
|
|
- row_data.append(cell_text)
|
|
|
- else:
|
|
|
- row_data.append("[图片内容-忽略]")
|
|
|
-
|
|
|
- if row_data: # 只添加非空行
|
|
|
- table_data.append(row_data)
|
|
|
-
|
|
|
- if table_data:
|
|
|
- tables.append(table_data)
|
|
|
-
|
|
|
- return tables
|
|
|
-
|
|
|
- def merge_split_paragraphs(self, lines: List[str]) -> List[str]:
|
|
|
- # 合并连续的非空行作为一个段落，且过滤图片内容
|
|
|
- merged_lines = []
|
|
|
- current_paragraph = ""
|
|
|
- for i, line in enumerate(lines):
|
|
|
- # 跳过空行
|
|
|
- if not line:
|
|
|
- if current_paragraph:
|
|
|
- merged_lines.append(current_paragraph)
|
|
|
- current_paragraph = ""
|
|
|
- continue
|
|
|
- # 跳过图片内容
|
|
|
- if self.is_image_reference(line):
|
|
|
- continue
|
|
|
-
|
|
|
- # 检查是否是标题（以数字、中文数字或特殊标记开头）
|
|
|
- is_title = (
|
|
|
- line.startswith(('一、', '二、', '三、', '四、', '五、', '六、', '七、', '八、', '九、', '十、')) or
|
|
|
- line.startswith(('1.', '2.', '3.', '4.', '5.', '6.', '7.', '8.', '9.')) or
|
|
|
- line.startswith('#')
|
|
|
- )
|
|
|
- # 如果是标题，结束当前段落
|
|
|
- if is_title:
|
|
|
- if current_paragraph:
|
|
|
- merged_lines.append(current_paragraph)
|
|
|
- current_paragraph = ""
|
|
|
- merged_lines.append(line)
|
|
|
- else:
|
|
|
- # 检查是否应该与前一行合并 # 如果当前段落不为空，且当前段落最后一个字符非空白字符
|
|
|
- if current_paragraph and not current_paragraph.endswith((' ', '\t')):
|
|
|
- current_paragraph += line
|
|
|
- else:
|
|
|
- current_paragraph = line
|
|
|
-
|
|
|
- # 处理最后一个段落
|
|
|
- if current_paragraph:
|
|
|
- merged_lines.append(current_paragraph)
|
|
|
-
|
|
|
- return merged_lines
|
|
|
-
|
|
|
- def extract_paragraphs(self, md_content: str) -> List[str]:
|
|
|
- """提取段落文本"""
|
|
|
- # 移除表格 - 修复正则表达式
|
|
|
- # 使用 IGNORECASE 和 DOTALL 标志
|
|
|
- content = re.sub(r'<table[^>]*>.*?</table>', '', md_content, flags=re.DOTALL | re.IGNORECASE)
|
|
|
-
|
|
|
- # 移除其他 HTML 标签
|
|
|
- content = re.sub(r'<[^>]+>', '', content)
|
|
|
-
|
|
|
- # 移除 Markdown 注释
|
|
|
- content = re.sub(r'<!--.*?-->', '', content, flags=re.DOTALL)
|
|
|
-
|
|
|
- # 分割段落
|
|
|
- paragraphs = []
|
|
|
- lines = content.split('\n')
|
|
|
- merged_lines = self.merge_split_paragraphs(lines)
|
|
|
-
|
|
|
- for line in merged_lines:
|
|
|
- normalized = self.normalize_text(line)
|
|
|
- if normalized:
|
|
|
- paragraphs.append(normalized)
|
|
|
- else:
|
|
|
- print(f"跳过的内容无效或图片段落: {line[0:30] if line else ''}...")
|
|
|
-
|
|
|
- return paragraphs
|
|
|
-
|
|
|
- def compare_tables(self, table1: List[List[str]], table2: List[List[str]]) -> List[Dict]:
|
|
|
- """比较表格数据"""
|
|
|
- differences = []
|
|
|
-
|
|
|
- # 确定最大行数
|
|
|
- max_rows = max(len(table1), len(table2))
|
|
|
-
|
|
|
- for i in range(max_rows):
|
|
|
- row1 = table1[i] if i < len(table1) else []
|
|
|
- row2 = table2[i] if i < len(table2) else []
|
|
|
-
|
|
|
- # 确定最大列数
|
|
|
- max_cols = max(len(row1), len(row2))
|
|
|
-
|
|
|
- for j in range(max_cols):
|
|
|
- cell1 = row1[j] if j < len(row1) else ""
|
|
|
- cell2 = row2[j] if j < len(row2) else ""
|
|
|
-
|
|
|
- # 跳过图片内容比较
|
|
|
- if "[图片内容-忽略]" in cell1 or "[图片内容-忽略]" in cell2:
|
|
|
- continue
|
|
|
-
|
|
|
- if cell1 != cell2:
|
|
|
- # 特别处理数字金额
|
|
|
- if self.is_numeric(cell1) and self.is_numeric(cell2):
|
|
|
- num1 = self.parse_number(cell1)
|
|
|
- num2 = self.parse_number(cell2)
|
|
|
- if abs(num1 - num2) > 0.001: # 允许小数精度误差
|
|
|
- differences.append({
|
|
|
- 'type': 'table_amount',
|
|
|
- 'position': f'行{i+1}列{j+1}',
|
|
|
- 'file1_value': cell1,
|
|
|
- 'file2_value': cell2,
|
|
|
- 'description': f'金额不一致: {cell1} vs {cell2}',
|
|
|
- 'row_index': i,
|
|
|
- 'col_index': j
|
|
|
- })
|
|
|
- else:
|
|
|
- differences.append({
|
|
|
- 'type': 'table_text',
|
|
|
- 'position': f'行{i+1}列{j+1}',
|
|
|
- 'file1_value': cell1,
|
|
|
- 'file2_value': cell2,
|
|
|
- 'description': f'文本不一致: {cell1} vs {cell2}',
|
|
|
- 'row_index': i,
|
|
|
- 'col_index': j
|
|
|
- })
|
|
|
-
|
|
|
- return differences
|
|
|
-
|
|
|
- def parse_number(self, text: str) -> float:
|
|
|
- """解析数字，处理千分位和货币符号"""
|
|
|
- if not text:
|
|
|
- return 0.0
|
|
|
-
|
|
|
- # 移除货币符号和千分位分隔符
|
|
|
- clean_text = re.sub(r'[¥$€£,，\s]', '', text)
|
|
|
-
|
|
|
- # 处理负号
|
|
|
- is_negative = False
|
|
|
- if clean_text.startswith('-') or clean_text.startswith('−'):
|
|
|
- is_negative = True
|
|
|
- clean_text = clean_text[1:]
|
|
|
-
|
|
|
- # 处理括号表示的负数 (123.45) -> -123.45
|
|
|
- if clean_text.startswith('(') and clean_text.endswith(')'):
|
|
|
- is_negative = True
|
|
|
- clean_text = clean_text[1:-1]
|
|
|
-
|
|
|
- try:
|
|
|
- number = float(clean_text)
|
|
|
- return -number if is_negative else number
|
|
|
- except ValueError:
|
|
|
- return 0.0
|
|
|
-
|
|
|
- def extract_datetime(self, text: str) -> str:
|
|
|
- """提取并标准化日期时间"""
|
|
|
- # 尝试匹配各种日期时间格式
|
|
|
- patterns = [
|
|
|
- (r'(\d{4})[-/](\d{1,2})[-/](\d{1,2})\s*(\d{1,2}):(\d{1,2}):(\d{1,2})',
|
|
|
- lambda m: f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)} {m.group(4).zfill(2)}:{m.group(5).zfill(2)}:{m.group(6).zfill(2)}"),
|
|
|
- (r'(\d{4})[-/](\d{1,2})[-/](\d{1,2})',
|
|
|
- lambda m: f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"),
|
|
|
- (r'(\d{4})年(\d{1,2})月(\d{1,2})日',
|
|
|
- lambda m: f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"),
|
|
|
- ]
|
|
|
-
|
|
|
- for pattern, formatter in patterns:
|
|
|
- match = re.search(pattern, text)
|
|
|
- if match:
|
|
|
- return formatter(match)
|
|
|
-
|
|
|
- return text
|
|
|
-
|
|
|
- def is_numeric(self, text: str) -> bool:
|
|
|
- """判断文本是否为数字 - 改进版：区分数值和长数字字符串"""
|
|
|
- """>15位的数字字符串视为文本型数字"""
|
|
|
- if not text:
|
|
|
- return False
|
|
|
-
|
|
|
- # 移除千分位分隔符、空格和负号
|
|
|
- clean_text = re.sub(r'[,，\s-]', '', text)
|
|
|
-
|
|
|
- # ✅ 新增：长数字字符串判断（超过15位，认为是文本型数字）
|
|
|
- if len(clean_text) > 15:
|
|
|
- return False
|
|
|
-
|
|
|
- try:
|
|
|
- float(clean_text)
|
|
|
- return True
|
|
|
- except ValueError:
|
|
|
- return False
|
|
|
-
|
|
|
- def is_text_number(self, text: str) -> bool:
|
|
|
- """
|
|
|
- 判断是否为文本型数字（如账号、订单号、流水号）
|
|
|
-
|
|
|
- 特征：
|
|
|
- 1. 长度超过15位的纯数字
|
|
|
- 2. 或者包含空格/连字符的数字序列
|
|
|
- """
|
|
|
- if not text:
|
|
|
- return False
|
|
|
-
|
|
|
- # 移除空格和连字符
|
|
|
- clean_text = re.sub(r'[\s-]', '', text)
|
|
|
-
|
|
|
- # 检查是否为纯数字且长度超过15位
|
|
|
- if clean_text.isdigit() and len(clean_text) > 15:
|
|
|
- return True
|
|
|
-
|
|
|
- # 检查是否为带空格/连字符的数字序列
|
|
|
- if re.match(r'^[\d\s-]+$', text) and len(clean_text) > 10:
|
|
|
- return True
|
|
|
-
|
|
|
- return False
|
|
|
-
|
|
|
- def detect_column_type(self, column_values: List[str]) -> str:
|
|
|
- """检测列的数据类型 - 改进版：区分数值和文本型数字"""
|
|
|
- if not column_values:
|
|
|
- return 'text'
|
|
|
-
|
|
|
- # 过滤空值, 如果只有1个代表空值的字符，如:"/"、"-"，也视为空值
|
|
|
- non_empty_values = [v for v in column_values if v and v.strip() and v not in ['/', '-']]
|
|
|
- if not non_empty_values:
|
|
|
- return 'text'
|
|
|
-
|
|
|
- # ✅ 优先检测文本型数字（账号、订单号等）
|
|
|
- text_number_count = 0
|
|
|
- for value in non_empty_values[:5]:
|
|
|
- if self.is_text_number(value):
|
|
|
- text_number_count += 1
|
|
|
-
|
|
|
- if text_number_count >= len(non_empty_values[:5]) * 0.6:
|
|
|
- return 'text' # ✅ 新增类型
|
|
|
-
|
|
|
- # 检测是否为日期时间
|
|
|
- datetime_patterns = [
|
|
|
- r'\d{4}[-/]\d{1,2}[-/]\d{1,2}', # YYYY-MM-DD
|
|
|
- r'\d{4}[-/]\d{1,2}[-/]\d{1,2}\s*\d{1,2}:\d{1,2}:\d{1,2}', # YYYY-MM-DD HH:MM:SS
|
|
|
- r'\d{4}年\d{1,2}月\d{1,2}日', # 中文日期
|
|
|
- ]
|
|
|
-
|
|
|
- datetime_count = 0
|
|
|
- for value in non_empty_values[:5]:
|
|
|
- for pattern in datetime_patterns:
|
|
|
- if re.search(pattern, value):
|
|
|
- datetime_count += 1
|
|
|
- break
|
|
|
-
|
|
|
- if datetime_count >= len(non_empty_values[:5]) * 0.6:
|
|
|
- return 'datetime'
|
|
|
-
|
|
|
- # 检测是否为数字/金额（短数字）
|
|
|
- numeric_count = 0
|
|
|
- for value in non_empty_values[:5]:
|
|
|
- if self.is_numeric(value) and not self.is_text_number(value):
|
|
|
- numeric_count += 1
|
|
|
-
|
|
|
- if numeric_count >= len(non_empty_values[:5]) * 0.6:
|
|
|
- return 'numeric'
|
|
|
-
|
|
|
- # 默认为文本
|
|
|
- return 'text'
|
|
|
-
|
|
|
- def normalize_text_number(self, text: str) -> str:
|
|
|
- """
|
|
|
- 标准化文本型数字：移除空格和连字符
|
|
|
-
|
|
|
- Args:
|
|
|
- text: 原始文本
|
|
|
-
|
|
|
- Returns:
|
|
|
- 标准化后的文本
|
|
|
- """
|
|
|
- if not text:
|
|
|
- return ""
|
|
|
-
|
|
|
- # 移除空格、连字符、全角空格
|
|
|
- text = re.sub(r'[\s\-\u3000]', '', text)
|
|
|
-
|
|
|
- return text
|
|
|
-
|
|
|
- def compare_cell_value(self, value1: str, value2: str, column_type: str,
|
|
|
- column_name: str = '') -> Dict:
|
|
|
- """比较单元格值 - 改进版：支持文本型数字"""
|
|
|
- result = {
|
|
|
- 'match': True,
|
|
|
- 'difference': None
|
|
|
- }
|
|
|
-
|
|
|
- # 标准化值
|
|
|
- v1 = self.normalize_text(value1)
|
|
|
- v2 = self.normalize_text(value2)
|
|
|
-
|
|
|
- if v1 == v2:
|
|
|
- return result
|
|
|
-
|
|
|
- # ✅ 新增：文本型数字比较
|
|
|
- if column_type == 'text_number':
|
|
|
- # 标准化后比较（移除空格和连字符）
|
|
|
- norm_v1 = self.normalize_text_number(v1)
|
|
|
- norm_v2 = self.normalize_text_number(v2)
|
|
|
-
|
|
|
- if norm_v1 == norm_v2:
|
|
|
- # 内容相同，只是格式不同（空格差异）
|
|
|
- result['match'] = False
|
|
|
- result['difference'] = {
|
|
|
- 'type': 'table_text',
|
|
|
- 'value1': value1,
|
|
|
- 'value2': value2,
|
|
|
- 'description': f'文本型数字格式差异: "{value1}" vs "{value2}" (内容相同，空格不同)',
|
|
|
- 'severity': 'low'
|
|
|
- }
|
|
|
- else:
|
|
|
- # 内容不同
|
|
|
- result['match'] = False
|
|
|
- result['difference'] = {
|
|
|
- 'type': 'table_text',
|
|
|
- 'value1': value1,
|
|
|
- 'value2': value2,
|
|
|
- 'description': f'文本型数字不一致: {value1} vs {value2}',
|
|
|
- 'severity': 'high'
|
|
|
- }
|
|
|
- return result
|
|
|
-
|
|
|
- # 根据列类型采用不同的比较策略
|
|
|
- if column_type == 'numeric':
|
|
|
- # 数字/金额比较
|
|
|
- if self.is_numeric(v1) and self.is_numeric(v2):
|
|
|
- num1 = self.parse_number(v1) # ✅ 使用 parse_number
|
|
|
- num2 = self.parse_number(v2)
|
|
|
- if abs(num1 - num2) > 0.01: # 允许0.01的误差
|
|
|
- result['match'] = False
|
|
|
- result['difference'] = {
|
|
|
- 'type': 'table_amount',
|
|
|
- 'value1': value1,
|
|
|
- 'value2': value2,
|
|
|
- 'diff_amount': abs(num1 - num2),
|
|
|
- 'description': f'金额不一致: {value1} vs {value2}'
|
|
|
- }
|
|
|
- else:
|
|
|
- # 虽然检测为 numeric，但实际是长数字，按文本比较
|
|
|
- result['match'] = False
|
|
|
- result['difference'] = {
|
|
|
- 'type': 'table_text',
|
|
|
- 'value1': value1,
|
|
|
- 'value2': value2,
|
|
|
- 'description': f'长数字字符串不一致: {value1} vs {value2}'
|
|
|
- }
|
|
|
- elif column_type == 'datetime':
|
|
|
- # 日期时间比较
|
|
|
- datetime1 = self.extract_datetime(v1) # ✅ 使用 extract_datetime
|
|
|
- datetime2 = self.extract_datetime(v2)
|
|
|
-
|
|
|
- if datetime1 != datetime2:
|
|
|
- result['match'] = False
|
|
|
- result['difference'] = {
|
|
|
- 'type': 'table_datetime',
|
|
|
- 'value1': value1,
|
|
|
- 'value2': value2,
|
|
|
- 'description': f'日期时间不一致: {value1} vs {value2}'
|
|
|
- }
|
|
|
- else:
|
|
|
- # 文本比较
|
|
|
- similarity = self.calculate_text_similarity(v1, v2)
|
|
|
- if similarity < self.content_similarity_threshold:
|
|
|
- result['match'] = False
|
|
|
- result['difference'] = {
|
|
|
- 'type': 'table_text',
|
|
|
- 'value1': value1,
|
|
|
- 'value2': value2,
|
|
|
- 'similarity': similarity,
|
|
|
- 'description': f'文本不一致: {value1} vs {value2} (相似度: {similarity:.1f}%)'
|
|
|
- }
|
|
|
-
|
|
|
- return result
|
|
|
-
|
|
|
- def calculate_text_similarity(self, text1: str, text2: str) -> float:
|
|
|
- """改进的相似度计算"""
|
|
|
- if not text1 and not text2:
|
|
|
- return 100.0
|
|
|
- if not text1 or not text2:
|
|
|
- return 0.0
|
|
|
-
|
|
|
- # 如果标准化后完全相同，返回100%
|
|
|
- if text1 == text2:
|
|
|
- return 100.0
|
|
|
-
|
|
|
- # 使用多种相似度算法
|
|
|
- similarity_scores = [
|
|
|
- fuzz.ratio(text1, text2),
|
|
|
- # fuzz.partial_ratio(text1, text2),
|
|
|
- # fuzz.token_sort_ratio(text1, text2),
|
|
|
- # fuzz.token_set_ratio(text1, text2)
|
|
|
- ]
|
|
|
-
|
|
|
- # 对于包含关系，给予更高的权重
|
|
|
- # if text1 in text2 or text2 in text1:
|
|
|
- # max_score = max(similarity_scores)
|
|
|
- # # 提升包含关系的相似度
|
|
|
- # return min(100.0, max_score + 10)
|
|
|
-
|
|
|
- return max(similarity_scores)
|
|
|
-
|
|
|
- def strip_markdown_formatting(self, text: str) -> str:
|
|
|
- """移除Markdown格式标记，只保留纯文本内容"""
|
|
|
- if not text:
|
|
|
- return ""
|
|
|
-
|
|
|
- # 移除标题标记 (# ## ### 等)
|
|
|
- text = re.sub(r'^#+\s*', '', text)
|
|
|
-
|
|
|
- # 移除粗体标记 (**text** 或 __text__)
|
|
|
- text = re.sub(r'\*\*(.+?)\*\*', r'\1', text)
|
|
|
- text = re.sub(r'__(.+?)__', r'\1', text)
|
|
|
-
|
|
|
- # 移除斜体标记 (*text* 或 _text_)
|
|
|
- text = re.sub(r'\*(.+?)\*', r'\1', text)
|
|
|
- text = re.sub(r'_(.+?)_', r'\1', text)
|
|
|
-
|
|
|
- # 移除链接 [text](url)
|
|
|
- text = re.sub(r'\[(.+?)\]\(.+?\)', r'\1', text)
|
|
|
-
|
|
|
- # 移除图片引用 
|
|
|
- text = re.sub(r'!\[.*?\]\(.+?\)', '', text)
|
|
|
-
|
|
|
- # 移除代码标记 `code`
|
|
|
- text = re.sub(r'`(.+?)`', r'\1', text)
|
|
|
-
|
|
|
- # 移除HTML标签
|
|
|
- text = re.sub(r'<[^>]+>', '', text)
|
|
|
-
|
|
|
- # 移除列表标记 (- * + 1. 2. 等)
|
|
|
- text = re.sub(r'^\s*[-*+]\s+', '', text)
|
|
|
- text = re.sub(r'^\s*\d+\.\s+', '', text)
|
|
|
-
|
|
|
- # 移除引用标记 (>)
|
|
|
- text = re.sub(r'^\s*>\s+', '', text)
|
|
|
-
|
|
|
- # 标准化空白字符
|
|
|
- text = re.sub(r'\s+', ' ', text.strip())
|
|
|
-
|
|
|
- return text
|
|
|
-
|
|
|
- def normalize_text_for_comparison(self, text: str) -> str:
|
|
|
- """
|
|
|
- 用于比较的文本标准化：移除格式 + 标准化空白 + 统一标点
|
|
|
-
|
|
|
- Args:
|
|
|
- text: 原始文本
|
|
|
-
|
|
|
- Returns:
|
|
|
- 标准化后的纯文本
|
|
|
- """
|
|
|
- # 第一步：移除Markdown格式
|
|
|
- text = self.strip_markdown_formatting(text)
|
|
|
-
|
|
|
- # 第二步：统一标点符号（中英文转换）
|
|
|
- text = self.normalize_punctuation(text)
|
|
|
-
|
|
|
- # 第三步：标准化空白字符
|
|
|
- text = self.normalize_text(text)
|
|
|
-
|
|
|
- return text
|
|
|
-
|
|
|
- def normalize_punctuation(self, text: str) -> str:
|
|
|
- """
|
|
|
- 统一标点符号 - 将中文标点转换为英文标点
|
|
|
-
|
|
|
- Args:
|
|
|
- text: 原始文本
|
|
|
-
|
|
|
- Returns:
|
|
|
- 标点统一后的文本
|
|
|
- """
|
|
|
- if not text:
|
|
|
- return ""
|
|
|
-
|
|
|
- # 中文标点到英文标点的映射
|
|
|
- punctuation_map = {
|
|
|
- '：': ':', # 冒号
|
|
|
- '；': ';', # 分号
|
|
|
- '，': ',', # 逗号
|
|
|
- '。': '.', # 句号
|
|
|
- '！': '!', # 感叹号
|
|
|
- '？': '?', # 问号
|
|
|
- '（': '(', # 左括号
|
|
|
- '）': ')', # 右括号
|
|
|
- '【': '[', # 左方括号
|
|
|
- '】': ']', # 右方括号
|
|
|
- '《': '<', # 左书名号
|
|
|
- '》': '>', # 右书名号
|
|
|
- '"': '"', # 左双引号
|
|
|
- '"': '"', # 右双引号
|
|
|
- ''': "'", # 左单引号
|
|
|
- ''': "'", # 右单引号
|
|
|
- '、': ',', # 顿号
|
|
|
- '—': '-', # 破折号
|
|
|
- '…': '...', # 省略号
|
|
|
- '～': '~', # 波浪号
|
|
|
- }
|
|
|
-
|
|
|
- for cn_punct, en_punct in punctuation_map.items():
|
|
|
- text = text.replace(cn_punct, en_punct)
|
|
|
-
|
|
|
- return text
|
|
|
-
|
|
|
- def check_punctuation_differences(self, text1: str, text2: str) -> List[Dict]:
|
|
|
- """
|
|
|
- 检查两段文本的标点符号差异
|
|
|
-
|
|
|
- Args:
|
|
|
- text1: 文本1
|
|
|
- text2: 文本2
|
|
|
-
|
|
|
- Returns:
|
|
|
- 标点差异列表
|
|
|
- """
|
|
|
- differences = []
|
|
|
-
|
|
|
- # 如果标准化后相同，说明只有标点差异
|
|
|
- normalized1 = self.normalize_punctuation(text1)
|
|
|
- normalized2 = self.normalize_punctuation(text2)
|
|
|
-
|
|
|
- if normalized1 == normalized2 and text1 != text2:
|
|
|
- # 找出具体的标点差异位置
|
|
|
- min_len = min(len(text1), len(text2))
|
|
|
-
|
|
|
- for i in range(min_len):
|
|
|
- if text1[i] != text2[i]:
|
|
|
- # 检查是否是全角半角标点的差异
|
|
|
- char1 = text1[i]
|
|
|
- char2 = text2[i]
|
|
|
-
|
|
|
- # 使用normalize_punctuation检查是否是对应的全角半角
|
|
|
- if self.normalize_punctuation(char1) == self.normalize_punctuation(char2):
|
|
|
- # 提取上下文（前后各3个字符）
|
|
|
- start = max(0, i - 3)
|
|
|
- end = min(len(text1), i + 4)
|
|
|
- context1 = text1[start:end]
|
|
|
- context2 = text2[start:end]
|
|
|
-
|
|
|
- differences.append({
|
|
|
- 'position': i,
|
|
|
- 'char1': char1,
|
|
|
- 'char2': char2,
|
|
|
- 'context1': context1,
|
|
|
- 'context2': context2,
|
|
|
- 'type': 'full_half_width'
|
|
|
- })
|
|
|
-
|
|
|
- return differences
|
|
|
-
|
|
|
- def compare_paragraphs_with_flexible_matching(self, paras1: List[str], paras2: List[str]) -> List[Dict]:
|
|
|
- """改进的段落匹配算法 - 更好地处理段落重组"""
|
|
|
- """_summary_
|
|
|
- paras1: 文件1的段落列表
|
|
|
- paras2: 文件2的段落列表
|
|
|
- paras1和paras2中的段落顺序有可能不一致，需要对窗口内的段落进行匹配，窗口的段落的顺序可以不一样
|
|
|
- para1和para2中的段落可能存在合并或拆分的情况，需要考虑这种情况
|
|
|
- """
|
|
|
- differences = []
|
|
|
-
|
|
|
- # ✅ 预处理：移除格式并统一标点（用于匹配）
|
|
|
- normalized_paras1 = [self.normalize_text_for_comparison(p) for p in paras1]
|
|
|
- normalized_paras2 = [self.normalize_text_for_comparison(p) for p in paras2]
|
|
|
-
|
|
|
- # 但保留原始文本（用于差异检测）
|
|
|
- original_paras1 = [self.strip_markdown_formatting(p) for p in paras1]
|
|
|
- original_paras2 = [self.strip_markdown_formatting(p) for p in paras2]
|
|
|
-
|
|
|
- # 使用预处理后的段落进行匹配
|
|
|
- used_paras1 = set()
|
|
|
- used_paras2 = set()
|
|
|
-
|
|
|
- # 文件1和文件2同时向下遍历
|
|
|
- start_index2 = 0
|
|
|
- last_match_index2 = 0
|
|
|
-
|
|
|
- for window_size1 in range(1, min(self.max_paragraph_window, len(normalized_paras1) + 1)):
|
|
|
- for i in range(len(normalized_paras1) - window_size1 + 1):
|
|
|
- # 跳过已使用的段落
|
|
|
- if any(idx in used_paras1 for idx in range(i, i + window_size1)):
|
|
|
- continue
|
|
|
-
|
|
|
- # 合并文件1中的段落（用于匹配的标准化版本）
|
|
|
- combined_normalized1 = "".join(normalized_paras1[i:i+window_size1])
|
|
|
-
|
|
|
- # 合并文件1中的段落（原始版本，用于差异检测）
|
|
|
- combined_original1 = "".join(original_paras1[i:i+window_size1])
|
|
|
-
|
|
|
- # 查找最佳匹配
|
|
|
- best_match = self._find_best_match_in_paras2_improved(
|
|
|
- combined_normalized1,
|
|
|
- normalized_paras2,
|
|
|
- start_index2,
|
|
|
- last_match_index2,
|
|
|
- used_paras2
|
|
|
- )
|
|
|
-
|
|
|
- if best_match and best_match['similarity'] >= self.paragraph_match_threshold:
|
|
|
- # 更新搜索位置
|
|
|
- matched_indices = best_match['indices']
|
|
|
- last_match_index2 = matched_indices[-1]
|
|
|
- start_index2 = last_match_index2 + 1
|
|
|
-
|
|
|
- # 记录匹配
|
|
|
- for idx in range(i, i + window_size1):
|
|
|
- used_paras1.add(idx)
|
|
|
- for idx in matched_indices:
|
|
|
- used_paras2.add(idx)
|
|
|
-
|
|
|
- # ✅ 获取原始文本（未标准化标点的版本）
|
|
|
- combined_original2 = "".join([original_paras2[idx] for idx in matched_indices])
|
|
|
-
|
|
|
- # ✅ 检查标点差异
|
|
|
- punctuation_diffs = self.check_punctuation_differences(
|
|
|
- combined_original1,
|
|
|
- combined_original2
|
|
|
- )
|
|
|
-
|
|
|
- if punctuation_diffs:
|
|
|
- # 有标点差异
|
|
|
- diff_description = []
|
|
|
- for pdiff in punctuation_diffs:
|
|
|
- diff_description.append(
|
|
|
- f"位置{pdiff['position']}: '{pdiff['char1']}' vs '{pdiff['char2']}' "
|
|
|
- f"(上下文: ...{pdiff['context1']}... vs ...{pdiff['context2']}...)"
|
|
|
- )
|
|
|
-
|
|
|
- differences.append({
|
|
|
- 'type': 'paragraph_punctuation', # ✅ 新类型
|
|
|
- 'position': f'段落{i+1}' + (f'-{i+window_size1}' if window_size1 > 1 else ''),
|
|
|
- 'file1_value': combined_original1,
|
|
|
- 'file2_value': combined_original2,
|
|
|
- 'description': f'段落全角半角标点差异: {"; ".join(diff_description)}',
|
|
|
- 'punctuation_differences': punctuation_diffs,
|
|
|
- 'similarity': 100.0, # 内容完全相同
|
|
|
- 'severity': 'low'
|
|
|
- })
|
|
|
-
|
|
|
- elif best_match['similarity'] < self.content_similarity_threshold:
|
|
|
- # 内容有差异
|
|
|
- severity = 'low' if best_match['similarity'] >= 90 else 'medium'
|
|
|
- differences.append({
|
|
|
- 'type': 'paragraph',
|
|
|
- 'position': f'段落{i+1}' + (f'-{i+window_size1}' if window_size1 > 1 else ''),
|
|
|
- 'file1_value': combined_original1,
|
|
|
- 'file2_value': combined_original2,
|
|
|
- 'description': f'段落内容差异 (相似度: {best_match["similarity"]:.1f}%)',
|
|
|
- 'similarity': best_match['similarity'],
|
|
|
- 'severity': severity
|
|
|
- })
|
|
|
-
|
|
|
- # 如果文件2已全部匹配完，退出
|
|
|
- if len(used_paras2) >= len(normalized_paras2):
|
|
|
- return differences
|
|
|
-
|
|
|
- # 处理未匹配的段落
|
|
|
- for i, para in enumerate(original_paras1):
|
|
|
- if i not in used_paras1:
|
|
|
- differences.append({
|
|
|
- 'type': 'paragraph',
|
|
|
- 'position': f'段落{i+1}',
|
|
|
- 'file1_value': para,
|
|
|
- 'file2_value': "",
|
|
|
- 'description': '文件1中独有的段落',
|
|
|
- 'similarity': 0.0,
|
|
|
- 'severity': 'medium'
|
|
|
- })
|
|
|
-
|
|
|
- for j, para in enumerate(original_paras2):
|
|
|
- if j not in used_paras2:
|
|
|
- differences.append({
|
|
|
- 'type': 'paragraph',
|
|
|
- 'position': f'段落{j+1}',
|
|
|
- 'file1_value': "",
|
|
|
- 'file2_value': para,
|
|
|
- 'description': '文件2中独有的段落',
|
|
|
- 'similarity': 0.0,
|
|
|
- 'severity': 'medium'
|
|
|
- })
|
|
|
-
|
|
|
- return differences
|
|
|
-
|
|
|
-
|
|
|
- def _find_best_match_in_paras2_improved(self, target_text: str, paras2: List[str],
|
|
|
- start_index: int, last_match_index: int,
|
|
|
- used_paras2: set) -> Dict:
|
|
|
- """
|
|
|
- 改进的段落匹配方法 - 借鉴 _find_matching_bbox 的窗口查找逻辑
|
|
|
-
|
|
|
- Args:
|
|
|
- target_text: 目标文本（已标准化）
|
|
|
- paras2: 文件2的段落列表（已标准化）
|
|
|
- start_index: 起始搜索索引（上次匹配后的下一个位置）
|
|
|
- last_match_index: 上次匹配成功的索引
|
|
|
- used_paras2: 已使用的段落索引集合
|
|
|
-
|
|
|
- Returns:
|
|
|
- 最佳匹配结果
|
|
|
- """
|
|
|
- # ✅ 向前查找窗口（类似 _find_matching_bbox）
|
|
|
- search_start = last_match_index - 1
|
|
|
- unused_count = 0
|
|
|
-
|
|
|
- # 向前找到 look_ahead_window 个未使用的段落
|
|
|
- while search_start >= 0:
|
|
|
- if search_start not in used_paras2:
|
|
|
- unused_count += 1
|
|
|
- if unused_count >= self.max_paragraph_window:
|
|
|
- break
|
|
|
- search_start -= 1
|
|
|
-
|
|
|
- if search_start < 0:
|
|
|
- search_start = 0
|
|
|
- # 跳过开头已使用的段落
|
|
|
- while search_start < start_index and search_start in used_paras2:
|
|
|
- search_start += 1
|
|
|
-
|
|
|
- # 搜索范围：从 search_start 到 start_index + window
|
|
|
- search_end = min(start_index + self.max_paragraph_window, len(paras2))
|
|
|
-
|
|
|
- best_match = None
|
|
|
-
|
|
|
- # ✅ 遍历不同窗口大小
|
|
|
- for window_size in range(1, self.max_paragraph_window + 1):
|
|
|
- for j in range(search_start, search_end):
|
|
|
- # ✅ 跳过已使用的段落
|
|
|
- if any(idx in used_paras2 for idx in range(j, min(j + window_size, len(paras2)))):
|
|
|
- continue
|
|
|
-
|
|
|
- # 确保不越界
|
|
|
- if j + window_size > len(paras2):
|
|
|
- break
|
|
|
-
|
|
|
- # 合并段落
|
|
|
- combined_para2 = "".join(paras2[j:j+window_size])
|
|
|
-
|
|
|
- # 计算相似度
|
|
|
- if target_text == combined_para2:
|
|
|
- similarity = 100.0
|
|
|
- else:
|
|
|
- similarity = self.calculate_text_similarity(target_text, combined_para2)
|
|
|
-
|
|
|
- # 更新最佳匹配
|
|
|
- if not best_match or similarity > best_match['similarity']:
|
|
|
- best_match = {
|
|
|
- 'text': combined_para2,
|
|
|
- 'similarity': similarity,
|
|
|
- 'indices': list(range(j, j + window_size))
|
|
|
- }
|
|
|
-
|
|
|
- # ✅ 如果找到完美匹配，提前返回
|
|
|
- if similarity == 100.0:
|
|
|
- return best_match
|
|
|
-
|
|
|
- # 如果没有找到匹配，返回空结果
|
|
|
- if best_match is None:
|
|
|
- return {
|
|
|
- 'text': '',
|
|
|
- 'similarity': 0.0,
|
|
|
- 'indices': []
|
|
|
- }
|
|
|
-
|
|
|
- return best_match
|
|
|
-
|
|
|
- def normalize_header_text(self, text: str) -> str:
|
|
|
- """标准化表头文本"""
|
|
|
- # 移除括号及其内容
|
|
|
- text = re.sub(r'[（(].*?[)）]', '', text)
|
|
|
- # 统一空格
|
|
|
- text = re.sub(r'\s+', '', text)
|
|
|
- # 移除特殊字符
|
|
|
- text = re.sub(r'[^\w\u4e00-\u9fff]', '', text)
|
|
|
- return text.lower().strip()
|
|
|
-
|
|
|
- def compare_table_headers(self, headers1: List[str], headers2: List[str]) -> Dict:
|
|
|
- """比较表格表头"""
|
|
|
- result = {
|
|
|
- 'match': True,
|
|
|
- 'differences': [],
|
|
|
- 'column_mapping': {}, # 列映射关系
|
|
|
- 'similarity_scores': []
|
|
|
- }
|
|
|
-
|
|
|
- if len(headers1) != len(headers2):
|
|
|
- result['match'] = False
|
|
|
- result['differences'].append({
|
|
|
- 'type': 'table_header_critical',
|
|
|
- 'description': f'表头列数不一致: {len(headers1)} vs {len(headers2)}',
|
|
|
- 'severity': 'critical'
|
|
|
- })
|
|
|
- return result
|
|
|
-
|
|
|
- # 逐列比较表头
|
|
|
- for i, (h1, h2) in enumerate(zip(headers1, headers2)):
|
|
|
- norm_h1 = self.normalize_header_text(h1)
|
|
|
- norm_h2 = self.normalize_header_text(h2)
|
|
|
-
|
|
|
- similarity = self.calculate_text_similarity(norm_h1, norm_h2)
|
|
|
- result['similarity_scores'].append({
|
|
|
- 'column_index': i,
|
|
|
- 'header1': h1,
|
|
|
- 'header2': h2,
|
|
|
- 'similarity': similarity
|
|
|
- })
|
|
|
-
|
|
|
- if similarity < self.header_similarity_threshold:
|
|
|
- result['match'] = False
|
|
|
- result['differences'].append({
|
|
|
- 'type': 'table_header_mismatch',
|
|
|
- 'column_index': i,
|
|
|
- 'header1': h1,
|
|
|
- 'header2': h2,
|
|
|
- 'similarity': similarity,
|
|
|
- 'description': f'第{i+1}列表头不匹配: "{h1}" vs "{h2}" (相似度: {similarity:.1f}%)',
|
|
|
- 'severity': 'medium' if similarity < 50 else 'high'
|
|
|
- })
|
|
|
- else:
|
|
|
- result['column_mapping'][i] = i # 建立列映射
|
|
|
-
|
|
|
- return result
|
|
|
-
|
|
|
- def detect_table_header_row(self, table: List[List[str]]) -> int:
|
|
|
- """
|
|
|
- 智能检测表格的表头行索引
|
|
|
-
|
|
|
- 策略：
|
|
|
- 1. 查找包含典型表头关键词的行（如：序号、编号、时间、日期、金额等）
|
|
|
- 2. 检查该行后续行是否为数据行（包含数字、日期等）
|
|
|
- 3. 返回表头行的索引，如果找不到则返回0
|
|
|
- """
|
|
|
- # 常见表头关键词
|
|
|
- header_keywords = [
|
|
|
- # 通用表头
|
|
|
- '序号', '编号', '时间', '日期', '名称', '类型', '金额', '数量', '单价',
|
|
|
- '备注', '说明', '状态', '类别', '方式', '账号', '单号', '订单',
|
|
|
- # 流水表格特定
|
|
|
- '交易单号', '交易时间', '交易类型', '收/支', '支出', '收入',
|
|
|
- '交易方式', '交易对方', '商户单号', '付款方式', '收款方',
|
|
|
- # 英文表头
|
|
|
- 'no', 'id', 'time', 'date', 'name', 'type', 'amount', 'status'
|
|
|
- ]
|
|
|
-
|
|
|
- for row_idx, row in enumerate(table):
|
|
|
- if not row:
|
|
|
- continue
|
|
|
-
|
|
|
- # 计算该行包含表头关键词的单元格数量
|
|
|
- keyword_count = 0
|
|
|
- for cell in row:
|
|
|
- cell_lower = cell.lower().strip()
|
|
|
- for keyword in header_keywords:
|
|
|
- if keyword in cell_lower:
|
|
|
- keyword_count += 1
|
|
|
- break
|
|
|
-
|
|
|
- # 如果超过一半的单元格包含表头关键词，认为是表头行
|
|
|
- if keyword_count >= len(row) * 0.4 and keyword_count >= 2:
|
|
|
- # 验证：检查下一行是否像数据行
|
|
|
- if row_idx + 1 < len(table):
|
|
|
- next_row = table[row_idx + 1]
|
|
|
- if self.is_data_row(next_row):
|
|
|
- print(f" 📍 检测到表头在第 {row_idx + 1} 行")
|
|
|
- return row_idx
|
|
|
-
|
|
|
- # 如果没有找到明确的表头行，返回0（默认第一行）
|
|
|
- print(f" ⚠️ 未检测到明确表头，默认使用第1行")
|
|
|
- return 0
|
|
|
-
|
|
|
- def is_data_row(self, row: List[str]) -> bool:
|
|
|
- """判断是否为数据行（包含数字、日期等）"""
|
|
|
- data_pattern_count = 0
|
|
|
-
|
|
|
- for cell in row:
|
|
|
- if not cell:
|
|
|
- continue
|
|
|
-
|
|
|
- # 检查是否包含数字
|
|
|
- if re.search(r'\d', cell):
|
|
|
- data_pattern_count += 1
|
|
|
-
|
|
|
- # 检查是否为日期时间格式
|
|
|
- if re.search(r'\d{4}[-/年]\d{1,2}[-/月]\d{1,2}', cell):
|
|
|
- data_pattern_count += 1
|
|
|
-
|
|
|
- # 如果超过一半的单元格包含数据特征，认为是数据行
|
|
|
- return data_pattern_count >= len(row) * 0.5
|
|
|
-
|
|
|
- def compare_table_flow_list(self, table1: List[List[str]], table2: List[List[str]]) -> List[Dict]:
|
|
|
- """专门的流水列表表格比较算法 - 支持表头不在第一行"""
|
|
|
- differences = []
|
|
|
-
|
|
|
- if not table1 or not table2:
|
|
|
- return [{
|
|
|
- 'type': 'table_empty',
|
|
|
- 'description': '表格为空',
|
|
|
- 'severity': 'critical'
|
|
|
- }]
|
|
|
-
|
|
|
- print(f"\n📋 开始流水表格对比...")
|
|
|
-
|
|
|
- # 第一步：智能检测表头位置
|
|
|
- header_row_idx1 = self.detect_table_header_row(table1)
|
|
|
- header_row_idx2 = self.detect_table_header_row(table2)
|
|
|
-
|
|
|
- if header_row_idx1 != header_row_idx2:
|
|
|
- differences.append({
|
|
|
- 'type': 'table_header_position',
|
|
|
- 'position': '表头位置',
|
|
|
- 'file1_value': f'第{header_row_idx1 + 1}行',
|
|
|
- 'file2_value': f'第{header_row_idx2 + 1}行',
|
|
|
- 'description': f'表头位置不一致: 文件1在第{header_row_idx1 + 1}行，文件2在第{header_row_idx2 + 1}行',
|
|
|
- 'severity': 'high'
|
|
|
- })
|
|
|
-
|
|
|
- # 第二步：比对表头前的内容（按单元格比对）
|
|
|
- if header_row_idx1 > 0 or header_row_idx2 > 0:
|
|
|
- print(f"\n📝 对比表头前的内容...")
|
|
|
-
|
|
|
- # 提取表头前的内容作为单独的"表格"
|
|
|
- pre_header_table1 = table1[:header_row_idx1] if header_row_idx1 > 0 else []
|
|
|
- pre_header_table2 = table2[:header_row_idx2] if header_row_idx2 > 0 else []
|
|
|
-
|
|
|
- if pre_header_table1 or pre_header_table2:
|
|
|
- # 复用compare_tables方法进行比对
|
|
|
- pre_header_diffs = self.compare_tables(pre_header_table1, pre_header_table2)
|
|
|
-
|
|
|
- # 修改：统一类型为 table_pre_header
|
|
|
- for diff in pre_header_diffs:
|
|
|
- diff['type'] = 'table_pre_header'
|
|
|
- diff['position'] = f"表头前{diff['position']}"
|
|
|
- diff['severity'] = 'medium'
|
|
|
- print(f" ⚠️ {diff['position']}: {diff['description']}")
|
|
|
-
|
|
|
- differences.extend(pre_header_diffs)
|
|
|
-
|
|
|
- # 第三步：比较表头
|
|
|
- headers1 = table1[header_row_idx1]
|
|
|
- headers2 = table2[header_row_idx2]
|
|
|
-
|
|
|
- print(f"\n📋 对比表头...")
|
|
|
- print(f" 文件1表头 (第{header_row_idx1 + 1}行): {headers1}")
|
|
|
- print(f" 文件2表头 (第{header_row_idx2 + 1}行): {headers2}")
|
|
|
-
|
|
|
- header_result = self.compare_table_headers(headers1, headers2)
|
|
|
-
|
|
|
- # ✅ 新增：检查列数是否一致
|
|
|
- column_count_match = len(headers1) == len(headers2)
|
|
|
- if not header_result['match']:
|
|
|
- print(f"\n⚠️ 表头文字存在差异")
|
|
|
- for diff in header_result['differences']:
|
|
|
- print(f" - {diff['description']}")
|
|
|
- differences.append({
|
|
|
- 'type': diff.get('type', 'table_header_mismatch'), # ✅ 改为 mismatch 而非 critical
|
|
|
- 'position': '表头',
|
|
|
- 'file1_value': diff.get('header1', ''),
|
|
|
- 'file2_value': diff.get('header2', ''),
|
|
|
- 'description': diff['description'],
|
|
|
- 'severity': diff.get('severity', 'high'),
|
|
|
- })
|
|
|
- if diff.get('severity', 'high') == 'critical':
|
|
|
- return differences
|
|
|
- else:
|
|
|
- print(f"✅ 表头匹配成功")
|
|
|
-
|
|
|
- # 第四步：检测列类型
|
|
|
- column_types1 = []
|
|
|
- column_types2 = []
|
|
|
-
|
|
|
- # 检测文件1的列类型
|
|
|
- for col_idx in range(len(headers1)):
|
|
|
- col_values1 = [
|
|
|
- row[col_idx]
|
|
|
- for row in table1[header_row_idx1 + 1:]
|
|
|
- if col_idx < len(row)
|
|
|
- ]
|
|
|
- col_type = self.detect_column_type(col_values1)
|
|
|
- column_types1.append(col_type)
|
|
|
- print(f" 文件1列 {col_idx + 1} ({headers1[col_idx]}): {col_type}")
|
|
|
-
|
|
|
- # 检测文件2的列类型
|
|
|
- for col_idx in range(len(headers2)):
|
|
|
- col_values2 = [
|
|
|
- row[col_idx]
|
|
|
- for row in table2[header_row_idx2 + 1:]
|
|
|
- if col_idx < len(row)
|
|
|
- ]
|
|
|
- col_type = self.detect_column_type(col_values2)
|
|
|
- column_types2.append(col_type)
|
|
|
- print(f" 文件2列 {col_idx + 1} ({headers2[col_idx]}): {col_type}")
|
|
|
-
|
|
|
- # ✅ 改进：统计列类型差异，只有超过阈值才停止比较
|
|
|
- mismatched_columns = []
|
|
|
- for col_idx in range(min(len(column_types1), len(column_types2))):
|
|
|
- if column_types1[col_idx] != column_types2[col_idx]:
|
|
|
- mismatched_columns.append(col_idx)
|
|
|
- differences.append({
|
|
|
- 'type': 'table_column_type_mismatch', # ✅ 新类型，区别于 critical
|
|
|
- 'position': f'第{col_idx + 1}列',
|
|
|
- 'file1_value': f'{headers1[col_idx]} ({column_types1[col_idx]})',
|
|
|
- 'file2_value': f'{headers2[col_idx]} ({column_types2[col_idx]})',
|
|
|
- 'description': f'列类型不一致: {column_types1[col_idx]} vs {column_types2[col_idx]}',
|
|
|
- 'severity': 'high',
|
|
|
- 'column_index': col_idx
|
|
|
- })
|
|
|
-
|
|
|
- # ✅ 计算列类型差异比例
|
|
|
- total_columns = min(len(column_types1), len(column_types2))
|
|
|
- mismatch_ratio = len(mismatched_columns) / total_columns if total_columns > 0 else 0
|
|
|
-
|
|
|
- # ✅ 只有当差异比例超过50%时才停止比较
|
|
|
- if mismatch_ratio > 0.5:
|
|
|
- print(f"\n⚠️ 列类型差异过大 ({len(mismatched_columns)}/{total_columns} = {mismatch_ratio:.1%})，不再比较单元格内容...")
|
|
|
- # 添加一个汇总差异
|
|
|
- differences.append({
|
|
|
- 'type': 'table_header_critical',
|
|
|
- 'position': '表格列类型',
|
|
|
- 'file1_value': f'{len(mismatched_columns)}列类型不一致',
|
|
|
- 'file2_value': f'共{total_columns}列',
|
|
|
- 'description': f'列类型差异过大: {len(mismatched_columns)}/{total_columns}列不匹配 ({mismatch_ratio:.1%})',
|
|
|
- 'severity': 'critical'
|
|
|
- })
|
|
|
- return differences
|
|
|
- elif mismatched_columns:
|
|
|
- print(f"\n⚠️ 检测到 {len(mismatched_columns)} 列类型差异，但仍继续比较单元格...")
|
|
|
- print(f" 不匹配的列: {[col_idx + 1 for col_idx in mismatched_columns]}")
|
|
|
-
|
|
|
- # ✅ 为每列选择更合适的类型（优先使用数据更丰富的文件）
|
|
|
- column_types = []
|
|
|
- for col_idx in range(max(len(column_types1), len(column_types2))):
|
|
|
- if col_idx >= len(column_types1):
|
|
|
- column_types.append(column_types2[col_idx])
|
|
|
- elif col_idx >= len(column_types2):
|
|
|
- column_types.append(column_types1[col_idx])
|
|
|
- elif col_idx in mismatched_columns:
|
|
|
- # ✅ 对于类型不一致的列，选择更通用的类型
|
|
|
- type1 = column_types1[col_idx]
|
|
|
- type2 = column_types2[col_idx]
|
|
|
-
|
|
|
- # 类型优先级: text > text_number > numeric/datetime
|
|
|
- if type1 == 'text' or type2 == 'text':
|
|
|
- column_types.append('text')
|
|
|
- elif type1 == 'text_number' or type2 == 'text_number':
|
|
|
- column_types.append('text_number')
|
|
|
- else:
|
|
|
- # 默认使用文件1的类型
|
|
|
- column_types.append(type1)
|
|
|
-
|
|
|
- print(f" 📝 第{col_idx + 1}列类型冲突，使用通用类型: {column_types[-1]}")
|
|
|
- else:
|
|
|
- column_types.append(column_types1[col_idx])
|
|
|
-
|
|
|
- # 第五步：逐行比较数据
|
|
|
- data_rows1 = table1[header_row_idx1 + 1:]
|
|
|
- data_rows2 = table2[header_row_idx2 + 1:]
|
|
|
-
|
|
|
- max_rows = max(len(data_rows1), len(data_rows2))
|
|
|
-
|
|
|
- print(f"\n📊 开始逐行对比数据 (共{max_rows}行)...")
|
|
|
-
|
|
|
- for row_idx in range(max_rows):
|
|
|
- row1 = data_rows1[row_idx] if row_idx < len(data_rows1) else []
|
|
|
- row2 = data_rows2[row_idx] if row_idx < len(data_rows2) else []
|
|
|
-
|
|
|
- # 实际行号（加上表头行索引）
|
|
|
- actual_row_num = header_row_idx1 + row_idx + 2
|
|
|
-
|
|
|
- if not row1:
|
|
|
- differences.append({
|
|
|
- 'type': 'table_row_missing',
|
|
|
- 'position': f'第{actual_row_num}行',
|
|
|
- 'file1_value': '',
|
|
|
- 'file2_value': ', '.join(row2),
|
|
|
- 'description': f'文件1缺少第{actual_row_num}行',
|
|
|
- 'severity': 'high',
|
|
|
- 'row_index': actual_row_num
|
|
|
- })
|
|
|
- continue
|
|
|
-
|
|
|
- if not row2:
|
|
|
- # ✅ 修改：整行缺失按单元格输出
|
|
|
- differences.append({
|
|
|
- 'type': 'table_row_missing',
|
|
|
- 'position': f'第{actual_row_num}行',
|
|
|
- 'file1_value': ', '.join(row1),
|
|
|
- 'file2_value': '',
|
|
|
- 'description': f'文件2缺少第{actual_row_num}行',
|
|
|
- 'severity': 'high',
|
|
|
- 'row_index': actual_row_num
|
|
|
- })
|
|
|
- continue
|
|
|
-
|
|
|
- # 逐列比较，每个单元格差异独立输出
|
|
|
- max_cols = max(len(row1), len(row2))
|
|
|
-
|
|
|
- for col_idx in range(max_cols):
|
|
|
- cell1 = row1[col_idx] if col_idx < len(row1) else ''
|
|
|
- cell2 = row2[col_idx] if col_idx < len(row2) else ''
|
|
|
-
|
|
|
- # 跳过图片内容
|
|
|
- if "[图片内容-忽略]" in cell1 or "[图片内容-忽略]" in cell2:
|
|
|
- continue
|
|
|
-
|
|
|
- # ✅ 使用合并后的列类型
|
|
|
- column_type = column_types[col_idx] if col_idx < len(column_types) else 'text'
|
|
|
-
|
|
|
- # ✅ 获取列名
|
|
|
- if header_result['match']:
|
|
|
- column_name = headers1[col_idx] if col_idx < len(headers1) else f'列{col_idx + 1}'
|
|
|
- else:
|
|
|
- col_name1 = headers1[col_idx] if col_idx < len(headers1) else f'列{col_idx + 1}'
|
|
|
- col_name2 = headers2[col_idx] if col_idx < len(headers2) else f'列{col_idx + 1}'
|
|
|
- column_name = f"{col_name1}/{col_name2}"
|
|
|
-
|
|
|
- # ✅ 如果该列类型不匹配，在描述中标注
|
|
|
- type_mismatch_note = ""
|
|
|
- if col_idx in mismatched_columns:
|
|
|
- type_mismatch_note = f" [列类型冲突: {column_types1[col_idx]} vs {column_types2[col_idx]}]"
|
|
|
-
|
|
|
- compare_result = self.compare_cell_value(cell1, cell2, column_type, column_name)
|
|
|
-
|
|
|
- if not compare_result['match']:
|
|
|
- # ✅ 直接将单元格差异添加到differences列表
|
|
|
- diff_info = compare_result['difference']
|
|
|
-
|
|
|
- differences.append({
|
|
|
- 'type': diff_info['type'], # 使用原始类型（table_amount, table_text等）
|
|
|
- 'position': f'第{actual_row_num}行第{col_idx + 1}列',
|
|
|
- 'file1_value': diff_info['value1'],
|
|
|
- 'file2_value': diff_info['value2'],
|
|
|
- 'description': diff_info['description'] + type_mismatch_note, # ✅ 添加类型冲突标注
|
|
|
- 'severity': 'high' if col_idx in mismatched_columns else 'medium', # ✅ 类型冲突的单元格提高严重度
|
|
|
- 'row_index': actual_row_num,
|
|
|
- 'col_index': col_idx,
|
|
|
- 'column_name': column_name,
|
|
|
- 'column_type': column_type,
|
|
|
- 'column_type_mismatch': col_idx in mismatched_columns, # ✅ 新增字段
|
|
|
- **{k: v for k, v in diff_info.items() if k not in ['type', 'value1', 'value2', 'description']}
|
|
|
- })
|
|
|
-
|
|
|
- print(f" ⚠️ 第{actual_row_num}行第{col_idx + 1}列({column_name}): {diff_info['description']}{type_mismatch_note}")
|
|
|
-
|
|
|
- print(f"\n✅ 流水表格对比完成，发现 {len(differences)} 个差异")
|
|
|
-
|
|
|
- return differences
|
|
|
-
|
|
|
- def compare_tables_with_mode(self, table1: List[List[str]], table2: List[List[str]],
|
|
|
- mode: str = 'standard') -> List[Dict]:
|
|
|
- """根据模式选择表格比较算法"""
|
|
|
- if mode == 'flow_list':
|
|
|
- return self.compare_table_flow_list(table1, table2)
|
|
|
- else:
|
|
|
- return self.compare_tables(table1, table2)
|
|
|
-
|
|
|
- def compare_files(self, file1_path: str, file2_path: str) -> Dict:
|
|
|
- """改进的文件比较方法 - 支持不同的表格比较模式"""
|
|
|
- # 读取文件
|
|
|
- with open(file1_path, 'r', encoding='utf-8') as f:
|
|
|
- content1 = f.read()
|
|
|
-
|
|
|
- with open(file2_path, 'r', encoding='utf-8') as f:
|
|
|
- content2 = f.read()
|
|
|
-
|
|
|
- # 提取表格和段落
|
|
|
- tables1 = self.extract_table_data(content1)
|
|
|
- tables2 = self.extract_table_data(content2)
|
|
|
-
|
|
|
- paras1 = self.extract_paragraphs(content1)
|
|
|
- paras2 = self.extract_paragraphs(content2)
|
|
|
-
|
|
|
- # 比较结果
|
|
|
- all_differences = []
|
|
|
-
|
|
|
- # 比较表格 - 使用指定的比较模式
|
|
|
- if tables1 and tables2:
|
|
|
- table_diffs = self.compare_tables_with_mode(
|
|
|
- tables1[0], tables2[0],
|
|
|
- mode=self.table_comparison_mode
|
|
|
- )
|
|
|
- all_differences.extend(table_diffs)
|
|
|
- elif tables1 and not tables2:
|
|
|
- all_differences.append({
|
|
|
- 'type': 'table_structure',
|
|
|
- 'position': '表格结构',
|
|
|
- 'file1_value': f'包含{len(tables1)}个表格',
|
|
|
- 'file2_value': '无表格',
|
|
|
- 'description': '文件1包含表格但文件2无表格',
|
|
|
- 'severity': 'high'
|
|
|
- })
|
|
|
- elif not tables1 and tables2:
|
|
|
- all_differences.append({
|
|
|
- 'type': 'table_structure',
|
|
|
- 'position': '表格结构',
|
|
|
- 'file1_value': '无表格',
|
|
|
- 'file2_value': f'包含{len(tables2)}个表格',
|
|
|
- 'description': '文件2包含表格但文件1无表格',
|
|
|
- 'severity': 'high'
|
|
|
- })
|
|
|
-
|
|
|
- # 使用增强的段落比较
|
|
|
- para_diffs = self.compare_paragraphs_with_flexible_matching(paras1, paras2)
|
|
|
- all_differences.extend(para_diffs)
|
|
|
-
|
|
|
- # ✅ 改进统计信息 - 细化分类
|
|
|
- stats = {
|
|
|
- 'total_differences': len(all_differences),
|
|
|
- 'table_differences': len([d for d in all_differences if d['type'].startswith('table')]),
|
|
|
- 'paragraph_differences': len([d for d in all_differences if d['type'] == 'paragraph']),
|
|
|
- 'amount_differences': len([d for d in all_differences if d['type'] == 'table_amount']),
|
|
|
- 'datetime_differences': len([d for d in all_differences if d['type'] == 'table_datetime']),
|
|
|
- 'text_differences': len([d for d in all_differences if d['type'] == 'table_text']),
|
|
|
- 'table_pre_header': len([d for d in all_differences if d['type'] == 'table_pre_header']),
|
|
|
- 'table_header_mismatch': len([d for d in all_differences if d['type'] == 'table_header_mismatch']), # ✅ 新增
|
|
|
- 'table_header_critical': len([d for d in all_differences if d['type'] == 'table_header_critical']), # ✅ 新增
|
|
|
- 'table_header_position': len([d for d in all_differences if d['type'] == 'table_header_position']),
|
|
|
- 'table_row_missing': len([d for d in all_differences if d['type'] == 'table_row_missing']),
|
|
|
- 'high_severity': len([d for d in all_differences if d.get('severity') == 'critical' or d.get('severity') == 'high']),
|
|
|
- 'medium_severity': len([d for d in all_differences if d.get('severity') == 'medium']),
|
|
|
- 'low_severity': len([d for d in all_differences if d.get('severity') == 'low'])
|
|
|
- }
|
|
|
-
|
|
|
- result = {
|
|
|
- 'differences': all_differences,
|
|
|
- 'statistics': stats,
|
|
|
- 'file1_tables': len(tables1),
|
|
|
- 'file2_tables': len(tables2),
|
|
|
- 'file1_paragraphs': len(paras1),
|
|
|
- 'file2_paragraphs': len(paras2),
|
|
|
- 'file1_path': file1_path,
|
|
|
- 'file2_path': file2_path,
|
|
|
- }
|
|
|
-
|
|
|
- return result
|
|
|
-
|
|
|
- def generate_json_report(self, comparison_result: Dict, output_file: str):
|
|
|
- """生成JSON格式的比较报告"""
|
|
|
- # report_data = {
|
|
|
- # 'comparison_summary': {
|
|
|
- # 'timestamp': re.sub(r'[^\w\-_\.]', '_', str(comparison_result.get('timestamp', ''))),
|
|
|
- # 'file1': comparison_result['file1_path'],
|
|
|
- # 'file2': comparison_result['file2_path'],
|
|
|
- # 'statistics': comparison_result['statistics'],
|
|
|
- # 'file_info': {
|
|
|
- # 'file1_tables': comparison_result['file1_tables'],
|
|
|
- # 'file2_tables': comparison_result['file2_tables'],
|
|
|
- # 'file1_paragraphs': comparison_result['file1_paragraphs'],
|
|
|
- # 'file2_paragraphs': comparison_result['file2_paragraphs']
|
|
|
- # }
|
|
|
- # },
|
|
|
- # 'differences': comparison_result['differences']
|
|
|
- # }
|
|
|
-
|
|
|
- with open(output_file, 'w', encoding='utf-8') as f:
|
|
|
- json.dump(comparison_result, f, ensure_ascii=False, indent=2)
|
|
|
-
|
|
|
- def generate_markdown_report(self, comparison_result: Dict, output_file: str):
|
|
|
- """生成Markdown格式的比较报告 - 修复类型映射"""
|
|
|
- with open(output_file, 'w', encoding='utf-8') as f:
|
|
|
- f.write("# OCR结果对比报告\n\n")
|
|
|
-
|
|
|
- # 基本信息
|
|
|
- f.write("## 基本信息\n\n")
|
|
|
- f.write(f"- **文件1**: `{comparison_result['file1_path']}`\n")
|
|
|
- f.write(f"- **文件2**: `{comparison_result['file2_path']}`\n")
|
|
|
- f.write(f"- **比较时间**: {comparison_result.get('timestamp', 'N/A')}\n\n")
|
|
|
-
|
|
|
- # 统计信息
|
|
|
- stats = comparison_result['statistics']
|
|
|
- f.write("## 统计信息\n\n")
|
|
|
- f.write(f"- 总差异数量: **{stats['total_differences']}**\n")
|
|
|
- f.write(f"- 表格差异: **{stats['table_differences']}**\n")
|
|
|
- f.write(f"- 其中表格金额差异: **{stats['amount_differences']}**\n")
|
|
|
- f.write(f"- 段落差异: **{stats['paragraph_differences']}**\n")
|
|
|
- f.write(f"- 高严重度: **{stats['high_severity']}**\n") # ✅ 新增
|
|
|
- f.write(f"- 中严重度: **{stats['medium_severity']}**\n") # ✅ 新增
|
|
|
- f.write(f"- 低严重度: **{stats['low_severity']}**\n") # ✅ 新增
|
|
|
- f.write(f"- 文件1表格数: {comparison_result['file1_tables']}\n")
|
|
|
- f.write(f"- 文件2表格数: {comparison_result['file2_tables']}\n")
|
|
|
- f.write(f"- 文件1段落数: {comparison_result['file1_paragraphs']}\n")
|
|
|
- f.write(f"- 文件2段落数: {comparison_result['file2_paragraphs']}\n\n")
|
|
|
-
|
|
|
- # 差异摘要
|
|
|
- if stats['total_differences'] == 0:
|
|
|
- f.write("## 结论\n\n")
|
|
|
- f.write("🎉 **完美匹配！没有发现任何差异。**\n\n")
|
|
|
- else:
|
|
|
- f.write("## 差异摘要\n\n")
|
|
|
-
|
|
|
- # ✅ 更新类型映射
|
|
|
- type_name_map = {
|
|
|
- 'table_amount': '💰 表格金额差异',
|
|
|
- 'table_text': '📝 表格文本差异',
|
|
|
- 'table_pre_header': '📋 表头前内容差异',
|
|
|
- 'table_header_position': '📍 表头位置差异',
|
|
|
- 'table_header_critical': '❌ 表头严重错误',
|
|
|
- 'table_row_missing': '🚫 表格行缺失',
|
|
|
- 'table_row_data': '📊 表格数据差异',
|
|
|
- 'table_structure': '🏗️ 表格结构差异',
|
|
|
- 'paragraph': '📄 段落差异'
|
|
|
- }
|
|
|
-
|
|
|
- # 按类型分组显示差异
|
|
|
- diff_by_type = {}
|
|
|
- for diff in comparison_result['differences']:
|
|
|
- diff_type = diff['type']
|
|
|
- if diff_type not in diff_by_type:
|
|
|
- diff_by_type[diff_type] = []
|
|
|
- diff_by_type[diff_type].append(diff)
|
|
|
-
|
|
|
- for diff_type, diffs in diff_by_type.items():
|
|
|
- type_name = type_name_map.get(diff_type, f'❓ {diff_type}')
|
|
|
-
|
|
|
- f.write(f"### {type_name} ({len(diffs)}个)\n\n")
|
|
|
-
|
|
|
- for i, diff in enumerate(diffs, 1):
|
|
|
- f.write(f"**{i}. {diff['position']}**\n")
|
|
|
- f.write(f"- 文件1: `{diff['file1_value']}`\n")
|
|
|
- f.write(f"- 文件2: `{diff['file2_value']}`\n")
|
|
|
- f.write(f"- 说明: {diff['description']}\n")
|
|
|
- if 'severity' in diff:
|
|
|
- severity_icon = {'critical': '🔴', 'high': '🟠', 'medium': '🟡', 'low': '🟢'}
|
|
|
- f.write(f"- 严重度: {severity_icon.get(diff['severity'], '⚪')} {diff['severity']}\n")
|
|
|
- f.write("\n")
|
|
|
-
|
|
|
- # 详细差异列表
|
|
|
- if comparison_result['differences']:
|
|
|
- f.write("## 详细差异列表\n\n")
|
|
|
- f.write("| 序号 | 类型 | 位置 | 文件1内容 | 文件2内容 | 描述 | 严重度 |\n")
|
|
|
- f.write("| --- | --- | --- | --- | --- | --- | --- |\n")
|
|
|
-
|
|
|
- for i, diff in enumerate(comparison_result['differences'], 1):
|
|
|
- severity = diff.get('severity', 'N/A')
|
|
|
- f.write(f"| {i} | {diff['type']} | {diff['position']} | ")
|
|
|
- f.write(f"`{diff['file1_value'][:50]}{'...' if len(diff['file1_value']) > 50 else ''}` | ")
|
|
|
- f.write(f"`{diff['file2_value'][:50]}{'...' if len(diff['file2_value']) > 50 else ''}` | ")
|
|
|
- f.write(f"{diff['description']} | {severity} |\n")
|
|
|
-
|
|
|
-def compare_ocr_results(file1_path: str, file2_path: str, output_file: str = "comparison_report",
|
|
|
- output_format: str = "markdown", ignore_images: bool = True,
|
|
|
- table_mode: str = 'standard', similarity_algorithm: str = 'ratio') -> Dict:
|
|
|
- """
|
|
|
- 比较两个OCR结果文件
|
|
|
-
|
|
|
- Args:
|
|
|
- file1_path: 第一个OCR结果文件路径
|
|
|
- file2_path: 第二个OCR结果文件路径
|
|
|
- output_file: 输出文件名（不含扩展名）
|
|
|
- output_format: 输出格式 ('json', 'markdown', 'both')
|
|
|
- ignore_images: 是否忽略图片内容
|
|
|
- table_mode: 表格比较模式 ('standard', 'flow_list')
|
|
|
- similarity_algorithm: 相似度算法 ('ratio', 'partial_ratio', 'token_sort_ratio', 'token_set_ratio')
|
|
|
- """
|
|
|
- comparator = OCRResultComparator()
|
|
|
- comparator.table_comparison_mode = table_mode
|
|
|
-
|
|
|
- # 根据参数选择相似度算法
|
|
|
- if similarity_algorithm == 'partial_ratio':
|
|
|
- comparator.calculate_text_similarity = lambda t1, t2: fuzz.partial_ratio(t1, t2)
|
|
|
- elif similarity_algorithm == 'token_sort_ratio':
|
|
|
- comparator.calculate_text_similarity = lambda t1, t2: fuzz.token_sort_ratio(t1, t2)
|
|
|
- elif similarity_algorithm == 'token_set_ratio':
|
|
|
- comparator.calculate_text_similarity = lambda t1, t2: fuzz.token_set_ratio(t1, t2)
|
|
|
-
|
|
|
- print("🔍 开始对比OCR结果...")
|
|
|
- print(f"📄 文件1: {file1_path}")
|
|
|
- print(f"📄 文件2: {file2_path}")
|
|
|
- print(f"📊 表格模式: {table_mode}")
|
|
|
- print(f"🔧 相似度算法: {similarity_algorithm}")
|
|
|
-
|
|
|
- try:
|
|
|
- # 执行比较
|
|
|
- result = comparator.compare_files(file1_path, file2_path)
|
|
|
-
|
|
|
- # 添加时间戳
|
|
|
- import datetime
|
|
|
- result['timestamp'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
|
-
|
|
|
- # 生成报告
|
|
|
- if output_format in ['json', 'both']:
|
|
|
- json_file = f"{output_file}.json"
|
|
|
- comparator.generate_json_report(result, json_file)
|
|
|
- print(f"📄 JSON报告已保存至: {json_file}")
|
|
|
-
|
|
|
- if output_format in ['markdown', 'both']:
|
|
|
- md_file = f"{output_file}.md"
|
|
|
- comparator.generate_markdown_report(result, md_file)
|
|
|
- print(f"📝 Markdown报告已保存至: {md_file}")
|
|
|
-
|
|
|
- # 打印简要结果
|
|
|
- print(f"\n📊 对比完成！")
|
|
|
- print(f" 总差异数: {result['statistics']['total_differences']}")
|
|
|
- print(f" 表格差异: {result['statistics']['table_differences']}")
|
|
|
- print(f" 其中表格金额差异: {result['statistics']['amount_differences']}")
|
|
|
- print(f" 段落差异: {result['statistics']['paragraph_differences']}")
|
|
|
-
|
|
|
- # 打印前几个重要差异
|
|
|
- if result['differences']:
|
|
|
- print(f"\n🔍 前3个重要差异:")
|
|
|
- for i, diff in enumerate(result['differences'][:3], 1):
|
|
|
- print(f" {i}. {diff['position']}: {diff['description']}")
|
|
|
- print(f" 文件1: '{diff['file1_value'][:50]}{'...' if len(diff['file1_value']) > 50 else ''}'")
|
|
|
- print(f" 文件2: '{diff['file2_value'][:50]}{'...' if len(diff['file2_value']) > 50 else ''}'")
|
|
|
- else:
|
|
|
- print(f"\n🎉 恭喜！两个文件内容完全一致！")
|
|
|
-
|
|
|
- # 添加处理统计信息（模仿 ocr_by_vlm.py 的风格）
|
|
|
- print("\n📊 对比处理统计")
|
|
|
- print(f" 文件1路径: {result['file1_path']}")
|
|
|
- print(f" 文件2路径: {result['file2_path']}")
|
|
|
- print(f" 输出文件: {output_file}")
|
|
|
- print(f" 输出格式: {output_format}")
|
|
|
- print(f" 忽略图片: {ignore_images}")
|
|
|
- print(f" 处理时间: {result['timestamp']}")
|
|
|
- print(f" 文件1表格数: {result['file1_tables']}")
|
|
|
- print(f" 文件2表格数: {result['file2_tables']}")
|
|
|
- print(f" 文件1段落数: {result['file1_paragraphs']}")
|
|
|
- print(f" 文件2段落数: {result['file2_paragraphs']}")
|
|
|
-
|
|
|
- return result
|
|
|
-
|
|
|
- except Exception as e:
|
|
|
- import traceback
|
|
|
- traceback.print_exc()
|
|
|
- raise Exception(f"OCR对比任务失败: {e}")
|
|
|
-
|
|
|
-if __name__ == "__main__":
|
|
|
- parser = argparse.ArgumentParser(description='OCR结果对比工具')
|
|
|
- parser.add_argument('file1', nargs='?', help='第一个OCR结果文件路径')
|
|
|
- parser.add_argument('file2', nargs='?', help='第二个OCR结果文件路径')
|
|
|
- parser.add_argument('-o', '--output', default='comparison_report', help='输出文件名')
|
|
|
- parser.add_argument('-f', '--format', choices=['json', 'markdown', 'both'],
|
|
|
- default='markdown', help='输出格式')
|
|
|
- parser.add_argument('--ignore-images', action='store_true', help='忽略图片内容')
|
|
|
- parser.add_argument('--table-mode', choices=['standard', 'flow_list'],
|
|
|
- default='standard', help='表格比较模式')
|
|
|
- parser.add_argument('--similarity-algorithm',
|
|
|
- choices=['ratio', 'partial_ratio', 'token_sort_ratio', 'token_set_ratio'],
|
|
|
- default='ratio', help='相似度算法')
|
|
|
-
|
|
|
- args = parser.parse_args()
|
|
|
-
|
|
|
- if args.file1 and args.file2:
|
|
|
- result = compare_ocr_results(
|
|
|
- file1_path=args.file1,
|
|
|
- file2_path=args.file2,
|
|
|
- output_file=args.output,
|
|
|
- output_format=args.format,
|
|
|
- ignore_images=args.ignore_images,
|
|
|
- table_mode=args.table_mode,
|
|
|
- similarity_algorithm=args.similarity_algorithm
|
|
|
- )
|
|
|
- else:
|
|
|
- # 测试流水表格对比
|
|
|
- result = compare_ocr_results(
|
|
|
- file1_path='/Users/zhch158/workspace/data/流水分析/对公_招商银行图/PaddleOCR_VL_Results/对公_招商银行图_page_006.md',
|
|
|
- file2_path='/Users/zhch158/workspace/data/流水分析/对公_招商银行图/mineru-vlm-2.5.3_Results_cell_bbox/对公_招商银行图_page_006.md',
|
|
|
- output_file=f'/Users/zhch158/workspace/repository.git/ocr_verify/output/flow_list_comparison_{time.strftime("%Y%m%d_%H%M%S")}.1',
|
|
|
- output_format='both',
|
|
|
- ignore_images=True,
|
|
|
- table_mode='flow_list', # 使用流水表格模式
|
|
|
- similarity_algorithm='ratio'
|
|
|
- )
|
zhch158_admin