|
|
@@ -0,0 +1,456 @@
|
|
|
+import json
|
|
|
+import pandas as pd
|
|
|
+import numpy as np
|
|
|
+from nltk.translate.bleu_score import sentence_bleu
|
|
|
+import argparse
|
|
|
+from sklearn.metrics import classification_report
|
|
|
+from collections import Counter
|
|
|
+from sklearn import metrics
|
|
|
+from pandas import isnull
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+def indicator_cal(json_standard,json_test):
|
|
|
+
|
|
|
+ json_standard = pd.DataFrame(json_standard)
|
|
|
+ json_test = pd.DataFrame(json_test)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ '''数据集总体指标'''
|
|
|
+
|
|
|
+ a=json_test[['id','mid_json']]
|
|
|
+ b=json_standard[['id','mid_json','pass_label']]
|
|
|
+
|
|
|
+ a=a.drop_duplicates(subset='id',keep='first')
|
|
|
+ a.index=range(len(a))
|
|
|
+ b=b.drop_duplicates(subset='id',keep='first')
|
|
|
+ b.index=range(len(b))
|
|
|
+
|
|
|
+ outer_merge=pd.merge(a,b,on='id',how='outer')
|
|
|
+ outer_merge.columns=['id','standard_mid_json','test_mid_json','pass_label']
|
|
|
+ standard_exist=outer_merge.standard_mid_json.apply(lambda x: not isnull(x))
|
|
|
+ test_exist=outer_merge.test_mid_json.apply(lambda x: not isnull(x))
|
|
|
+
|
|
|
+ overall_report = {}
|
|
|
+ overall_report['accuracy']=metrics.accuracy_score(standard_exist,test_exist)
|
|
|
+ overall_report['precision']=metrics.precision_score(standard_exist,test_exist)
|
|
|
+ overall_report['recall']=metrics.recall_score(standard_exist,test_exist)
|
|
|
+ overall_report['f1_score']=metrics.f1_score(standard_exist,test_exist)
|
|
|
+
|
|
|
+
|
|
|
+ inner_merge=pd.merge(a,b,on='id',how='inner')
|
|
|
+ inner_merge.columns=['id','standard_mid_json','test_mid_json','pass_label']
|
|
|
+ json_standard = inner_merge['standard_mid_json']#check一下是否对齐
|
|
|
+ json_test = inner_merge['test_mid_json']
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ '''批量读取中间生成的json文件'''
|
|
|
+ test_inline_equations=[]
|
|
|
+ test_interline_equations=[]
|
|
|
+ test_inline_euqations_bboxs=[]
|
|
|
+ test_interline_equations_bboxs=[]
|
|
|
+ test_dropped_text_bboxes=[]
|
|
|
+ test_dropped_text_tag=[]
|
|
|
+ test_dropped_image_bboxes=[]
|
|
|
+ test_dropped_table_bboxes=[]
|
|
|
+ test_preproc_num=[]#阅读顺序
|
|
|
+ test_para_num=[]
|
|
|
+ test_para_text=[]
|
|
|
+
|
|
|
+ for i in json_test:
|
|
|
+ mid_json=pd.DataFrame(i)
|
|
|
+ mid_json=mid_json.iloc[:,:-1]
|
|
|
+ for j1 in mid_json.loc['inline_equations',:]:
|
|
|
+ page_in_text=[]
|
|
|
+ page_in_bbox=[]
|
|
|
+ for k1 in j1:
|
|
|
+ page_in_text.append(k1['latex_text'])
|
|
|
+ page_in_bbox.append(k1['bbox'])
|
|
|
+ test_inline_equations.append(page_in_text)
|
|
|
+ test_inline_euqations_bboxs.append(page_in_bbox)
|
|
|
+ for j2 in mid_json.loc['interline_equations',:]:
|
|
|
+ page_in_text=[]
|
|
|
+ page_in_bbox=[]
|
|
|
+ for k2 in j2:
|
|
|
+ page_in_text.append(k2['latex_text'])
|
|
|
+ page_in_bbox.append(k2['bbox'])
|
|
|
+ test_interline_equations.append(page_in_text)
|
|
|
+ test_interline_equations_bboxs.append(page_in_bbox)
|
|
|
+
|
|
|
+ for j3 in mid_json.loc['droped_text_block',:]:
|
|
|
+ page_in_bbox=[]
|
|
|
+ page_in_tag=[]
|
|
|
+ for k3 in j3:
|
|
|
+ page_in_bbox.append(k3['bbox'])
|
|
|
+ #如果k3中存在tag这个key
|
|
|
+ if 'tag' in k3.keys():
|
|
|
+ page_in_tag.append(k3['tag'])
|
|
|
+ else:
|
|
|
+ page_in_tag.append('None')
|
|
|
+ test_dropped_text_tag.append(page_in_tag)
|
|
|
+ test_dropped_text_bboxes.append(page_in_bbox)
|
|
|
+ for j4 in mid_json.loc['droped_image_block',:]:
|
|
|
+ test_dropped_image_bboxes.append(j4)
|
|
|
+ for j5 in mid_json.loc['droped_table_block',:]:
|
|
|
+ test_dropped_table_bboxes.append(j5)
|
|
|
+ for j6 in mid_json.loc['preproc_blocks',:]:
|
|
|
+ page_in=[]
|
|
|
+ for k6 in j6:
|
|
|
+ page_in.append(k6['number'])
|
|
|
+ test_preproc_num.append(page_in)
|
|
|
+
|
|
|
+ test_pdf_text=[]
|
|
|
+ for j7 in mid_json.loc['para_blocks',:]:
|
|
|
+ test_para_num.append(len(j7))
|
|
|
+ for k7 in j7:
|
|
|
+ test_pdf_text.append(k7['text'])
|
|
|
+ test_para_text.append(test_pdf_text)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ standard_inline_equations=[]
|
|
|
+ standard_interline_equations=[]
|
|
|
+ standard_inline_euqations_bboxs=[]
|
|
|
+ standard_interline_equations_bboxs=[]
|
|
|
+ standard_dropped_text_bboxes=[]
|
|
|
+ standard_dropped_text_tag=[]
|
|
|
+ standard_dropped_image_bboxes=[]
|
|
|
+ standard_dropped_table_bboxes=[]
|
|
|
+ standard_preproc_num=[]#阅读顺序
|
|
|
+ standard_para_num=[]
|
|
|
+ standard_para_text=[]
|
|
|
+
|
|
|
+ for i in json_standard:
|
|
|
+ mid_json=pd.DataFrame(i)
|
|
|
+ mid_json=mid_json.iloc[:,:-1]
|
|
|
+ for j1 in mid_json.loc['inline_equations',:]:
|
|
|
+ page_in_text=[]
|
|
|
+ page_in_bbox=[]
|
|
|
+ for k1 in j1:
|
|
|
+ page_in_text.append(k1['latex_text'])
|
|
|
+ page_in_bbox.append(k1['bbox'])
|
|
|
+ standard_inline_equations.append(page_in_text)
|
|
|
+ standard_inline_euqations_bboxs.append(page_in_bbox)
|
|
|
+ for j2 in mid_json.loc['interline_equations',:]:
|
|
|
+ page_in_text=[]
|
|
|
+ page_in_bbox=[]
|
|
|
+ for k2 in j2:
|
|
|
+ page_in_text.append(k2['latex_text'])
|
|
|
+ page_in_bbox.append(k2['bbox'])
|
|
|
+ standard_interline_equations.append(page_in_text)
|
|
|
+ standard_interline_equations_bboxs.append(page_in_bbox)
|
|
|
+ for j3 in mid_json.loc['droped_text_block',:]:
|
|
|
+ page_in_bbox=[]
|
|
|
+ page_in_tag=[]
|
|
|
+ for k3 in j3:
|
|
|
+ page_in_bbox.append(k3['bbox'])
|
|
|
+ if 'tag' in k3.keys():
|
|
|
+ page_in_tag.append(k3['tag'])
|
|
|
+ else:
|
|
|
+ page_in_tag.append('None')
|
|
|
+ standard_dropped_text_bboxes.append(page_in_bbox)
|
|
|
+ standard_dropped_text_tag.append(page_in_tag)
|
|
|
+ for j4 in mid_json.loc['droped_image_block',:]:
|
|
|
+ standard_dropped_image_bboxes.append(j4)
|
|
|
+ for j5 in mid_json.loc['droped_table_block',:]:
|
|
|
+ standard_dropped_table_bboxes.append(j5)
|
|
|
+ for j6 in mid_json.loc['preproc_blocks',:]:
|
|
|
+ page_in=[]
|
|
|
+ for k6 in j6:
|
|
|
+ page_in.append(k6['number'])
|
|
|
+ standard_preproc_num.append(page_in)
|
|
|
+
|
|
|
+ standard_pdf_text=[]
|
|
|
+ for j7 in mid_json.loc['para_blocks',:]:
|
|
|
+ standard_para_num.append(len(j7))
|
|
|
+ for k7 in j7:
|
|
|
+ standard_pdf_text.append(k7['text'])
|
|
|
+ standard_para_text.append(standard_pdf_text)
|
|
|
+
|
|
|
+
|
|
|
+ """
|
|
|
+ 在计算指标之前最好先确认基本统计信息是否一致
|
|
|
+ """
|
|
|
+
|
|
|
+
|
|
|
+ '''
|
|
|
+ 计算pdf之间的总体编辑距离和bleu
|
|
|
+ 这里只计算正例的pdf
|
|
|
+ '''
|
|
|
+
|
|
|
+ test_para_text=np.asarray(test_para_text, dtype = object)[inner_merge['pass_label']=='yes']
|
|
|
+ standard_para_text=np.asarray(standard_para_text, dtype = object)[inner_merge['pass_label']=='yes']
|
|
|
+
|
|
|
+ pdf_dis=[]
|
|
|
+ pdf_bleu=[]
|
|
|
+ for a,b in zip(test_para_text,standard_para_text):
|
|
|
+ a1=[ ''.join(i) for i in a]
|
|
|
+ b1=[ ''.join(i) for i in b]
|
|
|
+ pdf_dis.append(Levenshtein_Distance(a1,b1))
|
|
|
+ pdf_bleu.append(sentence_bleu([a1],b1))
|
|
|
+ overall_report['pdf间的平均编辑距离']=np.mean(pdf_dis)
|
|
|
+ overall_report['pdf间的平均bleu']=np.mean(pdf_bleu)
|
|
|
+
|
|
|
+
|
|
|
+ '''行内公式和行间公式的编辑距离和bleu'''
|
|
|
+ inline_equations_edit_bleu=equations_indicator(test_inline_euqations_bboxs,standard_inline_euqations_bboxs,test_inline_equations,standard_inline_equations)
|
|
|
+ interline_equations_edit_bleu=equations_indicator(test_interline_equations_bboxs,standard_interline_equations_bboxs,test_interline_equations,standard_interline_equations)
|
|
|
+
|
|
|
+ '''行内公式bbox匹配相关指标'''
|
|
|
+ inline_equations_bbox_report=bbox_match_indicator(test_inline_euqations_bboxs,standard_inline_euqations_bboxs)
|
|
|
+
|
|
|
+ '''行间公式bbox匹配相关指标'''
|
|
|
+ interline_equations_bbox_report=bbox_match_indicator(test_interline_equations_bboxs,standard_interline_equations_bboxs)
|
|
|
+
|
|
|
+ '''可以先检查page和bbox数量是否一致'''
|
|
|
+ '''dropped_text_block的bbox匹配相关指标'''
|
|
|
+ test_text_bbox=[]
|
|
|
+ standard_text_bbox=[]
|
|
|
+ test_tag=[]
|
|
|
+ standard_tag=[]
|
|
|
+
|
|
|
+ index=0
|
|
|
+ for a,b in zip(test_dropped_text_bboxes,standard_dropped_text_bboxes):
|
|
|
+ test_page_tag=[]
|
|
|
+ standard_page_tag=[]
|
|
|
+ test_page_bbox=[]
|
|
|
+ standard_page_bbox=[]
|
|
|
+ if len(a)==0 and len(b)==0:
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ for i in range(len(b)):
|
|
|
+ judge=0
|
|
|
+ standard_page_tag.append(standard_dropped_text_tag[index][i])
|
|
|
+ standard_page_bbox.append(1)
|
|
|
+ for j in range(len(a)):
|
|
|
+ if bbox_offset(b[i],a[j]):
|
|
|
+ judge=1
|
|
|
+ test_page_tag.append(test_dropped_text_tag[index][j])
|
|
|
+ test_page_bbox.append(1)
|
|
|
+ break
|
|
|
+ if judge==0:
|
|
|
+ test_page_tag.append('None')
|
|
|
+ test_page_bbox.append(0)
|
|
|
+
|
|
|
+
|
|
|
+ if len(test_dropped_text_tag[index])+test_page_tag.count('None')>len(standard_dropped_text_tag[index]):#有多删的情况出现
|
|
|
+ test_page_tag1=test_page_tag.copy()
|
|
|
+ if 'None' in test_page_tag:
|
|
|
+ test_page_tag1=test_page_tag1.remove('None')
|
|
|
+ else:
|
|
|
+ test_page_tag1=test_page_tag
|
|
|
+
|
|
|
+ diff=list((Counter(test_dropped_text_tag[index]) - Counter(test_page_tag1)).elements())
|
|
|
+
|
|
|
+ test_page_tag.extend(diff)
|
|
|
+ standard_page_tag.extend(['None']*len(diff))
|
|
|
+ test_page_bbox.extend([1]*len(diff))
|
|
|
+ standard_page_bbox.extend([0]*len(diff))
|
|
|
+
|
|
|
+ test_tag.extend(test_page_tag)
|
|
|
+ standard_tag.extend(standard_page_tag)
|
|
|
+ test_text_bbox.extend(test_page_bbox)
|
|
|
+ standard_text_bbox.extend(standard_page_bbox)
|
|
|
+
|
|
|
+ index+=1
|
|
|
+
|
|
|
+
|
|
|
+ text_block_report = {}
|
|
|
+ text_block_report['accuracy']=metrics.accuracy_score(standard_text_bbox,test_text_bbox)
|
|
|
+ text_block_report['precision']=metrics.precision_score(standard_text_bbox,test_text_bbox)
|
|
|
+ text_block_report['recall']=metrics.recall_score(standard_text_bbox,test_text_bbox)
|
|
|
+ text_block_report['f1_score']=metrics.f1_score(standard_text_bbox,test_text_bbox)
|
|
|
+
|
|
|
+ '''删除的text_block的tag的准确率,召回率和f1-score'''
|
|
|
+ text_block_tag_report = classification_report(y_true=standard_tag , y_pred=test_tag,output_dict=True)
|
|
|
+ del text_block_tag_report['None']
|
|
|
+ del text_block_tag_report["macro avg"]
|
|
|
+ del text_block_tag_report["weighted avg"]
|
|
|
+
|
|
|
+
|
|
|
+ '''dropped_image_block的bbox匹配相关指标'''
|
|
|
+ '''有数据格式不一致的问题'''
|
|
|
+ image_block_report=bbox_match_indicator(test_dropped_image_bboxes,standard_dropped_image_bboxes)
|
|
|
+
|
|
|
+
|
|
|
+ '''dropped_table_block的bbox匹配相关指标'''
|
|
|
+ table_block_report=bbox_match_indicator(test_dropped_table_bboxes,standard_dropped_table_bboxes)
|
|
|
+
|
|
|
+
|
|
|
+ '''阅读顺序编辑距离的均值'''
|
|
|
+ preproc_num_dis=[]
|
|
|
+ for a,b in zip(test_preproc_num,standard_preproc_num):
|
|
|
+ preproc_num_dis.append(Levenshtein_Distance(a,b))
|
|
|
+ preproc_num_edit=np.mean(preproc_num_dis)
|
|
|
+
|
|
|
+
|
|
|
+ '''分段准确率'''
|
|
|
+ test_para_num=np.array(test_para_num)
|
|
|
+ standard_para_num=np.array(standard_para_num)
|
|
|
+ acc_para=np.mean(test_para_num==standard_para_num)
|
|
|
+
|
|
|
+
|
|
|
+ output=pd.DataFrame()
|
|
|
+ output['总体指标']=[overall_report]
|
|
|
+ output['行内公式平均编辑距离']=[inline_equations_edit_bleu[0]]
|
|
|
+ output['行内公式平均bleu']=[inline_equations_edit_bleu[1]]
|
|
|
+ output['行间公式平均编辑距离']=[interline_equations_edit_bleu[0]]
|
|
|
+ output['行间公式平均bleu']=[interline_equations_edit_bleu[1]]
|
|
|
+ output['行内公式识别相关指标']=[inline_equations_bbox_report]
|
|
|
+ output['行间公式识别相关指标']=[interline_equations_bbox_report]
|
|
|
+ output['阅读顺序平均编辑距离']=[preproc_num_edit]
|
|
|
+ output['分段准确率']=[acc_para]
|
|
|
+ output['删除的text block的相关指标']=[text_block_report]
|
|
|
+ output['删除的image block的相关指标']=[image_block_report]
|
|
|
+ output['删除的table block的相关指标']=[table_block_report]
|
|
|
+ output['删除的text block的tag相关指标']=[text_block_tag_report]
|
|
|
+
|
|
|
+
|
|
|
+ return output
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+"""
|
|
|
+计算编辑距离
|
|
|
+"""
|
|
|
+def Levenshtein_Distance(str1, str2):
|
|
|
+ matrix = [[ i + j for j in range(len(str2) + 1)] for i in range(len(str1) + 1)]
|
|
|
+ for i in range(1, len(str1)+1):
|
|
|
+ for j in range(1, len(str2)+1):
|
|
|
+ if(str1[i-1] == str2[j-1]):
|
|
|
+ d = 0
|
|
|
+ else:
|
|
|
+ d = 1
|
|
|
+ matrix[i][j] = min(matrix[i-1][j]+1, matrix[i][j-1]+1, matrix[i-1][j-1]+d)
|
|
|
+ return matrix[len(str1)][len(str2)]
|
|
|
+
|
|
|
+
|
|
|
+'''
|
|
|
+计算bbox偏移量是否符合标准的函数
|
|
|
+'''
|
|
|
+def bbox_offset(b_t,b_s):
|
|
|
+ '''b_t是test_doc里的bbox,b_s是standard_doc里的bbox'''
|
|
|
+ x1_t,y1_t,x2_t,y2_t=b_t
|
|
|
+ x1_s,y1_s,x2_s,y2_s=b_s
|
|
|
+ x1=max(x1_t,x1_s)
|
|
|
+ x2=min(x2_t,x2_s)
|
|
|
+ y1=max(y1_t,y1_s)
|
|
|
+ y2=min(y2_t,y2_s)
|
|
|
+ area_overlap=(x2-x1)*(y2-y1)
|
|
|
+ area_t=(x2_t-x1_t)*(y2_t-y1_t)+(x2_s-x1_s)*(y2_s-y1_s)-area_overlap
|
|
|
+ if area_t-area_overlap==0 or area_overlap/(area_t-area_overlap)>0.95:
|
|
|
+ return True
|
|
|
+ else:
|
|
|
+ return False
|
|
|
+
|
|
|
+
|
|
|
+'''bbox匹配和对齐函数，输出相关指标'''
|
|
|
+'''输入的是以page为单位的bbox列表'''
|
|
|
+def bbox_match_indicator(test_bbox_list,standard_bbox_list):
|
|
|
+
|
|
|
+ test_bbox=[]
|
|
|
+ standard_bbox=[]
|
|
|
+ for a,b in zip(test_bbox_list,standard_bbox_list):
|
|
|
+
|
|
|
+ test_page_bbox=[]
|
|
|
+ standard_page_bbox=[]
|
|
|
+ if len(a)==0 and len(b)==0:
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ for i in b:
|
|
|
+ if len(i)!=4:
|
|
|
+ continue
|
|
|
+ else:
|
|
|
+ judge=0
|
|
|
+ standard_page_bbox.append(1)
|
|
|
+ for j in a:
|
|
|
+ if bbox_offset(i,j):
|
|
|
+ judge=1
|
|
|
+ test_page_bbox.append(1)
|
|
|
+ break
|
|
|
+ if judge==0:
|
|
|
+ test_page_bbox.append(0)
|
|
|
+
|
|
|
+ diff_num=len(a)+test_page_bbox.count(0)-len(b)
|
|
|
+ if diff_num>0:#有多删的情况出现
|
|
|
+ test_page_bbox.extend([1]*diff_num)
|
|
|
+ standard_page_bbox.extend([0]*diff_num)
|
|
|
+
|
|
|
+
|
|
|
+ test_bbox.extend(test_page_bbox)
|
|
|
+ standard_bbox.extend(standard_page_bbox)
|
|
|
+
|
|
|
+
|
|
|
+ block_report = {}
|
|
|
+ block_report['accuracy']=metrics.accuracy_score(standard_bbox,test_bbox)
|
|
|
+ block_report['precision']=metrics.precision_score(standard_bbox,test_bbox)
|
|
|
+ block_report['recall']=metrics.recall_score(standard_bbox,test_bbox)
|
|
|
+ block_report['f1_score']=metrics.f1_score(standard_bbox,test_bbox)
|
|
|
+
|
|
|
+ return block_report
|
|
|
+
|
|
|
+'''公式编辑距离和bleu'''
|
|
|
+def equations_indicator(test_euqations_bboxs,standard_euqations_bboxs,test_equations,standard_equations):
|
|
|
+ test_match_equations=[]
|
|
|
+ standard_match_equations=[]
|
|
|
+
|
|
|
+ index=0
|
|
|
+ for a,b in zip(test_euqations_bboxs,standard_euqations_bboxs):
|
|
|
+ if len(a)==0 and len(b)==0:
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ for i in range(len(b)):
|
|
|
+ for j in range(len(a)):
|
|
|
+ if bbox_offset(b[i],a[j]):
|
|
|
+ standard_match_equations.append(standard_equations[index][i])
|
|
|
+ test_match_equations.append(test_equations[index][j])
|
|
|
+ break
|
|
|
+ index+=1
|
|
|
+
|
|
|
+
|
|
|
+ dis=[]
|
|
|
+ bleu=[]
|
|
|
+ for a,b in zip(test_match_equations,standard_match_equations):
|
|
|
+ if len(a)==0 and len(b)==0:
|
|
|
+ continue
|
|
|
+ else:
|
|
|
+ if a==b:
|
|
|
+ dis.append(0)
|
|
|
+ bleu.append(1)
|
|
|
+ else:
|
|
|
+ dis.append(Levenshtein_Distance(a,b))
|
|
|
+ bleu.append(sentence_bleu([a],b))
|
|
|
+
|
|
|
+ equations_edit=np.mean(dis)
|
|
|
+ equations_bleu=np.mean(bleu)
|
|
|
+ return (equations_edit,equations_bleu)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+parser = argparse.ArgumentParser()
|
|
|
+parser.add_argument('--test', type=str)
|
|
|
+parser.add_argument('--standard', type=str)
|
|
|
+args = parser.parse_args()
|
|
|
+pdf_json_test = args.test
|
|
|
+pdf_json_standard = args.standard
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+
|
|
|
+ pdf_json_test = [json.loads(line)
|
|
|
+ for line in open(pdf_json_test, 'r', encoding='utf-8')]
|
|
|
+ pdf_json_standard = [json.loads(line)
|
|
|
+ for line in open(pdf_json_standard, 'r', encoding='utf-8')]
|
|
|
+
|
|
|
+ overall_indicator=indicator_cal(pdf_json_standard,pdf_json_test)
|
|
|
+
|
|
|
+ '''计算的指标输出到overall_indicator_output.json中'''
|
|
|
+ overall_indicator.to_json('overall_indicator_output.json',orient='records',lines=True,force_ascii=False)
|
|
|
+
|
liusilu