# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re def _find_split_pos(text, chunk_size): """ Find the position to split the text into two chunks. Args: text (str): The original text to be split. chunk_size (int): The maximum size of each chunk. Returns: int: The index where the text should be split. """ center = len(text) // 2 # Search forward for i in range(center, len(text)): if text[i] in ["\n", ".", "。", ";", ";", "!", "!", "?", "?"]: if i + 1 < len(text) and len(text[: i + 1]) <= chunk_size: return i + 1 # Search backward for i in range(center, 0, -1): if text[i] in ["\n", ".", "。", ";", ";", "!", "!", "?", "?"]: if len(text[: i + 1]) <= chunk_size: return i + 1 # If no suitable position is found, split directly return min(chunk_size, len(text)) def split_text_recursive(text, chunk_size, translate_func, results): """ Split the text recursively and translate each chunk. Args: text (str): The original text to be split. chunk_size (int): The maximum size of each chunk. translate_func (callable): A function that translates a single chunk of text. results (list): A list to store the translated chunks. Returns: None """ text = text.strip() if len(text) <= chunk_size: results.append(translate_func(text)) else: split_pos = _find_split_pos(text, chunk_size) left = text[:split_pos].strip() right = text[split_pos:].strip() if left: split_text_recursive(left, chunk_size, translate_func, results) if right: split_text_recursive(right, chunk_size, translate_func, results) def translate_code_block(code_block, chunk_size, translate_func, results): """ Translate a code block and append the result to the results list. Args: code_block (str): The code block to be translated. chunk_size (int): The maximum size of each chunk. translate_func (callable): A function that translates a single chunk of text. results (list): A list to store the translated chunks. Returns: None """ lines = code_block.strip().split("\n") if lines[0].startswith("```") or lines[0].startswith("~~~"): header = lines[0] footer = ( lines[-1] if (lines[-1].startswith("```") or lines[-1].startswith("~~~")) else "" ) code_content = "\n".join(lines[1:-1]) if footer else "\n".join(lines[1:]) else: header = "" footer = "" code_content = code_block translated_code_lines = [] split_text_recursive( code_content, chunk_size, translate_func, translated_code_lines ) # drop ``` or ~~~ filtered_code_lines = [ line for line in translated_code_lines if not (line.strip().startswith("```") or line.strip().startswith("~~~")) ] translated_code = "\n".join(filtered_code_lines) result = f"{header}\n{translated_code}\n{footer}" if header else translated_code results.append(result) def translate_html_block(html_block, chunk_size, translate_func, results): """ Translate a HTML block and append the result to the results list. Args: html_block (str): The HTML block to be translated. chunk_size (int): The maximum size of each chunk. translate_func (callable): A function that translates a single chunk of text. results (list): A list to store the translated chunks. Returns: None """ from bs4 import BeautifulSoup soup = BeautifulSoup(html_block, "html.parser") # collect text nodes text_nodes = [] for node in soup.find_all(string=True, recursive=True): text = node.strip() if text: text_nodes.append(node) idx = 0 total = len(text_nodes) while idx < total: batch_nodes = [] li_texts = [] current_length = len("