Spaces:
Running
Running
feat(chunk): paragraph mode giu xuong dong - translator.py
Browse files- src/translator.py +12 -5
src/translator.py
CHANGED
|
@@ -22,6 +22,7 @@ from hardware import (
|
|
| 22 |
detect_hardware_profile,
|
| 23 |
resolve_gpu_indices,
|
| 24 |
)
|
|
|
|
| 25 |
from token_chunker import source_token_ids, split_for_translation
|
| 26 |
|
| 27 |
import ctranslate2
|
|
@@ -1200,11 +1201,17 @@ class HachimiTranslator:
|
|
| 1200 |
yield end, total, f"Đã xong {end}/{total} chunk", None, None
|
| 1201 |
|
| 1202 |
assemble_start = time.perf_counter()
|
| 1203 |
-
|
| 1204 |
-
|
| 1205 |
-
|
| 1206 |
-
|
| 1207 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1208 |
self._profile_add("assemble_s", time.perf_counter() - assemble_start)
|
| 1209 |
cache_stats = getattr(self._tokenizer, "cache_stats", None)
|
| 1210 |
if callable(cache_stats):
|
|
|
|
| 22 |
detect_hardware_profile,
|
| 23 |
resolve_gpu_indices,
|
| 24 |
)
|
| 25 |
+
from line_restore import assemble_paragraph_output
|
| 26 |
from token_chunker import source_token_ids, split_for_translation
|
| 27 |
|
| 28 |
import ctranslate2
|
|
|
|
| 1201 |
yield end, total, f"Đã xong {end}/{total} chunk", None, None
|
| 1202 |
|
| 1203 |
assemble_start = time.perf_counter()
|
| 1204 |
+
if chunk_mode == "paragraph":
|
| 1205 |
+
# Chunk gom nhiều dòng → model nuốt xuống dòng. Khôi phục bố cục dòng
|
| 1206 |
+
# gốc (cho full_text) + ghép rows 1:1 dòng-gốc/dòng-dịch để bảng đối
|
| 1207 |
+
# chiếu khớp bản tải về. Xem line_restore.assemble_paragraph_output.
|
| 1208 |
+
rows, full_text = assemble_paragraph_output(text, chunks, translations)
|
| 1209 |
+
else:
|
| 1210 |
+
rows = [
|
| 1211 |
+
(index, chunk, translated)
|
| 1212 |
+
for index, (chunk, translated) in enumerate(zip(chunks, translations), start=1)
|
| 1213 |
+
]
|
| 1214 |
+
full_text = "\n".join(translations)
|
| 1215 |
self._profile_add("assemble_s", time.perf_counter() - assemble_start)
|
| 1216 |
cache_stats = getattr(self._tokenizer, "cache_stats", None)
|
| 1217 |
if callable(cache_stats):
|