ngocdang83 commited on
Commit
f9f04c8
·
verified ·
1 Parent(s): 2894b2a

feat(chunk): paragraph mode giu xuong dong - translator.py

Browse files
Files changed (1) hide show
  1. src/translator.py +12 -5
src/translator.py CHANGED
@@ -22,6 +22,7 @@ from hardware import (
22
  detect_hardware_profile,
23
  resolve_gpu_indices,
24
  )
 
25
  from token_chunker import source_token_ids, split_for_translation
26
 
27
  import ctranslate2
@@ -1200,11 +1201,17 @@ class HachimiTranslator:
1200
  yield end, total, f"Đã xong {end}/{total} chunk", None, None
1201
 
1202
  assemble_start = time.perf_counter()
1203
- rows = [
1204
- (index, chunk, translated)
1205
- for index, (chunk, translated) in enumerate(zip(chunks, translations), start=1)
1206
- ]
1207
- full_text = "\n".join(translations)
 
 
 
 
 
 
1208
  self._profile_add("assemble_s", time.perf_counter() - assemble_start)
1209
  cache_stats = getattr(self._tokenizer, "cache_stats", None)
1210
  if callable(cache_stats):
 
22
  detect_hardware_profile,
23
  resolve_gpu_indices,
24
  )
25
+ from line_restore import assemble_paragraph_output
26
  from token_chunker import source_token_ids, split_for_translation
27
 
28
  import ctranslate2
 
1201
  yield end, total, f"Đã xong {end}/{total} chunk", None, None
1202
 
1203
  assemble_start = time.perf_counter()
1204
+ if chunk_mode == "paragraph":
1205
+ # Chunk gom nhiều dòng → model nuốt xuống dòng. Khôi phục bố cục dòng
1206
+ # gốc (cho full_text) + ghép rows 1:1 dòng-gốc/dòng-dịch để bảng đối
1207
+ # chiếu khớp bản tải về. Xem line_restore.assemble_paragraph_output.
1208
+ rows, full_text = assemble_paragraph_output(text, chunks, translations)
1209
+ else:
1210
+ rows = [
1211
+ (index, chunk, translated)
1212
+ for index, (chunk, translated) in enumerate(zip(chunks, translations), start=1)
1213
+ ]
1214
+ full_text = "\n".join(translations)
1215
  self._profile_add("assemble_s", time.perf_counter() - assemble_start)
1216
  cache_stats = getattr(self._tokenizer, "cache_stats", None)
1217
  if callable(cache_stats):