WolfDavid commited on
Commit
93dd3ff
Β·
1 Parent(s): 9fb7ec4

test(02-04): add failing translation contract tests

Browse files

- tests/test_translate.py (mt + slow markers): warm-up under 5 s and idempotent, six
contract sentences asserted on keywords with per-call ms printed, pieces end with </s>,
empty/whitespace raise ValueError, 200-char line bounded by max_decoding_length,
model_bytes matches disk, and no transformers/torch in sys.modules after warm-up
- Fails at collection: japanese_avatar.nlp.translate does not exist yet

Files changed (1) hide show
  1. tests/test_translate.py +101 -0
tests/test_translate.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """JPN-04 contract for the CPU translator: six sentences, no transformers, warm in under 5 s.
2
+
3
+ Marked ``mt`` (and ``slow``) because it loads the CTranslate2 model (~0.3 s, +100 MB), which
4
+ keeps it out of the quick loop; plan 02-05 runs the whole loop and records its wall time. Every
5
+ assertion here is a keyword, never an exact string - the model is pinned, but a beam search is
6
+ not a contract on wording. Per-call milliseconds are printed for the record and NOT bounded:
7
+ research measured 16-129 ms locally at two threads, and the number that counts is the deployed
8
+ one, which plan 02-10 measures on the Space.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import sys
14
+ import time
15
+
16
+ import pytest
17
+
18
+ from japanese_avatar.nlp import translate as mt
19
+
20
+ pytestmark = [pytest.mark.mt, pytest.mark.slow]
21
+
22
+ # Japanese -> substrings of which at least one must appear (case-insensitive). Research Β§ Q4
23
+ # measured the full outputs; the keywords are the parts a paraphrase cannot drop.
24
+ CONTRACT = [
25
+ ("こんにけは。", ("hello",)),
26
+ ("駅はどこですか。", ("station",)),
27
+ ("ζ—₯本θͺžγ‚’勉強しています。", ("japanese",)),
28
+ (
29
+ "今ζ—₯γ―γ„γ„ε€©ζ°—γ§γ™γ‹γ‚‰γ€ε…¬εœ’γ‚’ζ•£ζ­©γ—γ¦γ‹γ‚‰γ€θ²·γ„η‰©γ«θ‘ŒγγΎγ—γŸγ€‚",
30
+ ("park", "shopping", "weather"),
31
+ ),
32
+ ("ζ—₯本θͺžγ‚’練習しましょう。", ("japanese", "practice")),
33
+ ("昨ζ—₯δ½•γ‚’γ—γΎγ—γŸγ‹γ€‚", ("yesterday",)),
34
+ ]
35
+
36
+ WARMUP_FIRST_MAX_S = 5.0 # research: 0.25-0.58 s locally; the Space CPU is 3-6x slower
37
+ WARMUP_REPEAT_MAX_S = 0.01
38
+
39
+
40
+ @pytest.fixture(scope="module")
41
+ def warm() -> tuple[float, float]:
42
+ """Load the translator once per module; returns (first, second) warm-up seconds."""
43
+ first = mt.warmup()
44
+ second = mt.warmup()
45
+ print(f"\ntranslator warm-up {first:.3f} s, repeat {second:.4f} s")
46
+ return first, second
47
+
48
+
49
+ def test_warmup_is_fast_and_idempotent(warm):
50
+ first, second = warm
51
+ assert first < WARMUP_FIRST_MAX_S, f"first warm-up took {first:.2f} s"
52
+ assert second < WARMUP_REPEAT_MAX_S, f"repeat warm-up took {second:.4f} s; not cached"
53
+ assert mt.get_translator() is mt.get_translator()
54
+
55
+
56
+ @pytest.mark.parametrize(("text", "keywords"), CONTRACT, ids=[t for t, _ in CONTRACT])
57
+ def test_contract_sentences(warm, text, keywords):
58
+ started = time.perf_counter()
59
+ out = mt.translate(text)
60
+ ms = (time.perf_counter() - started) * 1000.0
61
+ print(f"\n{text} -> {out!r} ({ms:.0f} ms)")
62
+ assert isinstance(out, str) and out.strip(), f"empty translation for {text!r}"
63
+ assert any(k in out.lower() for k in keywords), f"{out!r} contains none of {keywords}"
64
+
65
+
66
+ def test_pieces_end_with_eos(warm):
67
+ pieces = mt.encode_pieces("こんにけは")
68
+ assert pieces[-1] == "</s>"
69
+ assert len(pieces) > 1
70
+ assert all(isinstance(p, str) for p in pieces)
71
+
72
+
73
+ @pytest.mark.parametrize("text", ["", " ", "\n\t"])
74
+ def test_empty_and_whitespace_raise(warm, text):
75
+ with pytest.raises(ValueError):
76
+ mt.translate(text)
77
+
78
+
79
+ def test_long_line_is_bounded(warm):
80
+ """A MAX_TEXT_CHARS-long line decodes within max_decoding_length and stays non-empty."""
81
+ text = ("ζ—₯本θͺžγ‚’勉強しています。" * 20)[: mt.MAX_TEXT_CHARS]
82
+ assert len(text) == mt.MAX_TEXT_CHARS
83
+ started = time.perf_counter()
84
+ out = mt.translate(text)
85
+ ms = (time.perf_counter() - started) * 1000.0
86
+ print(f"\n{len(text)}-char line -> {len(out)} chars ({ms:.0f} ms)")
87
+ assert out.strip()
88
+ _, _, target = mt.get_translator()
89
+ assert len(target.encode(out, out_type=str)) <= mt.DEFAULT_MAX_DECODING_LENGTH
90
+
91
+
92
+ def test_model_bytes_is_the_committed_model(warm):
93
+ assert mt.model_bytes() == (mt.MODEL_DIR / "model.bin").stat().st_size
94
+ assert mt.model_bytes() > 70_000_000
95
+
96
+
97
+ def test_no_transformers_or_torch_imported(warm):
98
+ """The Space's translation path is ctranslate2 + sentencepiece only (research Β§ Q4)."""
99
+ assert "transformers" not in sys.modules
100
+ assert "torch" not in sys.modules
101
+ assert "ctranslate2" in sys.modules and "sentencepiece" in sys.modules