VRDate commited on
Commit
f9736aa
·
1 Parent(s): 0cc841d

v2025.12.5: Simplify sentence breaking, consolidate download scripts

Browse files

## Sentence Breaking
- Remove multi-backend architecture (ICU, wtpsplit, regex fallback)
- Use pysbd exclusively for sentence segmentation
- Remove Protocol, WtpSplitBreaker, ICUBreaker, RegexBreaker, SentenceBreakerFactory
- Single SentenceBreaker class with pysbd

## Download Scripts
- Delete download_models.sh (bash)
- Delete download_models.ps1 (PowerShell)
- Keep only download_models.py (cross-platform, most reliable)

## Footer
- Add version number (v2025.12.5)
- Add JSON schema link (github.com/CLK-AL/HebrewNLP)
- Add full credits: Dicta, thewh1teagle, Piper, pysbd

## README
- Update to v2025.12.5
- Document single download script (download_models.py)
- Remove multi-backend sentence breaking section
- Add rhasspy/piper-voices to credits
- Clarify thewh1teagle credit (diacritization + Hebrew voice)
- Simplify Quick Start to two options (standard + UV)

## Files Changed
- app.py: Simplified sentence breaker, updated footer
- README.md: Updated documentation
- Deleted: download_models.sh, download_models.ps1

.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ /.streamlit/
2
+ /onnx/
.idea/.gitignore ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ # Default ignored files
2
+ /shelf/
3
+ /workspace.xml
4
+ # Editor-based HTTP Client requests
5
+ /httpRequests/
.idea/dictabert-joint-phonikud-tts.iml ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <module type="PYTHON_MODULE" version="4">
3
+ <component name="NewModuleRootManager">
4
+ <content url="file://$MODULE_DIR$">
5
+ <excludeFolder url="file://$MODULE_DIR$/.venv" />
6
+ <excludeFolder url="file://$MODULE_DIR$/.venv1" />
7
+ <excludeFolder url="file://$MODULE_DIR$/.venv2" />
8
+ <excludeFolder url="file://$MODULE_DIR$/.cache/huggingface/download" />
9
+ <excludeFolder url="file://$MODULE_DIR$/.streamlit" />
10
+ </content>
11
+ <orderEntry type="jdk" jdkName="Python 3.12 (hebrew-unified-nlp)" jdkType="Python SDK" />
12
+ <orderEntry type="sourceFolder" forTests="false" />
13
+ </component>
14
+ </module>
.idea/dictionaries/project.xml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ <component name="ProjectDictionaryState">
2
+ <dictionary name="project">
3
+ <words>
4
+ <w>libsndfile</w>
5
+ </words>
6
+ </dictionary>
7
+ </component>
.idea/inspectionProfiles/profiles_settings.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <component name="InspectionProjectProfileManager">
2
+ <settings>
3
+ <option name="USE_PROJECT_PROFILE" value="false" />
4
+ <version value="1.0" />
5
+ </settings>
6
+ </component>
.idea/misc.xml ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="Black">
4
+ <option name="sdkName" value="Python 3.13 (hebrew-unified-nlp)" />
5
+ </component>
6
+ <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12 (hebrew-unified-nlp)" project-jdk-type="Python SDK" />
7
+ </project>
.idea/modules.xml ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="ProjectModuleManager">
4
+ <modules>
5
+ <module fileurl="file://$PROJECT_DIR$/.idea/dictabert-joint-phonikud-tts.iml" filepath="$PROJECT_DIR$/.idea/dictabert-joint-phonikud-tts.iml" />
6
+ </modules>
7
+ </component>
8
+ </project>
.idea/vcs.xml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <project version="4">
3
+ <component name="VcsDirectoryMappings">
4
+ <mapping directory="" vcs="Git" />
5
+ </component>
6
+ </project>
Dockerfile CHANGED
@@ -1,23 +1,41 @@
1
  FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim
2
 
3
- RUN useradd -m -u 1000 user
4
-
5
- RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
 
 
 
 
6
 
7
  WORKDIR /app
8
 
9
- # Use system Python environment (recommended for containers)
10
- ENV UV_SYSTEM_PYTHON=1
11
-
12
  COPY requirements.txt .
13
- RUN uv pip install --no-cache -r requirements.txt
14
-
15
- COPY . .
16
 
17
- # Setup user
18
- RUN chown -R user:user /app
19
- USER user
20
- ENV HOME=/app
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
 
22
  EXPOSE 7860
23
- CMD ["uv", "run", "app.py", "--host", "0.0.0.0", "--port", "7860"]
 
 
 
 
 
1
  FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim
2
 
3
+ # Install system dependencies
4
+ RUN apt-get update && apt-get install -y --no-install-recommends \
5
+ graphviz \
6
+ libsndfile1 \
7
+ espeak-ng \
8
+ wget \
9
+ && rm -rf /var/lib/apt/lists/*
10
 
11
  WORKDIR /app
12
 
13
+ # Copy project files
14
+ COPY pyproject.toml .
 
15
  COPY requirements.txt .
 
 
 
16
 
17
+ # Install dependencies with UV (much faster than pip)
18
+ RUN uv pip install --system -r requirements.txt
19
+
20
+ # Create onnx directories and download models
21
+ RUN mkdir -p onnx/piper-voices && \
22
+ wget -q -O onnx/phonikud-1.0.int8.onnx \
23
+ https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx && \
24
+ wget -q -O onnx/piper-voices/he_IL-phonikud.onnx \
25
+ https://huggingface.co/thewh1teagle/phonikud-tts-checkpoints/resolve/main/model.onnx && \
26
+ wget -q -O onnx/piper-voices/he_IL-phonikud.onnx.json \
27
+ https://huggingface.co/thewh1teagle/phonikud-tts-checkpoints/resolve/main/model.config.json && \
28
+ wget -q -O onnx/piper-voices/en_US-ryan-high.onnx \
29
+ https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/ryan/high/en_US-ryan-high.onnx && \
30
+ wget -q -O onnx/piper-voices/en_US-ryan-high.onnx.json \
31
+ https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/ryan/high/en_US-ryan-high.onnx.json
32
+
33
+ # Copy application
34
+ COPY app.py schema.json ./
35
 
36
  EXPOSE 7860
37
+
38
+ ENV STREAMLIT_SERVER_PORT=7860
39
+ ENV STREAMLIT_SERVER_ADDRESS=0.0.0.0
40
+
41
+ CMD ["streamlit", "run", "app.py"]
README.md CHANGED
@@ -1,10 +1,254 @@
1
  ---
2
- title: Text to speech in Hebrew
3
- emoji: 🐢
4
- colorFrom: red
5
- colorTo: green
6
- sdk: docker
7
- sdk_version: "4.44.0"
8
  app_file: app.py
9
  pinned: false
10
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Hebrew Unified NLP
3
+ emoji: 🇮🇱
4
+ colorFrom: blue
5
+ colorTo: white
6
+ sdk: streamlit
7
+ sdk_version: 1.40.0
8
  app_file: app.py
9
  pinned: false
10
+ license: mit
11
+ models:
12
+ - dicta-il/dictabert-joint
13
+ - thewh1teagle/phonikud-onnx
14
+ - thewh1teagle/phonikud-tts-checkpoints
15
+ - rhasspy/piper-voices
16
+ ---
17
+
18
+ # 🇮🇱 Hebrew Unified NLP v2025.12.5
19
+
20
+ A comprehensive Hebrew NLP pipeline with multilingual TTS, bilingual UI (English/Hebrew), and per-sentence parallel processing.
21
+
22
+ ## Features
23
+
24
+ - **Morphological Analysis**: DictaBERT-Joint for POS tagging, dependency parsing, NER, segmentation
25
+ - **Phonetic Processing**: Phonikud for Hebrew diacritization (nikud) and phoneme generation
26
+ - **Multilingual TTS**: 37 Piper voices (36 English + 1 Hebrew) with per-sentence language detection
27
+ - **Bilingual UI**: Full English/Hebrew interface with RTL support and translated labels
28
+ - **Parallel Processing**: Async pipeline with configurable workers for batch processing
29
+ - **Rich Output**: SVG dependency trees, aligned audio playlist, structured JSON
30
+
31
+ ## Project Structure
32
+
33
+ ```
34
+ hebrew-unified-nlp/
35
+ ├── app.py # Main Streamlit application
36
+ ├── schema.json # JSON output schema (v2025.12.5)
37
+ ├── requirements.txt # Python dependencies
38
+ ├── Dockerfile # Container deployment
39
+ ├── download_models.py # Cross-platform model download script
40
+ └── onnx/
41
+ ├── phonikud-1.0.int8.onnx # Hebrew diacritization model
42
+ └── piper-voices/
43
+ ├── he_IL-phonikud.onnx # Hebrew TTS voice
44
+ ├── he_IL-phonikud.onnx.json
45
+ ├── en_US-ryan-high.onnx # Default English voice
46
+ ├── en_US-ryan-high.onnx.json
47
+ └── ... # Additional English voices
48
+ ```
49
+
50
+ ## ⚠️ Python Version Requirement
51
+
52
+ **Requires Python 3.10 - 3.12** (phonikud doesn't support Python 3.13 yet)
53
+
54
+ ## Quick Start
55
+
56
+ ### Standard Setup
57
+
58
+ ```bash
59
+ # Create virtual environment
60
+ python -m venv .venv
61
+ .venv\Scripts\activate # Windows
62
+ source .venv/bin/activate # Linux/Mac
63
+
64
+ # Install system dependencies (Linux only)
65
+ sudo apt install graphviz libsndfile1 espeak-ng # Ubuntu/Debian
66
+
67
+ # Install Python dependencies
68
+ pip install -r requirements.txt
69
+
70
+ # Download essential models (phonikud + Hebrew + English voice)
71
+ python download_models.py --essential
72
+
73
+ # Or download all 36 English voices
74
+ python download_models.py
75
+
76
+ # Run
77
+ streamlit run app.py
78
+ ```
79
+
80
+ ### With UV Package Manager (Faster)
81
+
82
+ ```bash
83
+ # Install UV
84
+ curl -LsSf https://astral.sh/uv/install.sh | sh # Linux/Mac
85
+ powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" # Windows
86
+
87
+ # Create venv with Python 3.11
88
+ uv venv --python 3.11
89
+ source .venv/bin/activate # or .venv\Scripts\activate on Windows
90
+
91
+ # Install dependencies (10x faster than pip)
92
+ uv pip install -r requirements.txt
93
+
94
+ # Download models
95
+ python download_models.py --essential
96
+
97
+ # Run
98
+ streamlit run app.py
99
+ ```
100
+
101
+ ## Download Models
102
+
103
+ ```bash
104
+ python download_models.py # All 36 English voices + Hebrew
105
+ python download_models.py --essential # Minimal: phonikud + Hebrew + en_US-ryan-high
106
+ ```
107
+
108
+ The script uses HuggingFace Hub, skips existing files, and works on all platforms.
109
+
110
+ ## Available Voices
111
+
112
+ ### Hebrew
113
+ - `he_IL-phonikud` - Hebrew voice with proper nikud support
114
+
115
+ ### English (36 voices)
116
+ **British (en_GB):**
117
+ alan, alba, aru, cori, jenny_dioco, northern_english_male, semaine, southern_english_female, vctk
118
+
119
+ **American (en_US):**
120
+ amy, arctic, bryce, danny, hfc_female, hfc_male, joe, john, kathleen, kusal, kristin, l2arctic, lessac, libritts, libritts_r, ljspeech, norman, ryan
121
+
122
+ Each voice has quality variants: low, medium, high (not all variants available for all voices).
123
+
124
+ ## JSON Output Format
125
+
126
+ ```json
127
+ {
128
+ "meta": {
129
+ "version": "2025.12.5",
130
+ "timestamp": "2025-12-05T14:30:00Z",
131
+ "models": {
132
+ "phonikud": "phonikud-1.0.int8",
133
+ "piper": "he_IL-phonikud / en_US-ryan-high",
134
+ "dictabert": "dicta-il/dictabert-joint"
135
+ },
136
+ "processing_time_ms": 245.5,
137
+ "sentence_count": 3,
138
+ "parallel_workers": 4,
139
+ "sentence_breaker": "pysbd"
140
+ },
141
+ "input": {
142
+ "text": "הילד הלך לבית הספר. He walked home.",
143
+ "language": "mixed"
144
+ },
145
+ "translations": {
146
+ "pos": { "NOUN": {"en": "Noun", "he": "שם עצם"}, "..." : "..." },
147
+ "dep": { "nsubj": {"en": "Subject", "he": "נושא"}, "..." : "..." },
148
+ "ner": { "PER": {"en": "Person", "he": "אדם"}, "..." : "..." },
149
+ "morph": { "Masc": {"en": "Masculine", "he": "זכר"}, "..." : "..." }
150
+ },
151
+ "sentences": [
152
+ {
153
+ "index": 0,
154
+ "text": "הילד הלך לבית הספר.",
155
+ "language": "he",
156
+ "phonetics": {
157
+ "diacritized": "הַיֶּלֶד הָלַךְ לְבֵית הַסֵּפֶר.",
158
+ "phonemes": "h a j e l e d | h a l a x | l e v e j t | h a s e f e r"
159
+ },
160
+ "tokens": ["..."],
161
+ "ner_entities": [],
162
+ "speech": {
163
+ "format": "wav",
164
+ "sample_rate": 22050,
165
+ "duration_ms": 1847,
166
+ "voice": "he_IL-phonikud",
167
+ "data_uri": "data:audio/wav;base64,..."
168
+ }
169
+ },
170
+ {
171
+ "index": 1,
172
+ "text": "He walked home.",
173
+ "language": "en",
174
+ "speech": {
175
+ "format": "wav",
176
+ "sample_rate": 22050,
177
+ "duration_ms": 1200,
178
+ "voice": "en_US-ryan-high",
179
+ "data_uri": "data:audio/wav;base64,..."
180
+ }
181
+ }
182
+ ]
183
+ }
184
+ ```
185
+
186
+ ## API Usage
187
+
188
+ ```python
189
+ from app import AsyncHebrewNLP
190
+ import asyncio
191
+
192
+ # Initialize
193
+ nlp = AsyncHebrewNLP(hf_token="YOUR_TOKEN", max_workers=4)
194
+
195
+ # Async processing with multilingual TTS
196
+ async def analyze():
197
+ result = await nlp.process(
198
+ "הילד הלך לבית הספר. He walked home.",
199
+ include_audio=True,
200
+ en_voice="en_US-ryan-high", # English voice selection
201
+ compute_mst=True
202
+ )
203
+ return result
204
+
205
+ result = asyncio.run(analyze())
206
+
207
+ # Or sync wrapper
208
+ result = nlp.process_sync("הילד הלך לבית הספר.")
209
+
210
+ # Access results
211
+ print(f"Sentences: {result['meta']['sentence_count']}")
212
+ for sent in result['sentences']:
213
+ print(f"[{sent['index']}] ({sent['language']}) {sent['text']}")
214
+ if sent['language'] == 'he':
215
+ print(f" Nikud: {sent['phonetics']['diacritized']}")
216
+ ```
217
+
218
+ ## Performance
219
+
220
+ | Sentences | Sequential | Parallel (4 workers) | Speedup |
221
+ |-----------|------------|---------------------|---------|
222
+ | 1 | ~120ms | ~120ms | 1x |
223
+ | 4 | ~480ms | ~150ms | 3.2x |
224
+ | 10 | ~1200ms | ~350ms | 3.4x |
225
+
226
+ ## Docker
227
+
228
+ ```bash
229
+ docker build -t hebrew-nlp .
230
+ docker run -p 8501:8501 hebrew-nlp
231
+ ```
232
+
233
+ The Dockerfile pre-downloads essential models (phonikud + Hebrew voice + en_US-ryan-high).
234
+
235
+ ## Bilingual UI
236
+
237
+ The interface supports both English and Hebrew with a language toggle in the sidebar. All labels are translated including:
238
+ - Part-of-speech tags (NOUN → שם עצם)
239
+ - Dependency relations (nsubj → נושא)
240
+ - NER entity types (PER → אדם)
241
+ - Morphological features (Masc → זכר, Sing → יחיד)
242
+ - Prefix types (DEF → ה׳ הידיעה)
243
+
244
+ ## Credits
245
+
246
+ - [Dicta](https://dicta.org.il/) - DictaBERT-Joint model
247
+ - [thewh1teagle](https://github.com/thewh1teagle) - Phonikud diacritization & Hebrew TTS voice
248
+ - [Piper](https://github.com/rhasspy/piper) - Neural TTS engine
249
+ - [rhasspy/piper-voices](https://huggingface.co/rhasspy/piper-voices) - English TTS voices
250
+ - [pysbd](https://github.com/nipunsadvilkar/pySBD) - Sentence boundary detection
251
+
252
+ ## License
253
+
254
+ MIT
app.py CHANGED
@@ -1,78 +1,1509 @@
1
  """
2
- uv sync
3
- uv pip install Flask>=3.1.2
4
- wget https://huggingface.co/thewh1teagle/phonikud-onnx/resolve/main/phonikud-1.0.int8.onnx
5
- wget https://huggingface.co/thewh1teagle/phonikud-tts-checkpoints/resolve/main/model.onnx
6
- wget https://huggingface.co/thewh1teagle/phonikud-tts-checkpoints/resolve/main/model.config.json
7
- uv run ./examples/space_v1/app.py
8
  """
9
 
10
- from flask import Flask, render_template, request, jsonify
11
- from phonikud_tts import Phonikud, phonemize, Piper
12
- import soundfile as sf
13
  import base64
14
- import io
15
- import argparse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
 
17
- app = Flask(__name__)
18
- phonikud = Phonikud("phonikud-1.0.int8.onnx")
19
- piper = Piper("model.onnx", "model.config.json")
 
 
20
 
21
- # Get commit information safely
22
- def get_phonikud_commit():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  try:
24
- metadata = phonikud.get_metadata()
25
- return metadata.get("commit", None)
26
  except Exception:
27
- return None
28
-
29
- phonikud_commit = get_phonikud_commit()
30
-
31
- @app.route("/")
32
- def index():
33
- return render_template("index.html", phonikud_commit=phonikud_commit)
34
-
35
- @app.route("/generate", methods=["POST"])
36
- def generate():
37
- mode = request.form["mode"]
38
- text = request.form.get("text", "")
39
- phonemes = request.form.get("phonemes", "")
40
-
41
- if mode == "text":
42
- with_diacritics = phonikud.add_diacritics(text)
43
- phonemes = phonemize(with_diacritics)
44
- elif mode == "diacritics":
45
- with_diacritics = text
46
- phonemes = phonemize(with_diacritics)
47
- else:
48
- with_diacritics = None
49
-
50
- samples, sample_rate = piper.create(phonemes, is_phonemes=True, length_scale=1.20, noise_scale=0.640, noise_w=1.0) # noise_w=0.8, noise_scale=0.667
51
-
52
- # Volume up
53
- volume_factor = 2
54
- samples = samples * volume_factor
55
- samples = samples.clip(-1.0, 1.0) # Ensure values are still in valid [-1, 1] range
56
-
57
- buffer = io.BytesIO()
58
- sf.write(buffer, samples, sample_rate, format="WAV")
59
- buffer.seek(0)
60
- b64_audio = base64.b64encode(buffer.read()).decode("utf-8")
61
- data_uri = f"data:audio/wav;base64,{b64_audio}"
62
-
63
- return jsonify({
64
- "diacritics": with_diacritics,
65
- "phonemes": phonemes,
66
- "audio": data_uri
67
- })
68
-
69
- @app.route("/audio/<filename>")
70
- def serve_audio(filename):
71
- return app.send_static_file(filename)
72
-
73
- if __name__ == "__main__":
74
- parser = argparse.ArgumentParser()
75
- parser.add_argument('--host', default='0.0.0.0')
76
- parser.add_argument('--port', type=int, default=7860)
77
- args = parser.parse_args()
78
- app.run(debug=True, host=args.host, port=args.port)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  """
2
+ Hebrew Unified NLP - Async Parallel Pipeline 2025.12.5
3
+ ======================================================
4
+
5
+ Architecture:
6
+ Text → pysbd Sentence Breaker → async.gather([process(s) for s in sentences]) → JSON Array
 
7
  """
8
 
9
+ import streamlit as st
10
+ import json
 
11
  import base64
12
+ import asyncio
13
+ from io import BytesIO
14
+ from typing import Optional
15
+ from datetime import datetime, timezone
16
+ from dataclasses import dataclass
17
+ from concurrent.futures import ThreadPoolExecutor
18
+ import time
19
+ import re
20
+
21
+ import numpy as np
22
+ import soundfile as sf
23
+ from graphviz import Digraph
24
+ from transformers import AutoModel, AutoTokenizer
25
+ import torch
26
+
27
+ # Phonikud TTS
28
+ from phonikud_tts import Phonikud, phonemize, Piper
29
+
30
+
31
+ def detect_script(text: str) -> str:
32
+ """Detect primary script of text: 'he' for Hebrew, 'en' for Latin/English."""
33
+ he_count = 0
34
+ en_count = 0
35
+ for char in text:
36
+ code = ord(char)
37
+ # Hebrew: U+0590-U+05FF
38
+ if 0x0590 <= code <= 0x05FF:
39
+ he_count += 1
40
+ # Latin: A-Z, a-z
41
+ elif (0x0041 <= code <= 0x005A) or (0x0061 <= code <= 0x007A):
42
+ en_count += 1
43
+ return "he" if he_count >= en_count else "en"
44
+
45
+
46
+ # ============================================================================
47
+ # Piper Voices (rhasspy/piper-voices + phonikud-tts)
48
+ # ============================================================================
49
+
50
+ PIPER_VOICES = {
51
+ # Hebrew (phonikud-tts) - special handling
52
+ "he_IL-phonikud": ("he_IL", "phonikud", None),
53
+ # en_GB
54
+ "en_GB-alan-low": ("en_GB", "alan", "low"),
55
+ "en_GB-alan-medium": ("en_GB", "alan", "medium"),
56
+ "en_GB-alba-medium": ("en_GB", "alba", "medium"),
57
+ "en_GB-aru-medium": ("en_GB", "aru", "medium"),
58
+ "en_GB-cori-medium": ("en_GB", "cori", "medium"),
59
+ "en_GB-cori-high": ("en_GB", "cori", "high"),
60
+ "en_GB-jenny_dioco-medium": ("en_GB", "jenny_dioco", "medium"),
61
+ "en_GB-northern_english_male-medium": ("en_GB", "northern_english_male", "medium"),
62
+ "en_GB-semaine-medium": ("en_GB", "semaine", "medium"),
63
+ "en_GB-southern_english_female-low": ("en_GB", "southern_english_female", "low"),
64
+ "en_GB-vctk-medium": ("en_GB", "vctk", "medium"),
65
+ # en_US
66
+ "en_US-amy-low": ("en_US", "amy", "low"),
67
+ "en_US-amy-medium": ("en_US", "amy", "medium"),
68
+ "en_US-arctic-medium": ("en_US", "arctic", "medium"),
69
+ "en_US-bryce-medium": ("en_US", "bryce", "medium"),
70
+ "en_US-danny-low": ("en_US", "danny", "low"),
71
+ "en_US-hfc_female-medium": ("en_US", "hfc_female", "medium"),
72
+ "en_US-hfc_male-medium": ("en_US", "hfc_male", "medium"),
73
+ "en_US-joe-medium": ("en_US", "joe", "medium"),
74
+ "en_US-john-medium": ("en_US", "john", "medium"),
75
+ "en_US-kathleen-low": ("en_US", "kathleen", "low"),
76
+ "en_US-kristin-medium": ("en_US", "kristin", "medium"),
77
+ "en_US-kusal-medium": ("en_US", "kusal", "medium"),
78
+ "en_US-l2arctic-medium": ("en_US", "l2arctic", "medium"),
79
+ "en_US-lessac-high": ("en_US", "lessac", "high"),
80
+ "en_US-lessac-low": ("en_US", "lessac", "low"),
81
+ "en_US-lessac-medium": ("en_US", "lessac", "medium"),
82
+ "en_US-libritts-high": ("en_US", "libritts", "high"),
83
+ "en_US-libritts_r-medium": ("en_US", "libritts_r", "medium"),
84
+ "en_US-ljspeech-high": ("en_US", "ljspeech", "high"),
85
+ "en_US-ljspeech-medium": ("en_US", "ljspeech", "medium"),
86
+ "en_US-norman-medium": ("en_US", "norman", "medium"),
87
+ "en_US-ryan-high": ("en_US", "ryan", "high"),
88
+ "en_US-ryan-low": ("en_US", "ryan", "low"),
89
+ "en_US-ryan-medium": ("en_US", "ryan", "medium"),
90
+ }
91
+
92
+
93
+ class PiperVoiceManager:
94
+ """Lazy-load Piper voices on demand (Hebrew and English)."""
95
+
96
+ def __init__(self, cache_dir: str = "./onnx/piper-voices"):
97
+ from pathlib import Path
98
+ self.cache_dir = Path(cache_dir)
99
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
100
+ self._loaded = {}
101
+
102
+ def get(self, voice_name: str) -> Piper:
103
+ """Get voice, download if needed."""
104
+ if voice_name in self._loaded:
105
+ return self._loaded[voice_name]
106
+
107
+ if voice_name not in PIPER_VOICES:
108
+ raise ValueError(f"Unknown voice: {voice_name}")
109
+
110
+ region, speaker, quality = PIPER_VOICES[voice_name]
111
+
112
+ model_path = self.cache_dir / f"{voice_name}.onnx"
113
+ config_path = self.cache_dir / f"{voice_name}.onnx.json"
114
+
115
+ # Download if needed
116
+ if not model_path.exists():
117
+ from huggingface_hub import hf_hub_download
118
+ import shutil
119
+
120
+ if voice_name == "he_IL-phonikud":
121
+ # Hebrew voice from phonikud-tts-checkpoints
122
+ downloaded_model = hf_hub_download(
123
+ repo_id="thewh1teagle/phonikud-tts-checkpoints",
124
+ filename="model.onnx",
125
+ local_dir=self.cache_dir / "_temp",
126
+ )
127
+ downloaded_config = hf_hub_download(
128
+ repo_id="thewh1teagle/phonikud-tts-checkpoints",
129
+ filename="model.config.json",
130
+ local_dir=self.cache_dir / "_temp",
131
+ )
132
+ else:
133
+ # English voices from rhasspy/piper-voices
134
+ subdir = f"en/{region}/{speaker}/{quality}"
135
+ downloaded_model = hf_hub_download(
136
+ repo_id="rhasspy/piper-voices",
137
+ filename=f"{subdir}/{voice_name}.onnx",
138
+ local_dir=self.cache_dir / "_temp",
139
+ )
140
+ downloaded_config = hf_hub_download(
141
+ repo_id="rhasspy/piper-voices",
142
+ filename=f"{subdir}/{voice_name}.onnx.json",
143
+ local_dir=self.cache_dir / "_temp",
144
+ )
145
+
146
+ # Move to cache root with consistent naming
147
+ shutil.move(downloaded_model, model_path)
148
+ shutil.move(downloaded_config, config_path)
149
+
150
+ # Clean up temp dir
151
+ temp_dir = self.cache_dir / "_temp"
152
+ if temp_dir.exists():
153
+ shutil.rmtree(temp_dir)
154
+
155
+ self._loaded[voice_name] = Piper(str(model_path), str(config_path))
156
+ return self._loaded[voice_name]
157
+
158
+
159
+ # ============================================================================
160
+ # Label Translations (Single Source of Truth)
161
+ # ============================================================================
162
+
163
+ LABEL_TRANSLATIONS = {
164
+ "pos": {
165
+ "NOUN": {"en": "Noun", "he": "שם עצם"},
166
+ "VERB": {"en": "Verb", "he": "פועל"},
167
+ "ADJ": {"en": "Adjective", "he": "שם תואר"},
168
+ "ADV": {"en": "Adverb", "he": "תואר הפועל"},
169
+ "PRON": {"en": "Pronoun", "he": "כינוי"},
170
+ "DET": {"en": "Determiner", "he": "מגדיר"},
171
+ "ADP": {"en": "Adposition", "he": "מילת יחס"},
172
+ "CCONJ": {"en": "Coord. Conj.", "he": "מילת קישור"},
173
+ "SCONJ": {"en": "Subord. Conj.", "he": "מילת זיקה"},
174
+ "NUM": {"en": "Number", "he": "מספר"},
175
+ "PUNCT": {"en": "Punctuation", "he": "פיסוק"},
176
+ "PROPN": {"en": "Proper Noun", "he": "שם פרטי"},
177
+ "AUX": {"en": "Auxiliary", "he": "עזר"},
178
+ "INTJ": {"en": "Interjection", "he": "קריאה"},
179
+ "PART": {"en": "Particle", "he": "מילית"},
180
+ "SYM": {"en": "Symbol", "he": "סמל"},
181
+ "X": {"en": "Other", "he": "אחר"}
182
+ },
183
+ "dep": {
184
+ "nsubj": {"en": "Subject", "he": "נושא"},
185
+ "obj": {"en": "Object", "he": "מושא"},
186
+ "iobj": {"en": "Indirect Obj.", "he": "מושא עקיף"},
187
+ "root": {"en": "Root", "he": "שורש"},
188
+ "csubj": {"en": "Clausal Subj.", "he": "נושא משפטי"},
189
+ "compound": {"en": "Compound", "he": "צירוף"},
190
+ "compound:smixut": {"en": "Smixut", "he": "סמיכות"},
191
+ "amod": {"en": "Adj. Mod.", "he": "תואר"},
192
+ "advmod": {"en": "Adv. Mod.", "he": "תואר פועל"},
193
+ "nummod": {"en": "Num. Mod.", "he": "מספר"},
194
+ "nmod": {"en": "Noun Mod.", "he": "שם נלווה"},
195
+ "nmod:poss": {"en": "Possessive", "he": "שייכות"},
196
+ "nmod:npmod": {"en": "NP Modifier", "he": "צירוף שמני"},
197
+ "nmod:tmod": {"en": "Temporal", "he": "זמן"},
198
+ "det": {"en": "Determiner", "he": "מגדיר"},
199
+ "case": {"en": "Case", "he": "יחוס"},
200
+ "case:acc": {"en": "Accusative", "he": "יחוס מושא"},
201
+ "case:gen": {"en": "Genitive", "he": "יחוס שייכות"},
202
+ "mark": {"en": "Marker", "he": "סמן"},
203
+ "cc": {"en": "Coord.", "he": "קישור"},
204
+ "conj": {"en": "Conjunct", "he": "צירוף"},
205
+ "punct": {"en": "Punctuation", "he": "פיסוק"},
206
+ "cop": {"en": "Copula", "he": "אוגד"},
207
+ "aux": {"en": "Auxiliary", "he": "עזר"},
208
+ "xcomp": {"en": "Open Comp.", "he": "משלים"},
209
+ "ccomp": {"en": "Clausal Comp.", "he": "משפט משלים"},
210
+ "advcl": {"en": "Adv. Clause", "he": "משפט תואר"},
211
+ "acl": {"en": "Adj. Clause", "he": "משפט תואר שם"},
212
+ "acl:relcl": {"en": "Rel. Clause", "he": "משפט זיקה"},
213
+ "appos": {"en": "Apposition", "he": "תמורה"},
214
+ "parataxis": {"en": "Parataxis", "he": "צירוף"},
215
+ "dep": {"en": "Dependency", "he": "תלות"},
216
+ "obl": {"en": "Oblique", "he": "נסיבה"},
217
+ "vocative": {"en": "Vocative", "he": "פנייה"},
218
+ "expl": {"en": "Expletive", "he": "מילוי"},
219
+ "dislocated": {"en": "Dislocated", "he": "מוקדם"},
220
+ "discourse": {"en": "Discourse", "he": "שיח"},
221
+ "fixed": {"en": "Fixed", "he": "קבוע"},
222
+ "flat": {"en": "Flat", "he": "שטוח"},
223
+ "flat:name": {"en": "Name", "he": "שם"}
224
+ },
225
+ "ner": {
226
+ "PER": {"en": "Person", "he": "אדם"},
227
+ "LOC": {"en": "Location", "he": "מקום"},
228
+ "GPE": {"en": "Geo-Political", "he": "מקום"},
229
+ "ORG": {"en": "Organization", "he": "ארגון"},
230
+ "TIME": {"en": "Time", "he": "זמן"},
231
+ "DATE": {"en": "Date", "he": "תאריך"},
232
+ "MISC": {"en": "Misc.", "he": "אחר"}
233
+ },
234
+ "prefix": {
235
+ "ADP": {"en": "Preposition", "he": "יחוס"},
236
+ "DET": {"en": "Definite", "he": "הידוע"},
237
+ "CCONJ": {"en": "Conjunction", "he": "חיבור"},
238
+ "SCONJ": {"en": "Subordinate", "he": "זיקה"},
239
+ "ADV": {"en": "Adverb", "he": "תואר"},
240
+ "CONJ": {"en": "Conjunction", "he": "חיבור"},
241
+ "PREP": {"en": "Preposition", "he": "יחוס"},
242
+ "DEF": {"en": "Definite", "he": "הידוע"},
243
+ "REL": {"en": "Relative", "he": "זיקה"},
244
+ "TEMP": {"en": "Temporal", "he": "זמן"}
245
+ },
246
+ "special": {
247
+ "[UNK]": {"en": "[Unknown]", "he": "[לא ידוע]"},
248
+ "[BLANK]": {"en": "[Blank]", "he": "[ריק]"},
249
+ "[PAD]": {"en": "[Padding]", "he": "[ריפוד]"},
250
+ "[CLS]": {"en": "[Start]", "he": "[התחלה]"},
251
+ "[SEP]": {"en": "[Separator]", "he": "[הפרדה]"},
252
+ "[MASK]": {"en": "[Mask]", "he": "[מסכה]"}
253
+ },
254
+ "morph": {
255
+ # Gender
256
+ "Masc": {"en": "Masculine", "he": "זכר"},
257
+ "Fem": {"en": "Feminine", "he": "נקבה"},
258
+ # Number
259
+ "Sing": {"en": "Singular", "he": "יחיד"},
260
+ "Plur": {"en": "Plural", "he": "רבים"},
261
+ "Dual": {"en": "Dual", "he": "זוגי"},
262
+ # Person
263
+ "1": {"en": "1st", "he": "ראשון"},
264
+ "2": {"en": "2nd", "he": "שני"},
265
+ "3": {"en": "3rd", "he": "שלישי"},
266
+ # Tense
267
+ "Past": {"en": "Past", "he": "עבר"},
268
+ "Present": {"en": "Present", "he": "הווה"},
269
+ "Future": {"en": "Future", "he": "עתיד"},
270
+ "Imp": {"en": "Imperative", "he": "ציווי"},
271
+ "Inf": {"en": "Infinitive", "he": "מקור"},
272
+ # Voice
273
+ "Act": {"en": "Active", "he": "פעיל"},
274
+ "Pass": {"en": "Passive", "he": "סביל"},
275
+ # Definiteness
276
+ "Def": {"en": "Definite", "he": "מיודע"},
277
+ "Ind": {"en": "Indefinite", "he": "סתמי"},
278
+ # Case
279
+ "Nom": {"en": "Nominative", "he": "נושא"},
280
+ "Acc": {"en": "Accusative", "he": "מושא"},
281
+ "Gen": {"en": "Genitive", "he": "שייכות"},
282
+ # Construct state
283
+ "Construct": {"en": "Construct", "he": "סמיכות"},
284
+ "Free": {"en": "Free", "he": "נפרד"}
285
+ }
286
+ }
287
+
288
+
289
+ def get_label(category: str, code: str, lang: str) -> str:
290
+ """Get translated label from LABEL_TRANSLATIONS.
291
+
292
+ Args:
293
+ category: One of 'pos', 'dep', 'ner', 'prefix', 'special', 'morph'
294
+ code: The code to translate (e.g., 'NOUN', 'nsubj', 'PER', 'Masc')
295
+ lang: Target language ('en' or 'he')
296
+
297
+ Returns:
298
+ Translated label, or original code if not found
299
+ """
300
+ cat_dict = LABEL_TRANSLATIONS.get(category, {})
301
+ code_dict = cat_dict.get(code, {})
302
+ return code_dict.get(lang, code)
303
+
304
+
305
+ def translate_special_tokens(text: str, lang: str) -> str:
306
+ """Replace special tokens like [UNK] with translations."""
307
+ for code, translations in LABEL_TRANSLATIONS["special"].items():
308
+ text = text.replace(code, translations.get(lang, code))
309
+ return text
310
+
311
+
312
+ def strip_punctuation(text: str) -> str:
313
+ """Remove punctuation from text, keeping Hebrew letters, diacritics, Latin, digits, spaces.
314
+
315
+ Preserves:
316
+ - Hebrew letters: U+05D0-U+05EA
317
+ - Hebrew cantillation/accents: U+0591-U+05AF
318
+ - Hebrew diacritics (niqqud): U+05B0-U+05BD, U+05BF, U+05C1-U+05C2, U+05C4-U+05C5, U+05C7
319
+ - Hebrew geresh (׳) and gershayim (״): U+05F3-U+05F4 (for acronyms like צה"ל)
320
+ - Regular quotes " ' used in acronyms
321
+ - Latin letters, digits, spaces
322
+
323
+ Removes: punctuation like . , ! ? ; : - ־ ׃ | etc.
324
+ """
325
+ import re
326
+ # Keep Hebrew letters, diacritics, cantillation, geresh/gershayim, quotes for acronyms, Latin, digits, spaces
327
+ result = re.sub(r'[^\u05D0-\u05EA\u0591-\u05AF\u05B0-\u05BD\u05BF\u05C1\u05C2\u05C4\u05C5\u05C7\u05F3\u05F4"\'A-Za-z0-9\s]', '', text)
328
+ # Clean up multiple spaces
329
+ return re.sub(r'\s+', ' ', result).strip()
330
+
331
+
332
+ # ============================================================================
333
+ # Sentence Breaker (pysbd)
334
+ # ============================================================================
335
+
336
+ class SentenceBreaker:
337
+ """pysbd - Python Sentence Boundary Disambiguation"""
338
+
339
+ SUPPORTED = {'es', 'fa', 'ar', 'sk', 'hy', 'am', 'en', 'fr', 'ru', 'my',
340
+ 'hi', 'pl', 'it', 'ja', 'de', 'zh', 'kk', 'nl', 'da', 'el',
341
+ 'mr', 'bg', 'ur'}
342
+
343
+ def __init__(self, language: str = "he"):
344
+ import pysbd
345
+ lang = language if language in self.SUPPORTED else "en"
346
+ self._segmenter = pysbd.Segmenter(language=lang, clean=False)
347
+ self._actual_lang = lang
348
+
349
+ @property
350
+ def name(self) -> str:
351
+ if self._actual_lang and self._actual_lang != "he":
352
+ return f"pysbd-{self._actual_lang}"
353
+ return "pysbd"
354
+
355
+ def break_sentences(self, text: str) -> list[str]:
356
+ if not text.strip():
357
+ return []
358
+ sentences = self._segmenter.segment(text)
359
+ return [s.strip() for s in sentences if s.strip()]
360
+
361
+
362
+ # ============================================================================
363
+ # Async Parallel Processor
364
+ # ============================================================================
365
+
366
+ def generate_tree_svg(tokens: list, rtl: bool = True) -> str:
367
+ """Generate SVG string for dependency tree (no UI translation, raw labels)."""
368
+ if not tokens:
369
+ return ""
370
+
371
+ dot = Digraph(engine='dot', format='svg')
372
+ dot.attr('graph', rankdir='LR', rank='same', size=f'{len(tokens)},5', dpi='300')
373
+ dot.attr('node', fontname='Arial')
374
+
375
+ n = len(tokens)
376
+
377
+ # Create nodes with nikud (diacritized text)
378
+ for i, tok in enumerate(tokens):
379
+ word = tok.get("nikud", tok.get("token", ""))
380
+ dot.node(str(i), word)
381
+
382
+ # Layout direction
383
+ if rtl:
384
+ # RTL: first word on right
385
+ for i in range(n - 1):
386
+ dot.edge(str(i + 1), str(i), style='invis')
387
+ else:
388
+ # LTR: first word on left
389
+ for i in range(n - 1):
390
+ dot.edge(str(i), str(i + 1), style='invis')
391
+
392
+ # Dependency edges
393
+ for i, tok in enumerate(tokens):
394
+ syntax = tok.get("syntax", {})
395
+ head_idx = syntax.get("dep_head_idx", -1)
396
+ dep_func = syntax.get("dep_func", "")
397
+
398
+ if head_idx >= 0:
399
+ dot.edge(str(head_idx), str(i), label=dep_func, constraint='False')
400
+
401
+ return dot.pipe(format='svg').decode('utf-8')
402
+
403
+
404
+ @dataclass
405
+ class SentenceResult:
406
+ """Result for a single sentence"""
407
+ index: int
408
+ text: str
409
+ phonetics: dict
410
+ tokens: list
411
+ ner_entities: list
412
+ audio: Optional[dict] = None
413
+
414
+
415
+ class AsyncHebrewNLP:
416
+ """Async parallel Hebrew NLP pipeline"""
417
+
418
+ VERSION = "2025.12.5"
419
+
420
+ def __init__(
421
+ self,
422
+ phonikud_model: str = "./onnx/phonikud-1.0.int8.onnx",
423
+ dictabert_model: str = "dicta-il/dictabert-joint",
424
+ hf_token: Optional[str] = None,
425
+ max_workers: int = 4,
426
+ ):
427
+ self.sentence_breaker = SentenceBreaker()
428
+
429
+ # Auto-download phonikud model if not found
430
+ from pathlib import Path
431
+ phonikud_path = Path(phonikud_model)
432
+ if not phonikud_path.exists():
433
+ phonikud_path.parent.mkdir(parents=True, exist_ok=True)
434
+ from huggingface_hub import hf_hub_download
435
+ hf_hub_download(
436
+ repo_id="thewh1teagle/phonikud-onnx",
437
+ filename="phonikud-1.0.int8.onnx",
438
+ local_dir=phonikud_path.parent,
439
+ )
440
+
441
+ self.phonikud = Phonikud(str(phonikud_path))
442
+ self.piper = PiperVoiceManager() # Unified voice manager (Hebrew + English, lazy-load)
443
+
444
+ self.tokenizer = AutoTokenizer.from_pretrained(
445
+ dictabert_model, token=hf_token
446
+ )
447
+ self.dictabert = AutoModel.from_pretrained(
448
+ dictabert_model, token=hf_token, trust_remote_code=True
449
+ )
450
+ self.dictabert.eval()
451
+
452
+ self.executor = ThreadPoolExecutor(max_workers=max_workers)
453
+
454
+ self._model_info = {
455
+ "phonikud": phonikud_path.name.replace(".onnx", ""),
456
+ "piper": "piper-voices (he_IL + en)",
457
+ "dictabert": dictabert_model
458
+ }
459
+
460
+ async def process(
461
+ self,
462
+ text: str,
463
+ include_audio: bool = True,
464
+ compute_mst: bool = True,
465
+ tts_params: Optional[dict] = None,
466
+ en_voice: str = "en_US-ryan-high",
467
+ ) -> dict:
468
+ start_time = time.perf_counter()
469
+
470
+ tts_params = tts_params or {
471
+ "length_scale": 1.20,
472
+ "noise_scale": 0.640,
473
+ "noise_w": 1.0
474
+ }
475
+
476
+ sentences = self.sentence_breaker.break_sentences(text)
477
+
478
+ if not sentences:
479
+ return self._empty_result(text, start_time)
480
+
481
+ loop = asyncio.get_event_loop()
482
+
483
+ tasks = [
484
+ loop.run_in_executor(
485
+ self.executor,
486
+ self._process_sentence,
487
+ idx, sentence, compute_mst, include_audio, tts_params, en_voice
488
+ )
489
+ for idx, sentence in enumerate(sentences)
490
+ ]
491
+
492
+ sentence_results = await asyncio.gather(*tasks)
493
+
494
+ processing_time = (time.perf_counter() - start_time) * 1000
495
+
496
+ return {
497
+ "meta": {
498
+ "version": self.VERSION,
499
+ "timestamp": datetime.now(timezone.utc).isoformat(),
500
+ "models": self._model_info,
501
+ "processing_time_ms": round(processing_time, 2),
502
+ "sentence_count": len(sentences),
503
+ "parallel_workers": self.executor._max_workers,
504
+ "sentence_breaker": self.sentence_breaker.name
505
+ },
506
+ "translations": LABEL_TRANSLATIONS,
507
+ "input": {
508
+ "text": text,
509
+ "language": "he"
510
+ },
511
+ "sentences": [self._sentence_to_dict(r) for r in sentence_results]
512
+ }
513
+
514
+ def process_sync(
515
+ self,
516
+ text: str,
517
+ include_audio: bool = True,
518
+ compute_mst: bool = True,
519
+ tts_params: Optional[dict] = None,
520
+ en_voice: str = "en_US-ryan-high",
521
+ ) -> dict:
522
+ loop = asyncio.new_event_loop()
523
+ asyncio.set_event_loop(loop)
524
+ try:
525
+ return loop.run_until_complete(
526
+ self.process(text, include_audio, compute_mst, tts_params, en_voice)
527
+ )
528
+ finally:
529
+ loop.close()
530
+
531
+ def _process_sentence(
532
+ self,
533
+ index: int,
534
+ sentence: str,
535
+ compute_mst: bool,
536
+ include_audio: bool,
537
+ tts_params: dict,
538
+ en_voice: str = "en_US-ryan-high"
539
+ ) -> SentenceResult:
540
+ # Detect language for this sentence
541
+ lang = detect_script(sentence)
542
+
543
+ if lang == 'he':
544
+ # Hebrew: use phonikud for diacritics and phonemes
545
+ diacritized = self.phonikud.add_diacritics(sentence)
546
+ phonemes = phonemize(diacritized)
547
+ diacritized = diacritized.replace('|', '') # Remove pipe markers after phonemize
548
+ else:
549
+ # English/other: no diacritics or phonemes
550
+ diacritized = sentence
551
+ phonemes = ''
552
+
553
+ with torch.no_grad():
554
+ dictabert_result = self.dictabert.predict(
555
+ [sentence], # Use original sentence for DictaBERT
556
+ self.tokenizer,
557
+ compute_syntax_mst=compute_mst,
558
+ output_style='json'
559
+ )[0]
560
+
561
+ tokens = self._enrich_tokens(
562
+ dictabert_result.get('tokens', []),
563
+ diacritized,
564
+ phonemes
565
+ )
566
+
567
+ audio = None
568
+ if include_audio:
569
+ # Hebrew uses phonemes, English uses text
570
+ speech_text = phonemes if lang == 'he' else sentence
571
+ audio = self._generate_audio(speech_text, tts_params, lang, en_voice)
572
+
573
+ return SentenceResult(
574
+ index=index,
575
+ text=sentence,
576
+ phonetics={"diacritized": diacritized, "phonemes": phonemes},
577
+ tokens=tokens,
578
+ ner_entities=dictabert_result.get('ner_entities', []),
579
+ audio=audio
580
+ )
581
+
582
+ def _enrich_tokens(
583
+ self,
584
+ dictabert_tokens: list,
585
+ diacritized: str,
586
+ phonemes: str
587
+ ) -> list:
588
+ """Enrich tokens with nikud and phonemes - align by skipping punctuation."""
589
+ # Split nikud words, also splitting on hyphens and slashes within words
590
+ nikud_words = []
591
+ for word in diacritized.split():
592
+ # Split on - and / but keep them as separate elements
593
+ import re
594
+ parts = re.split(r'([-/])', word)
595
+ for part in parts:
596
+ if part: # Skip empty strings
597
+ nikud_words.append(part)
598
+
599
+ phoneme_parts = phonemes.split(" | ") if " | " in phonemes else phonemes.split()
600
+
601
+ enriched = []
602
+ nikud_idx = 0 # Separate index for nikud words
603
+ phoneme_idx = 0 # Separate index for phonemes
604
+
605
+ for tok in dictabert_tokens:
606
+ token_text = tok.get('token', '')
607
+ pos = tok.get('morph', {}).get('pos', '')
608
+
609
+ # Treat PUNCT and SYM as punctuation (don't consume nikud words)
610
+ if pos in ('PUNCT', 'SYM'):
611
+ # Punctuation/symbol token - use original token
612
+ nikud = token_text
613
+ # Skip nikud_idx if it points to matching punctuation/symbol
614
+ if nikud_idx < len(nikud_words) and nikud_words[nikud_idx] == token_text:
615
+ nikud_idx += 1
616
+ token_phonemes = ''
617
+ else:
618
+ # Content token - get nikud from list
619
+ if nikud_idx < len(nikud_words):
620
+ nikud = strip_punctuation(nikud_words[nikud_idx])
621
+ nikud_idx += 1
622
+ else:
623
+ nikud = token_text
624
+
625
+ # Get phonemes
626
+ if phoneme_idx < len(phoneme_parts):
627
+ token_phonemes = phoneme_parts[phoneme_idx]
628
+ phoneme_idx += 1
629
+ else:
630
+ token_phonemes = ''
631
+
632
+ # Fallback if nikud became empty
633
+ if not nikud:
634
+ nikud = token_text
635
+
636
+ enriched.append({
637
+ "id": len(enriched) + 1,
638
+ "token": token_text,
639
+ "nikud": nikud,
640
+ "phonemes": token_phonemes,
641
+ "seg": tok.get('seg', []),
642
+ "lex": tok.get('lex', ''),
643
+ "offsets": tok.get('offsets', {}),
644
+ "morph": tok.get('morph', {}),
645
+ "syntax": tok.get('syntax', {})
646
+ })
647
+
648
+ return enriched
649
+
650
+ def _generate_audio(self, text: str, tts_params: dict, lang: str = 'he', en_voice: str = "en_US-ryan-high") -> dict:
651
+ """Generate speech audio for Hebrew (phonemes) or English (text)."""
652
+ if lang == 'he':
653
+ voice_name = "he_IL-phonikud"
654
+ is_phonemes = True
655
+ else:
656
+ voice_name = en_voice
657
+ is_phonemes = False
658
+
659
+ piper = self.piper.get(voice_name)
660
+
661
+ samples, rate = piper.create(
662
+ text,
663
+ is_phonemes=is_phonemes,
664
+ length_scale=tts_params.get('length_scale', 1.20),
665
+ noise_scale=tts_params.get('noise_scale', 0.640),
666
+ noise_w=tts_params.get('noise_w', 1.0)
667
+ )
668
+ samples = np.clip(samples * 2.0, -1.0, 1.0)
669
+
670
+ buf = BytesIO()
671
+ sf.write(buf, samples, rate, format="WAV")
672
+ buf.seek(0)
673
+ b64 = base64.b64encode(buf.read()).decode()
674
+
675
+ return {
676
+ "format": "wav",
677
+ "sample_rate": rate,
678
+ "duration_ms": round(len(samples) / rate * 1000, 2),
679
+ "data_uri": f"data:audio/wav;base64,{b64}"
680
+ }
681
+
682
+ def _sentence_to_dict(self, result: SentenceResult) -> dict:
683
+ lang = detect_script(result.text) # Detect per sentence
684
+ return {
685
+ "index": result.index,
686
+ "lang": lang,
687
+ "text": result.text,
688
+ "phonetics": result.phonetics,
689
+ "tokens": result.tokens,
690
+ "ner_entities": result.ner_entities,
691
+ "audio": result.audio,
692
+ "tree_svg": generate_tree_svg(result.tokens, rtl=(lang == "he"))
693
+ }
694
+
695
+ def _empty_result(self, text: str, start_time: float) -> dict:
696
+ return {
697
+ "meta": {
698
+ "version": self.VERSION,
699
+ "timestamp": datetime.now(timezone.utc).isoformat(),
700
+ "models": self._model_info,
701
+ "processing_time_ms": round((time.perf_counter() - start_time) * 1000, 2),
702
+ "sentence_count": 0,
703
+ "parallel_workers": self.executor._max_workers,
704
+ "sentence_breaker": self.sentence_breaker.name
705
+ },
706
+ "input": {"text": text, "language": "he"},
707
+ "sentences": []
708
+ }
709
+
710
+
711
+ # ============================================================================
712
+ # Visualization Functions
713
+ # ============================================================================
714
+
715
+ def display_sentence_tree(tokens: list, sentence_idx: int, rtl: bool = None):
716
+ if not tokens:
717
+ return
718
+
719
+ dot = Digraph(engine='dot', format='svg')
720
+ dot.attr('graph', rankdir='LR', rank='same', size=f'{len(tokens)},5', dpi='300')
721
+ dot.attr('node', fontname='Arial')
722
+
723
+ n = len(tokens)
724
+
725
+ # Use passed rtl param, or fall back to UI language
726
+ if rtl is None:
727
+ rtl_mode = st.session_state.get('ui_lang', 'he') == 'he'
728
+ else:
729
+ rtl_mode = rtl
730
+
731
+ # Get UI language for translations
732
+ ui_lang = st.session_state.get('ui_lang', 'he')
733
 
734
+ # Create nodes with nikud (diacritized text)
735
+ for i, tok in enumerate(tokens):
736
+ word = tok.get("nikud", tok.get("token", ""))
737
+ word = translate_special_tokens(word, ui_lang)
738
+ dot.node(str(i), word)
739
 
740
+ if rtl_mode:
741
+ # RTL: Invisible edges high->low puts first word on RIGHT
742
+ for i in range(n - 1):
743
+ dot.edge(str(i + 1), str(i), style='invis')
744
+ else:
745
+ # LTR: Invisible edges low->high puts first word on LEFT
746
+ for i in range(n - 1):
747
+ dot.edge(str(i), str(i + 1), style='invis')
748
+
749
+ # Dependency edges with translated labels
750
+ for i, tok in enumerate(tokens):
751
+ syntax = tok.get("syntax", {})
752
+ head_idx = syntax.get("dep_head_idx", -1)
753
+ dep_func = syntax.get("dep_func", "")
754
+
755
+ if head_idx >= 0:
756
+ label = get_label("dep", dep_func, ui_lang)
757
+ dot.edge(str(head_idx), str(i), label=label, constraint='False')
758
+
759
+ st.markdown(
760
+ f"""<div style="height:200px; overflow:auto; border:1px solid #ddd;
761
+ border-radius:8px; padding:8px; background:#fafafa;">
762
+ <img src="data:image/svg+xml;base64,{base64.b64encode(dot.pipe(format='svg')).decode()}"
763
+ style="display:block; margin:auto; max-height:180px;">
764
+ </div>""", unsafe_allow_html=True)
765
+
766
+
767
+ def display_tokens_table(tokens: list):
768
+ if not tokens:
769
+ return
770
+
771
+ # Get UI language
772
+ ui_lang = "he" if is_rtl() else "en"
773
+
774
+ rows = []
775
+ for tok in tokens:
776
+ morph = tok.get('morph', {})
777
+ syntax = tok.get('syntax', {})
778
+ pos = morph.get('pos', '-')
779
+ rel = syntax.get('dep_func', '-')
780
+ prefixes = morph.get('prefixes', [])
781
+
782
+ # Get token values
783
+ token_text = tok['token']
784
+ nikud_text = tok['nikud']
785
+ phonemes_text = tok['phonemes']
786
+ lemma_text = tok['lex']
787
+
788
+ # Translate special tokens and labels based on UI language
789
+ token_text = translate_special_tokens(token_text, ui_lang)
790
+ nikud_text = translate_special_tokens(nikud_text, ui_lang)
791
+ lemma_text = translate_special_tokens(lemma_text, ui_lang)
792
+
793
+ display_pos = get_label("pos", pos, ui_lang)
794
+ display_rel = get_label("dep", rel, ui_lang)
795
+ display_prefixes = ', '.join(get_label("prefix", p, ui_lang) for p in prefixes) or '-'
796
+
797
+ rows.append({
798
+ "#": tok['id'],
799
+ t("token"): token_text,
800
+ t("nikud"): nikud_text,
801
+ t("phonemes"): phonemes_text,
802
+ t("lemma"): lemma_text,
803
+ t("pos"): display_pos,
804
+ t("prefixes"): display_prefixes,
805
+ t("head"): syntax.get('dep_head_idx', -1) + 1,
806
+ t("rel"): display_rel
807
+ })
808
+
809
+ st.dataframe(rows, width="stretch")
810
+
811
+
812
+ def display_ner_entities(entities: list):
813
+ if not entities:
814
+ st.info(t("no_entities"))
815
+ return
816
+
817
+ # Dark theme colors with light text
818
+ colors = {
819
+ 'PER': '#c2185b', 'LOC': '#2e7d32', 'ORG': '#1565c0',
820
+ 'GPE': '#2e7d32', 'TIME': '#f57c00', 'DATE': '#f57c00', 'MISC': '#616161'
821
+ }
822
+
823
+ # Get UI language
824
+ ui_lang = "he" if is_rtl() else "en"
825
+
826
+ html = ""
827
+ for ent in entities:
828
+ label = ent.get('label', 'MISC')
829
+ phrase = ent.get('phrase', '')
830
+ color = colors.get(label, '#616161')
831
+ display_label = get_label("ner", label, ui_lang)
832
+ html += f"""<span style="background:{color}; color:white; padding:4px 8px;
833
+ border-radius:4px; margin:2px; display:inline-block; font-weight:500;">
834
+ {phrase} <small>[{display_label}]</small></span>"""
835
+
836
+ st.markdown(html, unsafe_allow_html=True)
837
+
838
+
839
+ # ============================================================================
840
+ # Internationalization (i18n)
841
+ # ============================================================================
842
+
843
+ TRANSLATIONS = {
844
+ "he": {
845
+ "title": "🇮🇱 עיבוד שפה עברית מאוחד",
846
+ "subtitle": "צינור מקבילי אסינכרוני | שבירת משפטים חוצה-פלטפורמה | ארכיטקטורת Map-Reduce",
847
+ "settings": "⚙️ הגדרות",
848
+ "language": "🌐 שפה",
849
+ "compute_mst": "צור עץ תחבירי",
850
+ "generate_audio": "צור דיבור",
851
+ "tts_params": "הגדרות דיבור",
852
+ "en_voice": "🇬🇧 קול אנגלית",
853
+ "speed": "מהירות",
854
+ "noise": "רעש",
855
+ "noise_w": "רעש W",
856
+ "sentence_pause": "השהיה בין משפטים (שניות)",
857
+ "sentence_breaker": "מפצל משפטים",
858
+ "available_backends": "מנועים זמינים",
859
+ "load_example": "📚 טען דוגמה:",
860
+ "enter_text": "הזן טקסט (תמיכה במספר משפטים):",
861
+ "placeholder": "הקלד טקסט...",
862
+ "analyze": "🔍 נתח",
863
+ "processing": "מעבד משפטים במקביל...",
864
+ "enter_text_warning": "אנא הזן טקסט לניתוח.",
865
+ "sentences": "משפטים",
866
+ "processing_time": "זמן עיבוד",
867
+ "workers": "עובדים",
868
+ "breaker": "מפצל",
869
+ "sentence": "משפט",
870
+ "original": "מקור:",
871
+ "nikud": "ניקוד:",
872
+ "phonemes": "פונמות:",
873
+ "dep_tree": "🌳 עץ תלויות",
874
+ "all_trees": "🌳 כל העצים",
875
+ "sentence_json": "📄 JSON משפט",
876
+ "morphology": "🔠 מורפולוגיה",
877
+ "ner": "🏷️ ישויות",
878
+ "audio": "🔊 אודיו",
879
+ "speech": "🔊 דיבור",
880
+ "no_audio": "אין אודיו - הפעל 'צור דיבור' בהגדרות",
881
+ "duration": "משך",
882
+ "json_output": "📄 JSON",
883
+ "full_json": "פלט JSON מלא",
884
+ "download_json": "⬇️ הורד JSON",
885
+ "no_entities": "אין ישויות",
886
+ "models_not_loaded": "המודלים לא נטענו.",
887
+ "token": "טוקן",
888
+ "lemma": "למה",
889
+ "pos": "חלק דיבר",
890
+ "prefixes": "תחיליות",
891
+ "head": "ראש",
892
+ "rel": "יחס",
893
+ "play_all": "נגן הכל",
894
+ "stop": "עצור",
895
+ "playing": "מנגן משפט",
896
+ "finished": "✓ סיום",
897
+ "stopped": "נעצר",
898
+ "download_all_audio": "הורד הכל (WAV)",
899
+ "download_audio_files": "הורד קבצי אודיו",
900
+ "breaking_sentences": "מפצל משפטים...",
901
+ "processing_sentence": "מעבד משפט",
902
+ "finalizing": "מסיים...",
903
+ "done": "✓ הסתיים",
904
+ "no_sentences": "לא נמצאו משפטים",
905
+ "loading_models": "טוען מודלים...",
906
+ "loading_dictabert": "טוען DictaBERT...",
907
+ "loading_phonikud": "טוען Phonikud...",
908
+ "loading_piper": "טוען Piper TTS...",
909
+ },
910
+ "en": {
911
+ "title": "🇮🇱 Hebrew Unified NLP",
912
+ "subtitle": "Async Parallel Pipeline | Cross-Platform Sentence Breaking | Map-Reduce Architecture",
913
+ "settings": "⚙️ Settings",
914
+ "language": "🌐 Language",
915
+ "compute_mst": "Create Syntax Tree",
916
+ "generate_audio": "Generate Audio",
917
+ "tts_params": "TTS Parameters",
918
+ "en_voice": "🇬🇧 English Voice",
919
+ "speed": "Speed",
920
+ "noise": "Noise",
921
+ "noise_w": "Noise W",
922
+ "sentence_pause": "Pause between sentences (sec)",
923
+ "sentence_breaker": "Sentence Breaker",
924
+ "available_backends": "Available backends",
925
+ "load_example": "📚 Load Example:",
926
+ "enter_text": "Enter text (multiple sentences supported):",
927
+ "placeholder": "Enter text...",
928
+ "analyze": "🔍 Analyze",
929
+ "processing": "Processing sentences in parallel...",
930
+ "enter_text_warning": "Please enter some text to analyze.",
931
+ "sentences": "Sentences",
932
+ "processing_time": "Processing",
933
+ "workers": "Workers",
934
+ "breaker": "Breaker",
935
+ "sentence": "Sentence",
936
+ "original": "Original:",
937
+ "nikud": "Nikud:",
938
+ "phonemes": "Phonemes:",
939
+ "dep_tree": "🌳 Dependency Tree",
940
+ "all_trees": "🌳 All Trees",
941
+ "sentence_json": "📄 Sentence JSON",
942
+ "morphology": "🔠 Morphology",
943
+ "ner": "🏷️ Named Entities",
944
+ "audio": "🔊 Audio",
945
+ "speech": "🔊 Speech",
946
+ "no_audio": "No audio - enable 'Generate speech' in settings",
947
+ "duration": "Duration",
948
+ "json_output": "📄 JSON",
949
+ "full_json": "Full JSON Output",
950
+ "download_json": "⬇️ Download JSON",
951
+ "no_entities": "No named entities",
952
+ "models_not_loaded": "Models not loaded.",
953
+ "token": "Token",
954
+ "lemma": "Lemma",
955
+ "pos": "POS",
956
+ "prefixes": "Prefixes",
957
+ "head": "Head",
958
+ "rel": "Rel",
959
+ "play_all": "Play All",
960
+ "stop": "Stop",
961
+ "playing": "Playing sentence",
962
+ "finished": "✓ Finished",
963
+ "stopped": "Stopped",
964
+ "download_all_audio": "Download All (WAV)",
965
+ "download_audio_files": "Download Audio Files",
966
+ "breaking_sentences": "Breaking sentences...",
967
+ "processing_sentence": "Processing sentence",
968
+ "finalizing": "Finalizing...",
969
+ "done": "✓ Done",
970
+ "no_sentences": "No sentences found",
971
+ "loading_models": "Loading models...",
972
+ "loading_dictabert": "Loading DictaBERT...",
973
+ "loading_phonikud": "Loading Phonikud...",
974
+ "loading_piper": "Loading Piper TTS...",
975
+ }
976
+ }
977
+
978
+ def t(key: str) -> str:
979
+ """Get translated string for current language"""
980
+ lang = st.session_state.get('ui_lang', 'he')
981
+ return TRANSLATIONS.get(lang, TRANSLATIONS['en']).get(key, key)
982
+
983
+ def is_rtl() -> bool:
984
+ """Check if current language is RTL"""
985
+ return st.session_state.get('ui_lang', 'he') == 'he'
986
+
987
+
988
+ # ============================================================================
989
+ # Streamlit App
990
+ # ============================================================================
991
+
992
+ st.set_page_config(
993
+ page_title="Hebrew Unified NLP",
994
+ page_icon="🇮🇱",
995
+ layout="wide"
996
+ )
997
+
998
+ # Initialize language
999
+ if 'ui_lang' not in st.session_state:
1000
+ st.session_state['ui_lang'] = 'he'
1001
+
1002
+ # Dynamic RTL/LTR CSS based on language
1003
+ if is_rtl():
1004
+ st.markdown("""
1005
+ <style>
1006
+ /* Text inputs RTL */
1007
+ textarea { direction: rtl !important; text-align: right !important; font-family: 'David', 'Noto Sans Hebrew', sans-serif !important; }
1008
+ input[type="text"] { direction: rtl !important; text-align: right !important; }
1009
+
1010
+ /* Tables RTL */
1011
+ .stDataFrame td, .stDataFrame th { direction: rtl !important; text-align: right !important; }
1012
+
1013
+ /* Main container RTL */
1014
+ .main .block-container { direction: rtl; }
1015
+
1016
+ /* All text elements RTL */
1017
+ h1, h2, h3, p, label, .stMarkdown, .stText { direction: rtl !important; text-align: right !important; }
1018
+
1019
+ /* Sidebar RTL */
1020
+ [data-testid="stSidebar"] { direction: rtl; }
1021
+ [data-testid="stSidebar"] label { direction: rtl !important; text-align: right !important; display: block !important; }
1022
+ [data-testid="stSidebar"] .stMarkdown { direction: rtl !important; text-align: right !important; }
1023
+ [data-testid="stSidebar"] h1, [data-testid="stSidebar"] h2, [data-testid="stSidebar"] h3 { direction: rtl !important; text-align: right !important; }
1024
+
1025
+ /* Selectbox and other widget labels */
1026
+ .stSelectbox label, .stTextArea label, .stSlider label, .stCheckbox label {
1027
+ direction: rtl !important;
1028
+ text-align: right !important;
1029
+ display: block !important;
1030
+ width: 100% !important;
1031
+ }
1032
+
1033
+ /* Selectbox dropdown RTL */
1034
+ .stSelectbox > div > div { direction: rtl !important; text-align: right !important; }
1035
+ .stSelectbox [data-baseweb="select"] { direction: rtl !important; }
1036
+ .stSelectbox [data-baseweb="select"] > div { direction: rtl !important; text-align: right !important; }
1037
+ [data-baseweb="popover"] { direction: rtl !important; }
1038
+ [data-baseweb="menu"] { direction: rtl !important; }
1039
+ [role="listbox"] { direction: rtl !important; }
1040
+ [role="option"] { direction: rtl !important; text-align: right !important; }
1041
+
1042
+ /* Metrics RTL */
1043
+ [data-testid="stMetricLabel"] { direction: rtl !important; text-align: right !important; }
1044
+
1045
+ /* Expander headers RTL */
1046
+ .streamlit-expanderHeader { direction: rtl !important; text-align: right !important; }
1047
+ [data-testid="stExpander"] { direction: rtl !important; }
1048
+ [data-testid="stExpander"] summary { direction: rtl !important; text-align: right !important; }
1049
+ [data-testid="stExpander"] summary span { direction: rtl !important; }
1050
+ details summary { direction: rtl !important; text-align: right !important; justify-content: flex-end !important; }
1051
+
1052
+ /* Tabs RTL */
1053
+ .stTabs [data-baseweb="tab-list"] { direction: rtl; }
1054
+ </style>
1055
+ """, unsafe_allow_html=True)
1056
+ else:
1057
+ st.markdown("""
1058
+ <style>
1059
+ textarea { direction: ltr !important; text-align: left !important; }
1060
+ input[type="text"] { direction: ltr !important; text-align: left !important; }
1061
+ </style>
1062
+ """, unsafe_allow_html=True)
1063
+
1064
+ st.title(t("title"))
1065
+ st.markdown(t("subtitle"))
1066
+
1067
+
1068
+ @st.cache_resource
1069
+ def load_models_cached():
1070
  try:
1071
+ hf_token = st.secrets.get("HF_TOKEN", None)
 
1072
  except Exception:
1073
+ hf_token = None
1074
+ return AsyncHebrewNLP(hf_token=hf_token, max_workers=4)
1075
+
1076
+
1077
+ # Load models with progress
1078
+ if 'models_loaded' not in st.session_state:
1079
+ st.session_state['models_loaded'] = False
1080
+ st.session_state['nlp'] = None
1081
+
1082
+ if not st.session_state['models_loaded']:
1083
+ try:
1084
+ progress = st.progress(0, text=t("loading_models"))
1085
+ progress.progress(20, text=t("loading_dictabert"))
1086
+ nlp = load_models_cached()
1087
+ progress.progress(60, text=t("loading_phonikud"))
1088
+ progress.progress(80, text=t("loading_piper"))
1089
+ progress.progress(100, text=t("done"))
1090
+ progress.empty()
1091
+
1092
+ st.session_state['nlp'] = nlp
1093
+ st.session_state['models_loaded'] = True
1094
+ models_loaded = True
1095
+ except Exception as e:
1096
+ models_loaded = False
1097
+ st.error(f"Failed to load models: {e}")
1098
+ else:
1099
+ nlp = st.session_state['nlp']
1100
+ models_loaded = True
1101
+
1102
+ if models_loaded:
1103
+ # Sidebar
1104
+ with st.sidebar:
1105
+ st.header(t("settings"))
1106
+
1107
+ # Language selector - radio buttons
1108
+ lang_options = {"🇮🇱 עברית": "he", "🇬🇧 English": "en"}
1109
+ current_lang = st.session_state.get('ui_lang', 'he')
1110
+
1111
+ def change_language():
1112
+ selected = st.session_state.get('lang_radio', '🇮🇱 עברית')
1113
+ new_lang = lang_options[selected]
1114
+ old_lang = st.session_state.get('ui_lang', 'he')
1115
+ st.session_state['ui_lang'] = new_lang
1116
+ # Reset text and example when language changes
1117
+ if new_lang != old_lang:
1118
+ st.session_state['example_idx'] = 0
1119
+ if new_lang == 'he':
1120
+ st.session_state['text_input'] = "הילד הלך לבית הספר. הוא למד מתמטיקה ועברית. בערב הוא חזר הביתה."
1121
+ else:
1122
+ st.session_state['text_input'] = "The boy went to school. He studied math and Hebrew. In the evening he came home."
1123
+ # Clear results when language changes
1124
+ if 'result' in st.session_state:
1125
+ del st.session_state['result']
1126
+
1127
+ st.radio(
1128
+ t("language"),
1129
+ list(lang_options.keys()),
1130
+ index=0 if current_lang == "he" else 1,
1131
+ key="lang_radio",
1132
+ on_change=change_language,
1133
+ horizontal=True
1134
+ )
1135
+
1136
+ st.divider()
1137
+
1138
+ compute_mst = st.checkbox(t("compute_mst"), value=True)
1139
+ include_audio = st.checkbox(t("generate_audio"), value=True)
1140
+
1141
+ st.subheader(t("tts_params"))
1142
+
1143
+ # English voice selector (filter PIPER_VOICES for en_* voices)
1144
+ en_voices = sorted([v for v in PIPER_VOICES.keys() if v.startswith("en_")])
1145
+ default_voice = "en_US-ryan-high"
1146
+ try:
1147
+ default_idx = en_voices.index(default_voice)
1148
+ except ValueError:
1149
+ default_idx = 0 # Fallback to first voice
1150
+ en_voice = st.selectbox(
1151
+ t("en_voice"),
1152
+ en_voices,
1153
+ index=default_idx
1154
+ )
1155
+
1156
+ length_scale = st.slider(t("speed"), 0.5, 2.0, 1.20, 0.05)
1157
+ noise_scale = st.slider(t("noise"), 0.0, 1.0, 0.640, 0.01)
1158
+ noise_w = st.slider(t("noise_w"), 0.0, 2.0, 1.0, 0.1)
1159
+ sentence_pause = st.slider(t("sentence_pause"), 0.1, 1.0, 0.5, 0.1)
1160
+
1161
+ # ========== MAIN INPUT SECTION ==========
1162
+
1163
+ # Example options (bilingual)
1164
+ example_options_he = [
1165
+ "הילד הלך לבית הספר. הוא למד מתמטיקה ועברית. בערב הוא חזר הביתה.",
1166
+ "דוד בן-גוריון נולד בפלונסק. הוא עלה לארץ ישראל בשנת 1906. בשנת 1948 הכריז על הקמת המדינה.",
1167
+ "השמש זורחת במזרח. היא שוקעת במערב. זה קורה כל יום.",
1168
+ ]
1169
+
1170
+ example_options_en = [
1171
+ "The boy went to school. He studied math and Hebrew. In the evening he came home.",
1172
+ "David Ben-Gurion was born in Plonsk. He immigrated to Israel in 1906. In 1948 he declared the establishment of the state.",
1173
+ "The sun rises in the east. It sets in the west. This happens every day.",
1174
+ ]
1175
+
1176
+ # Switch examples based on UI language
1177
+ example_options = example_options_he if is_rtl() else example_options_en
1178
+
1179
+ # Callback to load example
1180
+ def load_example():
1181
+ idx = st.session_state.get('example_idx', 0)
1182
+ # Get examples for current language
1183
+ examples = example_options_he if st.session_state.get('ui_lang', 'he') == 'he' else example_options_en
1184
+ if idx < len(examples):
1185
+ st.session_state['text_input'] = examples[idx]
1186
+
1187
+ # Initialize text input
1188
+ if 'text_input' not in st.session_state:
1189
+ st.session_state['text_input'] = example_options[0]
1190
+
1191
+ # Check if currently processing
1192
+ is_processing = st.session_state.get('processing', False)
1193
+
1194
+ # Example selector - single label approach (disabled while processing)
1195
+ st.selectbox(
1196
+ t('load_example'),
1197
+ range(len(example_options)),
1198
+ format_func=lambda i: example_options[i][:50] + "...",
1199
+ key="example_idx",
1200
+ on_change=load_example,
1201
+ disabled=is_processing
1202
+ )
1203
+
1204
+ # Text input (disabled while processing)
1205
+ text = st.text_area(
1206
+ t('enter_text'),
1207
+ height=200,
1208
+ placeholder=t("placeholder"),
1209
+ key="text_input",
1210
+ disabled=is_processing
1211
+ )
1212
+
1213
+ # Analyze button (disabled while processing)
1214
+ if st.button(t("analyze"), type="primary", width="stretch", disabled=is_processing):
1215
+ if text.strip():
1216
+ st.session_state['processing'] = True
1217
+ st.rerun()
1218
+ else:
1219
+ st.warning(t("enter_text_warning"))
1220
+
1221
+ # Process with progress bar
1222
+ if st.session_state.get('processing', False) and text.strip():
1223
+ progress_bar = st.progress(0, text=t("processing"))
1224
+ status_text = st.empty()
1225
+
1226
+ try:
1227
+ # Step 1: Break sentences
1228
+ progress_bar.progress(10, text=t("breaking_sentences"))
1229
+ sentences = nlp.sentence_breaker.break_sentences(text)
1230
+
1231
+ if not sentences:
1232
+ st.warning(t("no_sentences"))
1233
+ st.session_state['processing'] = False
1234
+ st.rerun()
1235
+
1236
+ # Step 2: Process each sentence
1237
+ sentence_results = []
1238
+ for idx, sentence in enumerate(sentences):
1239
+ progress_pct = 10 + int(80 * (idx + 1) / len(sentences))
1240
+ status_text.text(f"{t('processing_sentence')} {idx + 1}/{len(sentences)}")
1241
+ progress_bar.progress(progress_pct, text=f"{t('processing_sentence')} {idx + 1}/{len(sentences)}")
1242
+
1243
+ result = nlp._process_sentence(
1244
+ idx, sentence, compute_mst, include_audio,
1245
+ {
1246
+ "length_scale": length_scale,
1247
+ "noise_scale": noise_scale,
1248
+ "noise_w": noise_w
1249
+ },
1250
+ en_voice=en_voice
1251
+ )
1252
+ sentence_results.append(result)
1253
+
1254
+ # Step 3: Build final result
1255
+ progress_bar.progress(95, text=t("finalizing"))
1256
+
1257
+ result = {
1258
+ "meta": {
1259
+ "version": nlp.VERSION,
1260
+ "timestamp": datetime.now(timezone.utc).isoformat(),
1261
+ "models": nlp._model_info,
1262
+ "processing_time_ms": 0,
1263
+ "sentence_count": len(sentences),
1264
+ "parallel_workers": nlp.executor._max_workers,
1265
+ "sentence_breaker": nlp.sentence_breaker.name
1266
+ },
1267
+ "translations": LABEL_TRANSLATIONS,
1268
+ "input": {"text": text, "language": "he"},
1269
+ "sentences": [nlp._sentence_to_dict(r) for r in sentence_results]
1270
+ }
1271
+
1272
+ progress_bar.progress(100, text=t("done"))
1273
+ st.session_state['result'] = result
1274
+
1275
+ except Exception as e:
1276
+ st.error(f"Error: {e}")
1277
+ finally:
1278
+ st.session_state['processing'] = False
1279
+ st.rerun()
1280
+
1281
+ # ========== RESULTS SECTION ==========
1282
+
1283
+ if 'result' in st.session_state:
1284
+ result = st.session_state['result']
1285
+
1286
+ # Metrics
1287
+ col1, col2, col3 = st.columns(3)
1288
+ with col1:
1289
+ st.metric(t("sentences"), result['meta']['sentence_count'])
1290
+ with col2:
1291
+ st.metric(t("processing_time"), f"{result['meta']['processing_time_ms']:.0f}ms")
1292
+ with col3:
1293
+ st.metric(t("workers"), result['meta']['parallel_workers'])
1294
+
1295
+ # Text direction based on UI language
1296
+ text_dir = "rtl" if is_rtl() else "ltr"
1297
+ text_align = "right" if is_rtl() else "left"
1298
+
1299
+ # Tabs: Speech, All Trees, JSON, then sentence numbers
1300
+ if result['sentences']:
1301
+ tabs = st.tabs([t("speech"), t("all_trees"), t("json_output")] + [str(s['index']+1) for s in result['sentences']])
1302
+
1303
+ # Speech tab (first)
1304
+ with tabs[0]:
1305
+ # Get audio with sentence text to check punctuation
1306
+ # Extract base64 from data_uri (format: data:audio/wav;base64,{b64})
1307
+ audio_data_list = [
1308
+ (i, s['audio']['data_uri'].split(',', 1)[1], s['text'])
1309
+ for i, s in enumerate(result['sentences'])
1310
+ if s.get('audio') and s['audio'].get('data_uri')
1311
+ ]
1312
+
1313
+ if audio_data_list:
1314
+ st.subheader(f"🎵 {t('play_all')}")
1315
+
1316
+ # Combine all WAV files into one with silence between sentences
1317
+ try:
1318
+ import wave
1319
+ import io
1320
+ import struct
1321
+
1322
+ combined_audio = io.BytesIO()
1323
+ combined_writer = None
1324
+ sample_rate = None
1325
+ n_channels = None
1326
+ sample_width = None
1327
+
1328
+ for file_idx, (idx, b64_data, sent_text) in enumerate(audio_data_list):
1329
+ wav_bytes = base64.b64decode(b64_data)
1330
+ wav_io = io.BytesIO(wav_bytes)
1331
+
1332
+ with wave.open(wav_io, 'rb') as wav_reader:
1333
+ if combined_writer is None:
1334
+ combined_audio = io.BytesIO()
1335
+ combined_writer = wave.open(combined_audio, 'wb')
1336
+ n_channels = wav_reader.getnchannels()
1337
+ sample_width = wav_reader.getsampwidth()
1338
+ sample_rate = wav_reader.getframerate()
1339
+ combined_writer.setnchannels(n_channels)
1340
+ combined_writer.setsampwidth(sample_width)
1341
+ combined_writer.setframerate(sample_rate)
1342
+
1343
+ # Write audio frames
1344
+ combined_writer.writeframes(wav_reader.readframes(wav_reader.getnframes()))
1345
+
1346
+ # Add silence based on punctuation (not after last)
1347
+ if file_idx < len(audio_data_list) - 1:
1348
+ sent_text_stripped = sent_text.strip()
1349
+ pause_duration = 0
1350
+
1351
+ # Full pause for . ! ? or \n
1352
+ if sent_text_stripped.endswith(('.', '!', '?', '。')) or '\n' in sent_text:
1353
+ pause_duration = sentence_pause
1354
+ # Half pause for comma
1355
+ elif sent_text_stripped.endswith(','):
1356
+ pause_duration = sentence_pause * 0.5
1357
+
1358
+ if pause_duration > 0:
1359
+ silence_frames = int(sample_rate * pause_duration)
1360
+ silence_bytes = b'\x00' * (silence_frames * n_channels * sample_width)
1361
+ combined_writer.writeframes(silence_bytes)
1362
+
1363
+ if combined_writer:
1364
+ combined_writer.close()
1365
+ combined_audio.seek(0)
1366
+ combined_b64 = base64.b64encode(combined_audio.read()).decode()
1367
+ combined_uri = f"data:audio/wav;base64,{combined_b64}"
1368
+
1369
+ st.audio(combined_uri, format="audio/wav")
1370
+
1371
+ # Download combined WAV
1372
+ combined_audio.seek(0)
1373
+ st.download_button(
1374
+ f"⬇️ {t('download_all_audio')}",
1375
+ combined_audio.read(),
1376
+ file_name="all_sentences.wav",
1377
+ mime="audio/wav"
1378
+ )
1379
+ except Exception as e:
1380
+ st.warning(f"Could not combine audio: {e}")
1381
+
1382
+ st.divider()
1383
+
1384
+ # Individual WAV download links
1385
+ st.subheader(f"📥 {t('download_audio_files')}")
1386
+ cols = st.columns(min(len(audio_data_list), 4))
1387
+ for i, (idx, b64_data, _) in enumerate(audio_data_list):
1388
+ wav_bytes = base64.b64decode(b64_data)
1389
+ with cols[i % 4]:
1390
+ st.download_button(
1391
+ f"🔊 {idx + 1}",
1392
+ wav_bytes,
1393
+ file_name=f"sentence_{idx + 1}.wav",
1394
+ mime="audio/wav",
1395
+ key=f"wav_download_{idx}"
1396
+ )
1397
+ else:
1398
+ st.info(t("no_audio"))
1399
+
1400
+ # All Trees tab (second)
1401
+ with tabs[1]:
1402
+ st.subheader(f"🌳 {t('all_trees')}")
1403
+ for i, sentence_result in enumerate(result['sentences']):
1404
+ sent_lang = sentence_result.get('lang', 'he')
1405
+ st.markdown(f"**{i+1}:** {sentence_result['text']}")
1406
+ display_sentence_tree(sentence_result['tokens'], i, rtl=(sent_lang == "he"))
1407
+ if i < len(result['sentences']) - 1:
1408
+ st.divider()
1409
+
1410
+ # JSON tab (third)
1411
+ with tabs[2]:
1412
+ st.subheader(t("full_json"))
1413
+
1414
+ display_result = json.loads(json.dumps(result))
1415
+ for sent in display_result.get('sentences', []):
1416
+ if sent.get('audio'):
1417
+ sent['audio']['data_uri'] = '[truncated]'
1418
+
1419
+ st.download_button(
1420
+ t("download_json"),
1421
+ json.dumps(result, ensure_ascii=False, indent=2),
1422
+ file_name="hebrew_nlp_result.json",
1423
+ mime="application/json",
1424
+ width="stretch"
1425
+ )
1426
+
1427
+ st.json(display_result)
1428
+
1429
+ # Sentence tabs
1430
+ for i, sentence_result in enumerate(result['sentences']):
1431
+ with tabs[i + 3]: # +3 because Speech, All Trees, JSON are first
1432
+ # Per-sentence direction based on detected lang
1433
+ sent_lang = sentence_result.get('lang', 'he')
1434
+ sent_dir = "rtl" if sent_lang == "he" else "ltr"
1435
+ sent_align = "right" if sent_lang == "he" else "left"
1436
+
1437
+ st.markdown(f"### {i+1}")
1438
+
1439
+ col1, col2 = st.columns(2)
1440
+ with col1:
1441
+ st.markdown(f"**{t('original')}**")
1442
+ st.markdown(f"<div dir='{sent_dir}' style='font-size:1.2em; text-align:{sent_align};'>{sentence_result['text']}</div>",
1443
+ unsafe_allow_html=True)
1444
+ with col2:
1445
+ st.markdown(f"**{t('nikud')}**")
1446
+ st.markdown(f"<div dir='{sent_dir}' style='font-size:1.2em; text-align:{sent_align};'>{sentence_result['phonetics']['diacritized']}</div>",
1447
+ unsafe_allow_html=True)
1448
+
1449
+ # Audio above phonemes
1450
+ if sentence_result.get('audio'):
1451
+ with st.expander(t("audio"), expanded=True):
1452
+ st.audio(sentence_result['audio']['data_uri'])
1453
+ st.caption(f"{t('duration')}: {sentence_result['audio']['duration_ms']:.0f}ms")
1454
+
1455
+ st.markdown(f"**{t('phonemes')}**")
1456
+ st.code(sentence_result['phonetics']['phonemes'], language=None)
1457
+
1458
+ with st.expander(t("dep_tree"), expanded=True):
1459
+ display_sentence_tree(sentence_result['tokens'], i, rtl=(sent_lang == "he"))
1460
+
1461
+ with st.expander(t("morphology"), expanded=True):
1462
+ display_tokens_table(sentence_result['tokens'])
1463
+
1464
+ with st.expander(t("ner"), expanded=True):
1465
+ display_ner_entities(sentence_result['ner_entities'])
1466
+
1467
+ # Sentence JSON
1468
+ with st.expander(t("sentence_json"), expanded=True):
1469
+ # Create sentence JSON without audio data
1470
+ sentence_json = json.loads(json.dumps(sentence_result))
1471
+ if sentence_json.get('audio'):
1472
+ sentence_json['audio']['data_uri'] = '[truncated]'
1473
+
1474
+ st.download_button(
1475
+ t("download_json"),
1476
+ json.dumps(sentence_result, ensure_ascii=False, indent=2),
1477
+ file_name=f"sentence_{i+1}.json",
1478
+ mime="application/json",
1479
+ width="stretch",
1480
+ key=f"dl_json_{i}"
1481
+ )
1482
+
1483
+ st.json(sentence_json)
1484
+
1485
+ else:
1486
+ st.warning(t("models_not_loaded"))
1487
+ st.code("""
1488
+ # Install dependencies:
1489
+ pip install pysbd huggingface-hub
1490
+
1491
+ # All models are downloaded automatically on first use to ./onnx/:
1492
+ # - phonikud-1.0.int8.onnx (from thewh1teagle/phonikud-onnx)
1493
+ # - piper-voices/he_IL-phonikud.onnx (from thewh1teagle/phonikud-tts-checkpoints)
1494
+ # - piper-voices/en_US-ryan-high.onnx, etc. (from rhasspy/piper-voices)
1495
+ # - DictaBERT cached by HuggingFace transformers
1496
+ """, language="bash")
1497
+
1498
+ # Footer
1499
+ st.divider()
1500
+ st.markdown(f"""
1501
+ <div style='text-align:center; color:gray; font-size:0.9em;'>
1502
+ v{AsyncHebrewNLP.VERSION} (<a href="https://github.com/CLK-AL/HebrewNLP/blob/main/schema.json">schema</a>) |
1503
+ Async Parallel Pipeline | Cross-Platform |
1504
+ <a href="https://dicta.org.il/">Dicta</a> (<a href="https://huggingface.co/dicta-il/dictabert-joint">DictaBERT</a>) |
1505
+ <a href="https://github.com/thewh1teagle">thewh1teagle</a> (<a href="https://github.com/thewh1teagle/phonikud">Phonikud</a>) |
1506
+ <a href="https://github.com/rhasspy/piper">Piper</a> (<a href="https://huggingface.co/rhasspy/piper-voices">voices</a>) |
1507
+ <a href="https://github.com/nipunsadvilkar/pySBD">pysbd</a>
1508
+ </div>
1509
+ """, unsafe_allow_html=True)
download_models.py ADDED
@@ -0,0 +1,154 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Download all ONNX models for Hebrew Unified NLP
4
+
5
+ Usage:
6
+ python download_models.py # Download all 36 voices
7
+ python download_models.py --essential # Download essential only (phonikud + Hebrew + ryan-high)
8
+ """
9
+
10
+ import sys
11
+ import shutil
12
+ from pathlib import Path
13
+ from huggingface_hub import hf_hub_download
14
+
15
+ ONNX_DIR = Path("./onnx")
16
+ PIPER_DIR = ONNX_DIR / "piper-voices"
17
+
18
+ # All available English voices
19
+ EN_VOICES = {
20
+ # en_GB
21
+ "en_GB-alan-low": ("en_GB", "alan", "low"),
22
+ "en_GB-alan-medium": ("en_GB", "alan", "medium"),
23
+ "en_GB-alba-medium": ("en_GB", "alba", "medium"),
24
+ "en_GB-aru-medium": ("en_GB", "aru", "medium"),
25
+ "en_GB-cori-medium": ("en_GB", "cori", "medium"),
26
+ "en_GB-cori-high": ("en_GB", "cori", "high"),
27
+ "en_GB-jenny_dioco-medium": ("en_GB", "jenny_dioco", "medium"),
28
+ "en_GB-northern_english_male-medium": ("en_GB", "northern_english_male", "medium"),
29
+ "en_GB-semaine-medium": ("en_GB", "semaine", "medium"),
30
+ "en_GB-southern_english_female-low": ("en_GB", "southern_english_female", "low"),
31
+ "en_GB-vctk-medium": ("en_GB", "vctk", "medium"),
32
+ # en_US
33
+ "en_US-amy-low": ("en_US", "amy", "low"),
34
+ "en_US-amy-medium": ("en_US", "amy", "medium"),
35
+ "en_US-arctic-medium": ("en_US", "arctic", "medium"),
36
+ "en_US-bryce-medium": ("en_US", "bryce", "medium"),
37
+ "en_US-danny-low": ("en_US", "danny", "low"),
38
+ "en_US-hfc_female-medium": ("en_US", "hfc_female", "medium"),
39
+ "en_US-hfc_male-medium": ("en_US", "hfc_male", "medium"),
40
+ "en_US-joe-medium": ("en_US", "joe", "medium"),
41
+ "en_US-john-medium": ("en_US", "john", "medium"),
42
+ "en_US-kathleen-low": ("en_US", "kathleen", "low"),
43
+ "en_US-kristin-medium": ("en_US", "kristin", "medium"),
44
+ "en_US-kusal-medium": ("en_US", "kusal", "medium"),
45
+ "en_US-l2arctic-medium": ("en_US", "l2arctic", "medium"),
46
+ "en_US-lessac-high": ("en_US", "lessac", "high"),
47
+ "en_US-lessac-low": ("en_US", "lessac", "low"),
48
+ "en_US-lessac-medium": ("en_US", "lessac", "medium"),
49
+ "en_US-libritts-high": ("en_US", "libritts", "high"),
50
+ "en_US-libritts_r-medium": ("en_US", "libritts_r", "medium"),
51
+ "en_US-ljspeech-high": ("en_US", "ljspeech", "high"),
52
+ "en_US-ljspeech-medium": ("en_US", "ljspeech", "medium"),
53
+ "en_US-norman-medium": ("en_US", "norman", "medium"),
54
+ "en_US-ryan-high": ("en_US", "ryan", "high"),
55
+ "en_US-ryan-low": ("en_US", "ryan", "low"),
56
+ "en_US-ryan-medium": ("en_US", "ryan", "medium"),
57
+ }
58
+
59
+
60
+ def download_file(repo_id: str, filename: str, dest_path: Path):
61
+ """Download a file from HuggingFace Hub to destination."""
62
+ if dest_path.exists():
63
+ print(f" [skip] {dest_path.name} already exists")
64
+ return False
65
+
66
+ print(f" [download] {filename}...")
67
+ downloaded = hf_hub_download(repo_id=repo_id, filename=filename)
68
+ dest_path.parent.mkdir(parents=True, exist_ok=True)
69
+ shutil.copy(downloaded, dest_path)
70
+ print(f" [done] {dest_path.name}")
71
+ return True
72
+
73
+
74
+ def download_phonikud():
75
+ """Download Phonikud ONNX model."""
76
+ print("\n=== Phonikud Model ===")
77
+ download_file(
78
+ "thewh1teagle/phonikud-onnx",
79
+ "phonikud-1.0.int8.onnx",
80
+ ONNX_DIR / "phonikud-1.0.int8.onnx"
81
+ )
82
+
83
+
84
+ def download_hebrew_voice():
85
+ """Download Hebrew Piper voice."""
86
+ print("\n=== Hebrew Voice (he_IL-phonikud) ===")
87
+ download_file(
88
+ "thewh1teagle/phonikud-tts-checkpoints",
89
+ "model.onnx",
90
+ PIPER_DIR / "he_IL-phonikud.onnx"
91
+ )
92
+ download_file(
93
+ "thewh1teagle/phonikud-tts-checkpoints",
94
+ "model.config.json",
95
+ PIPER_DIR / "he_IL-phonikud.onnx.json"
96
+ )
97
+
98
+
99
+ def download_english_voice(voice_name: str):
100
+ """Download an English Piper voice."""
101
+ region, speaker, quality = EN_VOICES[voice_name]
102
+ subdir = f"en/{region}/{speaker}/{quality}"
103
+
104
+ download_file(
105
+ "rhasspy/piper-voices",
106
+ f"{subdir}/{voice_name}.onnx",
107
+ PIPER_DIR / f"{voice_name}.onnx"
108
+ )
109
+ download_file(
110
+ "rhasspy/piper-voices",
111
+ f"{subdir}/{voice_name}.onnx.json",
112
+ PIPER_DIR / f"{voice_name}.onnx.json"
113
+ )
114
+
115
+
116
+ def main():
117
+ essential_only = "--essential" in sys.argv
118
+
119
+ print("=" * 50)
120
+ print("Hebrew Unified NLP - Model Downloader")
121
+ print("=" * 50)
122
+
123
+ # Create directories
124
+ PIPER_DIR.mkdir(parents=True, exist_ok=True)
125
+
126
+ # Essential downloads
127
+ download_phonikud()
128
+ download_hebrew_voice()
129
+
130
+ print("\n=== English Voice (en_US-ryan-high) ===")
131
+ download_english_voice("en_US-ryan-high")
132
+
133
+ if not essential_only:
134
+ print("\n=== All English Voices ===")
135
+ for voice in EN_VOICES:
136
+ if voice != "en_US-ryan-high": # Already downloaded
137
+ print(f"\n [{voice}]")
138
+ download_english_voice(voice)
139
+
140
+ # Summary
141
+ print("\n" + "=" * 50)
142
+ print("Download Complete!")
143
+ print("=" * 50)
144
+
145
+ total_size = sum(f.stat().st_size for f in ONNX_DIR.rglob("*") if f.is_file())
146
+ print(f"\nLocation: {ONNX_DIR.absolute()}")
147
+ print(f"Total size: {total_size / 1024 / 1024:.1f} MB")
148
+
149
+ files = list(PIPER_DIR.glob("*.onnx"))
150
+ print(f"Voices: {len(files)} ({len(files) - 1} English + 1 Hebrew)")
151
+
152
+
153
+ if __name__ == "__main__":
154
+ main()
model.config.json DELETED
@@ -1,497 +0,0 @@
1
- {
2
- "dataset": "",
3
- "audio": {
4
- "sample_rate": 22050,
5
- "quality": "train"
6
- },
7
- "espeak": {
8
- "voice": "he"
9
- },
10
- "language": {
11
- "code": "he"
12
- },
13
- "inference": {
14
- "noise_scale": 0.667,
15
- "length_scale": 1,
16
- "noise_w": 0.8
17
- },
18
- "phoneme_type": "raw",
19
- "phoneme_map": {},
20
- "phoneme_id_map": {
21
- " ": [
22
- 3
23
- ],
24
- "!": [
25
- 4
26
- ],
27
- "\"": [
28
- 150
29
- ],
30
- "#": [
31
- 149
32
- ],
33
- "$": [
34
- 2
35
- ],
36
- "'": [
37
- 5
38
- ],
39
- "(": [
40
- 6
41
- ],
42
- ")": [
43
- 7
44
- ],
45
- ",": [
46
- 8
47
- ],
48
- "-": [
49
- 9
50
- ],
51
- ".": [
52
- 10
53
- ],
54
- "0": [
55
- 130
56
- ],
57
- "1": [
58
- 131
59
- ],
60
- "2": [
61
- 132
62
- ],
63
- "3": [
64
- 133
65
- ],
66
- "4": [
67
- 134
68
- ],
69
- "5": [
70
- 135
71
- ],
72
- "6": [
73
- 136
74
- ],
75
- "7": [
76
- 137
77
- ],
78
- "8": [
79
- 138
80
- ],
81
- "9": [
82
- 139
83
- ],
84
- ":": [
85
- 11
86
- ],
87
- ";": [
88
- 12
89
- ],
90
- "?": [
91
- 13
92
- ],
93
- "X": [
94
- 156
95
- ],
96
- "^": [
97
- 1
98
- ],
99
- "_": [
100
- 0
101
- ],
102
- "a": [
103
- 14
104
- ],
105
- "b": [
106
- 15
107
- ],
108
- "c": [
109
- 16
110
- ],
111
- "d": [
112
- 17
113
- ],
114
- "e": [
115
- 18
116
- ],
117
- "f": [
118
- 19
119
- ],
120
- "g": [
121
- 154
122
- ],
123
- "h": [
124
- 20
125
- ],
126
- "i": [
127
- 21
128
- ],
129
- "j": [
130
- 22
131
- ],
132
- "k": [
133
- 23
134
- ],
135
- "l": [
136
- 24
137
- ],
138
- "m": [
139
- 25
140
- ],
141
- "n": [
142
- 26
143
- ],
144
- "o": [
145
- 27
146
- ],
147
- "p": [
148
- 28
149
- ],
150
- "q": [
151
- 29
152
- ],
153
- "r": [
154
- 30
155
- ],
156
- "s": [
157
- 31
158
- ],
159
- "t": [
160
- 32
161
- ],
162
- "u": [
163
- 33
164
- ],
165
- "v": [
166
- 34
167
- ],
168
- "w": [
169
- 35
170
- ],
171
- "x": [
172
- 36
173
- ],
174
- "y": [
175
- 37
176
- ],
177
- "z": [
178
- 38
179
- ],
180
- "æ": [
181
- 39
182
- ],
183
- "ç": [
184
- 40
185
- ],
186
- "ð": [
187
- 41
188
- ],
189
- "ø": [
190
- 42
191
- ],
192
- "ħ": [
193
- 43
194
- ],
195
- "ŋ": [
196
- 44
197
- ],
198
- "œ": [
199
- 45
200
- ],
201
- "ǀ": [
202
- 46
203
- ],
204
- "ǁ": [
205
- 47
206
- ],
207
- "ǂ": [
208
- 48
209
- ],
210
- "ǃ": [
211
- 49
212
- ],
213
- "ɐ": [
214
- 50
215
- ],
216
- "ɑ": [
217
- 51
218
- ],
219
- "ɒ": [
220
- 52
221
- ],
222
- "ɓ": [
223
- 53
224
- ],
225
- "ɔ": [
226
- 54
227
- ],
228
- "ɕ": [
229
- 55
230
- ],
231
- "ɖ": [
232
- 56
233
- ],
234
- "ɗ": [
235
- 57
236
- ],
237
- "ɘ": [
238
- 58
239
- ],
240
- "ə": [
241
- 59
242
- ],
243
- "ɚ": [
244
- 60
245
- ],
246
- "ɛ": [
247
- 61
248
- ],
249
- "ɜ": [
250
- 62
251
- ],
252
- "ɞ": [
253
- 63
254
- ],
255
- "ɟ": [
256
- 64
257
- ],
258
- "ɠ": [
259
- 65
260
- ],
261
- "ɡ": [
262
- 66
263
- ],
264
- "ɢ": [
265
- 67
266
- ],
267
- "ɣ": [
268
- 68
269
- ],
270
- "ɤ": [
271
- 69
272
- ],
273
- "ɥ": [
274
- 70
275
- ],
276
- "ɦ": [
277
- 71
278
- ],
279
- "ɧ": [
280
- 72
281
- ],
282
- "ɨ": [
283
- 73
284
- ],
285
- "ɪ": [
286
- 74
287
- ],
288
- "ɫ": [
289
- 75
290
- ],
291
- "ɬ": [
292
- 76
293
- ],
294
- "ɭ": [
295
- 77
296
- ],
297
- "ɮ": [
298
- 78
299
- ],
300
- "ɯ": [
301
- 79
302
- ],
303
- "ɰ": [
304
- 80
305
- ],
306
- "ɱ": [
307
- 81
308
- ],
309
- "ɲ": [
310
- 82
311
- ],
312
- "ɳ": [
313
- 83
314
- ],
315
- "ɴ": [
316
- 84
317
- ],
318
- "ɵ": [
319
- 85
320
- ],
321
- "ɶ": [
322
- 86
323
- ],
324
- "ɸ": [
325
- 87
326
- ],
327
- "ɹ": [
328
- 88
329
- ],
330
- "ɺ": [
331
- 89
332
- ],
333
- "ɻ": [
334
- 90
335
- ],
336
- "ɽ": [
337
- 91
338
- ],
339
- "ɾ": [
340
- 92
341
- ],
342
- "ʀ": [
343
- 93
344
- ],
345
- "ʁ": [
346
- 94
347
- ],
348
- "ʂ": [
349
- 95
350
- ],
351
- "ʃ": [
352
- 96
353
- ],
354
- "ʄ": [
355
- 97
356
- ],
357
- "ʈ": [
358
- 98
359
- ],
360
- "ʉ": [
361
- 99
362
- ],
363
- "ʊ": [
364
- 100
365
- ],
366
- "ʋ": [
367
- 101
368
- ],
369
- "ʌ": [
370
- 102
371
- ],
372
- "ʍ": [
373
- 103
374
- ],
375
- "ʎ": [
376
- 104
377
- ],
378
- "ʏ": [
379
- 105
380
- ],
381
- "ʐ": [
382
- 106
383
- ],
384
- "ʑ": [
385
- 107
386
- ],
387
- "ʒ": [
388
- 108
389
- ],
390
- "ʔ": [
391
- 109
392
- ],
393
- "ʕ": [
394
- 110
395
- ],
396
- "ʘ": [
397
- 111
398
- ],
399
- "ʙ": [
400
- 112
401
- ],
402
- "ʛ": [
403
- 113
404
- ],
405
- "ʜ": [
406
- 114
407
- ],
408
- "ʝ": [
409
- 115
410
- ],
411
- "ʟ": [
412
- 116
413
- ],
414
- "ʡ": [
415
- 117
416
- ],
417
- "ʢ": [
418
- 118
419
- ],
420
- "ʦ": [
421
- 155
422
- ],
423
- "ʰ": [
424
- 145
425
- ],
426
- "ʲ": [
427
- 119
428
- ],
429
- "ˈ": [
430
- 120
431
- ],
432
- "ˌ": [
433
- 121
434
- ],
435
- "ː": [
436
- 122
437
- ],
438
- "ˑ": [
439
- 123
440
- ],
441
- "˞": [
442
- 124
443
- ],
444
- "ˤ": [
445
- 146
446
- ],
447
- "̃": [
448
- 141
449
- ],
450
- "̧": [
451
- 140
452
- ],
453
- "̩": [
454
- 144
455
- ],
456
- "̪": [
457
- 142
458
- ],
459
- "̯": [
460
- 143
461
- ],
462
- "̺": [
463
- 152
464
- ],
465
- "̻": [
466
- 153
467
- ],
468
- "β": [
469
- 125
470
- ],
471
- "ε": [
472
- 147
473
- ],
474
- "θ": [
475
- 126
476
- ],
477
- "χ": [
478
- 127
479
- ],
480
- "ᵻ": [
481
- 128
482
- ],
483
- "↑": [
484
- 151
485
- ],
486
- "↓": [
487
- 148
488
- ],
489
- "ⱱ": [
490
- 129
491
- ]
492
- },
493
- "num_symbols": 256,
494
- "num_speakers": 1,
495
- "speaker_id_map": {},
496
- "piper_version": "1.0.0"
497
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
model.onnx DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:592363804a09e3d70b05bd86b366e12e0a12c4a4d0100997cf8f0832132c55e7
3
- size 63511038
 
 
 
 
packages.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ graphviz
2
+ libsndfile1
3
+ espeak-ng
phonikud-1.0.int8.onnx DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:113afb58d3140502aa1e7691cdc6b240b56cf97e5852fc870e1a7fb5a400dd62
3
- size 307844158
 
 
 
 
pyproject.toml ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [project]
2
+ name = "hebrew-unified-nlp"
3
+ version = "2.0.0"
4
+ description = "Async parallel Hebrew NLP combining Phonikud TTS + DictaBERT-Joint"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ requires-python = ">=3.10,<3.13"
8
+ authors = [
9
+ { name = "TomAR" }
10
+ ]
11
+ keywords = ["hebrew", "nlp", "tts", "phonikud", "dictabert", "morphology", "syntax"]
12
+
13
+ dependencies = [
14
+ "phonikud-tts>=0.1.0",
15
+ "transformers>=4.40.0",
16
+ "torch>=2.0.0",
17
+ "streamlit>=1.40.0",
18
+ "graphviz>=0.20.0",
19
+ "arabic-reshaper>=3.0.0",
20
+ "python-bidi>=0.6.0",
21
+ "pysbd>=0.3.4",
22
+ "soundfile>=0.13.0",
23
+ ]
24
+
25
+ [project.optional-dependencies]
26
+ icu = ["PyICU>=2.12"]
27
+ neural = ["wtpsplit>=2.0.0"]
28
+
29
+ [project.urls]
30
+ Homepage = "https://huggingface.co/spaces/YOUR_USERNAME/hebrew-unified-nlp"
31
+ Repository = "https://github.com/YOUR_USERNAME/hebrew-unified-nlp"
32
+
33
+ [build-system]
34
+ requires = ["hatchling"]
35
+ build-backend = "hatchling.build"
requirements.txt CHANGED
@@ -1,5 +1,8 @@
1
  # This file was autogenerated by uv via the following command:
2
  # uv export --no-hashes --no-emit-project
 
 
 
3
  attrs==25.3.0
4
  # via
5
  # csvw
@@ -74,10 +77,10 @@ packaging==25.0
74
  # onnxruntime
75
  phonemizer-fork==3.3.2
76
  # via piper-onnx
77
- phonikud @ git+https://github.com/thewh1teagle/phonikud@23d3b887f1665ae831ec476be71573a9ca4a61c6
78
- # via phonikud-tts
79
  phonikud-onnx==1.0.6
80
  # via phonikud-tts
 
81
  piper-onnx==1.0.6
82
  # via phonikud-tts
83
  protobuf==6.32.1
@@ -137,4 +140,27 @@ uritemplate==4.2.0
137
  # via csvw
138
  urllib3==2.5.0
139
  # via requests
140
- flask>=3.1.1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # This file was autogenerated by uv via the following command:
2
  # uv export --no-hashes --no-emit-project
3
+ #
4
+ # IMPORTANT: Requires Python 3.10-3.12 (phonikud doesn't support 3.13 yet)
5
+ #
6
  attrs==25.3.0
7
  # via
8
  # csvw
 
77
  # onnxruntime
78
  phonemizer-fork==3.3.2
79
  # via piper-onnx
80
+ # phonikud is installed via phonikud-tts
 
81
  phonikud-onnx==1.0.6
82
  # via phonikud-tts
83
+ phonikud-tts>=0.1.0
84
  piper-onnx==1.0.6
85
  # via phonikud-tts
86
  protobuf==6.32.1
 
140
  # via csvw
141
  urllib3==2.5.0
142
  # via requests
143
+
144
+ # Web framework
145
+ streamlit>=1.40.0
146
+
147
+ # DictaBERT dependencies
148
+ transformers>=4.40.0
149
+ --extra-index-url https://download.pytorch.org/whl/cpu
150
+ torch>=2.0.0
151
+
152
+ # Visualization
153
+ graphviz>=0.20.0
154
+
155
+ # RTL text handling
156
+ arabic_reshaper>=3.0.0
157
+ python-bidi>=0.6.0
158
+
159
+ # Sentence breaking (choose one - pysbd recommended for cross-platform)
160
+ pysbd>=0.3.4
161
+
162
+ # Optional: ICU CLDR sentence breaking (Linux/Mac only, requires libicu-dev)
163
+ # PyICU>=2.12
164
+
165
+ # Optional: Neural sentence splitter
166
+ # wtpsplit>=2.0.0
schema.json ADDED
@@ -0,0 +1,483 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "$id": "https://github.com/hebrew-unified-nlp/schema/v2.1.0",
4
+ "title": "HebrewUnifiedNLPResult",
5
+ "description": "Async parallel Hebrew/English NLP with cross-platform sentence breaking - outputs sentence array with per-sentence language detection",
6
+ "type": "object",
7
+ "required": [
8
+ "meta",
9
+ "input",
10
+ "sentences"
11
+ ],
12
+ "properties": {
13
+ "meta": {
14
+ "type": "object",
15
+ "description": "Processing metadata",
16
+ "required": [
17
+ "version",
18
+ "timestamp",
19
+ "sentence_count"
20
+ ],
21
+ "properties": {
22
+ "version": {
23
+ "type": "string",
24
+ "pattern": "^\\d+\\.\\d+\\.\\d+$",
25
+ "example": "2025.12.5"
26
+ },
27
+ "timestamp": {
28
+ "type": "string",
29
+ "format": "date-time"
30
+ },
31
+ "models": {
32
+ "type": "object",
33
+ "properties": {
34
+ "phonikud": {
35
+ "type": "string",
36
+ "description": "Phonikud model name"
37
+ },
38
+ "piper": {
39
+ "type": "string",
40
+ "description": "Piper voice manager info"
41
+ },
42
+ "dictabert": {
43
+ "type": "string",
44
+ "description": "DictaBERT model path"
45
+ }
46
+ }
47
+ },
48
+ "processing_time_ms": {
49
+ "type": "number",
50
+ "minimum": 0,
51
+ "description": "Total processing time for all sentences"
52
+ },
53
+ "sentence_count": {
54
+ "type": "integer",
55
+ "minimum": 0,
56
+ "description": "Number of sentences detected"
57
+ },
58
+ "parallel_workers": {
59
+ "type": "integer",
60
+ "minimum": 1,
61
+ "description": "Number of parallel workers used"
62
+ },
63
+ "sentence_breaker": {
64
+ "type": "string",
65
+ "description": "Sentence breaking backend used",
66
+ "enum": [
67
+ "pysbd",
68
+ "pysbd-en",
69
+ "icu",
70
+ "wtpsplit",
71
+ "regex"
72
+ ]
73
+ }
74
+ }
75
+ },
76
+ "translations": {
77
+ "type": "object",
78
+ "description": "Bilingual label translations (Hebrew/English)",
79
+ "properties": {
80
+ "pos": {
81
+ "type": "object",
82
+ "description": "Part-of-speech translations",
83
+ "additionalProperties": {
84
+ "$ref": "#/definitions/BilingualLabel"
85
+ }
86
+ },
87
+ "dep": {
88
+ "type": "object",
89
+ "description": "Dependency relation translations",
90
+ "additionalProperties": {
91
+ "$ref": "#/definitions/BilingualLabel"
92
+ }
93
+ },
94
+ "ner": {
95
+ "type": "object",
96
+ "description": "Named entity type translations",
97
+ "additionalProperties": {
98
+ "$ref": "#/definitions/BilingualLabel"
99
+ }
100
+ },
101
+ "prefix": {
102
+ "type": "object",
103
+ "description": "Prefix type translations",
104
+ "additionalProperties": {
105
+ "$ref": "#/definitions/BilingualLabel"
106
+ }
107
+ },
108
+ "special": {
109
+ "type": "object",
110
+ "description": "Special token translations",
111
+ "additionalProperties": {
112
+ "$ref": "#/definitions/BilingualLabel"
113
+ }
114
+ },
115
+ "morph": {
116
+ "type": "object",
117
+ "description": "Morphological feature value translations",
118
+ "additionalProperties": {
119
+ "$ref": "#/definitions/BilingualLabel"
120
+ }
121
+ }
122
+ }
123
+ },
124
+ "input": {
125
+ "type": "object",
126
+ "required": [
127
+ "text"
128
+ ],
129
+ "properties": {
130
+ "text": {
131
+ "type": "string",
132
+ "description": "Original input text (may contain multiple sentences)"
133
+ },
134
+ "language": {
135
+ "type": "string",
136
+ "description": "Primary input language",
137
+ "enum": [
138
+ "he",
139
+ "en"
140
+ ]
141
+ }
142
+ }
143
+ },
144
+ "sentences": {
145
+ "type": "array",
146
+ "description": "Array of sentence analysis results (parallel processed)",
147
+ "items": {
148
+ "$ref": "#/definitions/SentenceResult"
149
+ }
150
+ }
151
+ },
152
+ "definitions": {
153
+ "BilingualLabel": {
154
+ "type": "object",
155
+ "description": "Label with Hebrew and English translations",
156
+ "properties": {
157
+ "en": {
158
+ "type": "string"
159
+ },
160
+ "he": {
161
+ "type": "string"
162
+ }
163
+ },
164
+ "required": [
165
+ "en",
166
+ "he"
167
+ ]
168
+ },
169
+ "SentenceResult": {
170
+ "type": "object",
171
+ "description": "Analysis result for a single sentence",
172
+ "required": [
173
+ "index",
174
+ "lang",
175
+ "text",
176
+ "phonetics",
177
+ "tokens"
178
+ ],
179
+ "properties": {
180
+ "index": {
181
+ "type": "integer",
182
+ "minimum": 0,
183
+ "description": "Sentence index in original text"
184
+ },
185
+ "lang": {
186
+ "type": "string",
187
+ "enum": [
188
+ "he",
189
+ "en"
190
+ ],
191
+ "description": "Detected language for this sentence"
192
+ },
193
+ "text": {
194
+ "type": "string",
195
+ "description": "Original sentence text"
196
+ },
197
+ "phonetics": {
198
+ "$ref": "#/definitions/Phonetics"
199
+ },
200
+ "tokens": {
201
+ "type": "array",
202
+ "items": {
203
+ "$ref": "#/definitions/Token"
204
+ }
205
+ },
206
+ "ner_entities": {
207
+ "type": "array",
208
+ "items": {
209
+ "$ref": "#/definitions/NEREntity"
210
+ }
211
+ },
212
+ "speech": {
213
+ "oneOf": [
214
+ {
215
+ "$ref": "#/definitions/Speech"
216
+ },
217
+ {
218
+ "type": "null"
219
+ }
220
+ ]
221
+ },
222
+ "tree_svg": {
223
+ "type": "string",
224
+ "description": "SVG string of dependency tree visualization"
225
+ }
226
+ }
227
+ },
228
+ "Phonetics": {
229
+ "type": "object",
230
+ "description": "Phonikud output for sentence (Hebrew) or passthrough (English)",
231
+ "required": [
232
+ "diacritized",
233
+ "phonemes"
234
+ ],
235
+ "properties": {
236
+ "diacritized": {
237
+ "type": "string",
238
+ "description": "Sentence with nikud (Hebrew) or original text (English)"
239
+ },
240
+ "phonemes": {
241
+ "type": "string",
242
+ "description": "Phoneme sequence (Hebrew) or empty string (English)"
243
+ }
244
+ }
245
+ },
246
+ "Token": {
247
+ "type": "object",
248
+ "description": "Unified token: DictaBERT + Phonikud enrichment",
249
+ "required": [
250
+ "id",
251
+ "token"
252
+ ],
253
+ "properties": {
254
+ "id": {
255
+ "type": "integer",
256
+ "minimum": 1,
257
+ "description": "1-indexed token position within sentence"
258
+ },
259
+ "token": {
260
+ "type": "string",
261
+ "description": "Surface form (DictaBERT)"
262
+ },
263
+ "nikud": {
264
+ "type": "string",
265
+ "description": "Diacritized form (Phonikud for Hebrew, original for English)"
266
+ },
267
+ "phonemes": {
268
+ "type": "string",
269
+ "description": "Token phonemes (Hebrew) or empty (English)"
270
+ },
271
+ "seg": {
272
+ "type": "array",
273
+ "description": "Morphological segmentation (DictaBERT)",
274
+ "items": {
275
+ "type": "string"
276
+ }
277
+ },
278
+ "lex": {
279
+ "type": "string",
280
+ "description": "Lemma (DictaBERT)"
281
+ },
282
+ "offsets": {
283
+ "type": "object",
284
+ "properties": {
285
+ "start": {
286
+ "type": "integer"
287
+ },
288
+ "end": {
289
+ "type": "integer"
290
+ }
291
+ }
292
+ },
293
+ "morph": {
294
+ "$ref": "#/definitions/Morphology"
295
+ },
296
+ "syntax": {
297
+ "$ref": "#/definitions/Syntax"
298
+ }
299
+ }
300
+ },
301
+ "Morphology": {
302
+ "type": "object",
303
+ "description": "DictaBERT morphological analysis",
304
+ "properties": {
305
+ "token": {
306
+ "type": "string"
307
+ },
308
+ "pos": {
309
+ "type": "string",
310
+ "enum": [
311
+ "ADJ",
312
+ "ADP",
313
+ "ADV",
314
+ "AUX",
315
+ "CCONJ",
316
+ "DET",
317
+ "INTJ",
318
+ "NOUN",
319
+ "NUM",
320
+ "PART",
321
+ "PRON",
322
+ "PROPN",
323
+ "PUNCT",
324
+ "SCONJ",
325
+ "SYM",
326
+ "VERB",
327
+ "X"
328
+ ]
329
+ },
330
+ "feats": {
331
+ "type": "object",
332
+ "properties": {
333
+ "Gender": {
334
+ "type": "string",
335
+ "enum": [
336
+ "Masc",
337
+ "Fem"
338
+ ]
339
+ },
340
+ "Number": {
341
+ "type": "string",
342
+ "enum": [
343
+ "Sing",
344
+ "Plur",
345
+ "Dual"
346
+ ]
347
+ },
348
+ "Person": {
349
+ "type": "string",
350
+ "enum": [
351
+ "1",
352
+ "2",
353
+ "3"
354
+ ]
355
+ },
356
+ "Tense": {
357
+ "type": "string",
358
+ "enum": [
359
+ "Past",
360
+ "Present",
361
+ "Future",
362
+ "Imp",
363
+ "Inf"
364
+ ]
365
+ },
366
+ "Voice": {
367
+ "type": "string",
368
+ "enum": [
369
+ "Act",
370
+ "Pass"
371
+ ]
372
+ },
373
+ "Definite": {
374
+ "type": "string",
375
+ "enum": [
376
+ "Def",
377
+ "Ind"
378
+ ]
379
+ },
380
+ "Case": {
381
+ "type": "string",
382
+ "enum": [
383
+ "Nom",
384
+ "Acc",
385
+ "Gen"
386
+ ]
387
+ },
388
+ "Construct": {
389
+ "type": "string",
390
+ "enum": [
391
+ "Construct",
392
+ "Free"
393
+ ]
394
+ }
395
+ },
396
+ "additionalProperties": true
397
+ },
398
+ "prefixes": {
399
+ "type": "array",
400
+ "items": {
401
+ "type": "string"
402
+ }
403
+ },
404
+ "suffix": {
405
+ "type": "boolean"
406
+ }
407
+ }
408
+ },
409
+ "Syntax": {
410
+ "type": "object",
411
+ "description": "DictaBERT dependency syntax",
412
+ "properties": {
413
+ "word": {
414
+ "type": "string"
415
+ },
416
+ "dep_head_idx": {
417
+ "type": "integer",
418
+ "minimum": -1,
419
+ "description": "-1 for root"
420
+ },
421
+ "dep_func": {
422
+ "type": "string"
423
+ },
424
+ "dep_head": {
425
+ "type": "string"
426
+ }
427
+ }
428
+ },
429
+ "NEREntity": {
430
+ "type": "object",
431
+ "properties": {
432
+ "phrase": {
433
+ "type": "string",
434
+ "description": "Entity text"
435
+ },
436
+ "label": {
437
+ "type": "string",
438
+ "enum": [
439
+ "PER",
440
+ "LOC",
441
+ "GPE",
442
+ "ORG",
443
+ "TIME",
444
+ "DATE",
445
+ "MISC",
446
+ "MONEY",
447
+ "PERCENT"
448
+ ]
449
+ },
450
+ "start": {
451
+ "type": "integer"
452
+ },
453
+ "end": {
454
+ "type": "integer"
455
+ }
456
+ }
457
+ },
458
+ "Speech": {
459
+ "type": "object",
460
+ "description": "TTS audio for sentence",
461
+ "properties": {
462
+ "format": {
463
+ "type": "string",
464
+ "enum": [
465
+ "wav",
466
+ "mp3",
467
+ "ogg"
468
+ ]
469
+ },
470
+ "sample_rate": {
471
+ "type": "integer"
472
+ },
473
+ "duration_ms": {
474
+ "type": "number"
475
+ },
476
+ "data_uri": {
477
+ "type": "string",
478
+ "description": "Base64 data URI (data:audio/wav;base64,...)"
479
+ }
480
+ }
481
+ }
482
+ }
483
+ }