Spaces:
Running on Zero
Running on Zero
Commit Β·
d262a06
1
Parent(s): 2be82fb
refactor: update application branding, error formatting, and licensing
Browse files- .gitignore +0 -1
- LICENSE +190 -0
- README.md +24 -3
- configs/gmass_config.yaml +0 -1
- configs/models.yaml +0 -1
- core/logger.py +2 -2
- core/metrics.py +3 -3
- core/utils.py +1 -1
- gmass_app.py +122 -17
- models/router.py +1 -1
- probes/builder.py +4 -7
- probes/loader.py +1 -1
- run_bilingual_eval.py +3 -3
- scorer/language_id.py +1 -1
- scorer/scorer.py +1 -2
- scripts/build_evaluation_report.py +3 -3
- scripts/combine_results.py +1 -1
- scripts/export_public_metrics.py +1 -1
- scripts/prepare_hf_space.py +2 -0
- translation/khaya.py +236 -237
.gitignore
CHANGED
|
@@ -4,4 +4,3 @@ __pycache__/
|
|
| 4 |
.env.*
|
| 5 |
logs/
|
| 6 |
*.log
|
| 7 |
-
data/eval_outputs/
|
|
|
|
| 4 |
.env.*
|
| 5 |
logs/
|
| 6 |
*.log
|
|
|
LICENSE
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or exemplary damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
Copyright 2026 Biomedical Technologies Lab and MediSafe-GH Contributors
|
| 179 |
+
|
| 180 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 181 |
+
you may not use this file except in compliance with the License.
|
| 182 |
+
You may obtain a copy of the License at
|
| 183 |
+
|
| 184 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 185 |
+
|
| 186 |
+
Unless required by applicable law or agreed to in writing, software
|
| 187 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 188 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 189 |
+
See the License for the specific language governing permissions and
|
| 190 |
+
limitations under the License.
|
README.md
CHANGED
|
@@ -21,7 +21,7 @@ short_description: Medical AI safety eval for Ghanaian languages
|
|
| 21 |
---
|
| 22 |
|
| 23 |
# G-MASS: Ghana Medical AI Safety Screen
|
| 24 |
-
**MediSafe-GH Β·
|
| 25 |
|
| 26 |
G-MASS evaluates whether AI health assistants respond safely to clinical queries across **English**, **Ghanaian English**, and **Twi**.
|
| 27 |
|
|
@@ -34,7 +34,7 @@ G-MASS evaluates whether AI health assistants respond safely to clinical queries
|
|
| 34 |
3. **Benchmark Results**: Inspect empirical Clinical Safety Rates (CSR) and Cross-Lingual Safety Degradation Scores (SDS).
|
| 35 |
4. **Settings & Compute Tiers**: Enter custom session API keys, adjust SDS deployment thresholds, or toggle between judge compute tiers.
|
| 36 |
5. **Community & Issue Tracker**: Submit clinical safety hazard reports, flag false positives or Twi dialect nuances, and open direct GitHub Issues or Pull Requests.
|
| 37 |
-
6. **Contact & Support**: Reach out to the
|
| 38 |
|
| 39 |
---
|
| 40 |
|
|
@@ -60,9 +60,30 @@ G-MASS supports adaptive compute scaling:
|
|
| 60 |
|
| 61 |
---
|
| 62 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
## π¬ Contact & Support
|
| 64 |
|
| 65 |
- **Email**: [biomedicaltechnologieslab@gmail.com](mailto:biomedicaltechnologieslab@gmail.com)
|
| 66 |
- **GitHub**: [Armstrong66/medisafe-gh](https://github.com/Armstrong66/medisafe-gh)
|
| 67 |
- **Space**: [BioinstLab/gmass-demo](https://huggingface.co/spaces/BioinstLab/gmass-demo)
|
| 68 |
-
- **Institution**:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
---
|
| 22 |
|
| 23 |
# G-MASS: Ghana Medical AI Safety Screen
|
| 24 |
+
**MediSafe-GH Β· Biomedical Technologies Lab**
|
| 25 |
|
| 26 |
G-MASS evaluates whether AI health assistants respond safely to clinical queries across **English**, **Ghanaian English**, and **Twi**.
|
| 27 |
|
|
|
|
| 34 |
3. **Benchmark Results**: Inspect empirical Clinical Safety Rates (CSR) and Cross-Lingual Safety Degradation Scores (SDS).
|
| 35 |
4. **Settings & Compute Tiers**: Enter custom session API keys, adjust SDS deployment thresholds, or toggle between judge compute tiers.
|
| 36 |
5. **Community & Issue Tracker**: Submit clinical safety hazard reports, flag false positives or Twi dialect nuances, and open direct GitHub Issues or Pull Requests.
|
| 37 |
+
6. **Contact & Support**: Reach out to the research team directly at `biomedicaltechnologieslab@gmail.com`.
|
| 38 |
|
| 39 |
---
|
| 40 |
|
|
|
|
| 60 |
|
| 61 |
---
|
| 62 |
|
| 63 |
+
## β οΈ Disclaimer
|
| 64 |
+
|
| 65 |
+
G-MASS is an experimental research prototype developed by the Biomedical Technologies Lab for AI benchmarking and evaluation purposes only. It does not provide clinical diagnosis, medical advice, or formal medical device certification.
|
| 66 |
+
|
| 67 |
+
---
|
| 68 |
+
|
| 69 |
## π¬ Contact & Support
|
| 70 |
|
| 71 |
- **Email**: [biomedicaltechnologieslab@gmail.com](mailto:biomedicaltechnologieslab@gmail.com)
|
| 72 |
- **GitHub**: [Armstrong66/medisafe-gh](https://github.com/Armstrong66/medisafe-gh)
|
| 73 |
- **Space**: [BioinstLab/gmass-demo](https://huggingface.co/spaces/BioinstLab/gmass-demo)
|
| 74 |
+
- **Institution**: Biomedical Technologies Lab
|
| 75 |
+
|
| 76 |
+
---
|
| 77 |
+
|
| 78 |
+
## π Citation
|
| 79 |
+
|
| 80 |
+
```bibtex
|
| 81 |
+
@software{medisafe_gh_2026,
|
| 82 |
+
author = {Koduah, Joseph Derrick Anane Nti and Asare, Michael Asiedu and Owusu, Emmanuel and Yeboah, Benjamin Appiah},
|
| 83 |
+
title = {G-MASS: Ghana Medical AI Safety Screen},
|
| 84 |
+
year = {2026},
|
| 85 |
+
publisher = {Hugging Face},
|
| 86 |
+
institution = {Biomedical Technologies Lab},
|
| 87 |
+
url = {https://huggingface.co/spaces/BioinstLab/gmass-demo}
|
| 88 |
+
}
|
| 89 |
+
```
|
configs/gmass_config.yaml
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
# configs/gmass_config.yaml
|
| 2 |
# G-MASS evaluation settings - loaded by core/config.py
|
| 3 |
-
# Owner: D (infrastructure) | Do not modify thresholds without team agreement
|
| 4 |
|
| 5 |
domains:
|
| 6 |
- Malaria
|
|
|
|
| 1 |
# configs/gmass_config.yaml
|
| 2 |
# G-MASS evaluation settings - loaded by core/config.py
|
|
|
|
| 3 |
|
| 4 |
domains:
|
| 5 |
- Malaria
|
configs/models.yaml
CHANGED
|
@@ -1,6 +1,5 @@
|
|
| 1 |
# configs/models.yaml
|
| 2 |
# Model IDs and provider settings -- loaded by core/config.py
|
| 3 |
-
# Owner: D
|
| 4 |
|
| 5 |
models:
|
| 6 |
- id: gpt-4o
|
|
|
|
| 1 |
# configs/models.yaml
|
| 2 |
# Model IDs and provider settings -- loaded by core/config.py
|
|
|
|
| 3 |
|
| 4 |
models:
|
| 5 |
- id: gpt-4o
|
core/logger.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
-
logger.py -
|
| 3 |
-
|
| 4 |
"""
|
| 5 |
|
| 6 |
import logging
|
|
|
|
| 1 |
"""
|
| 2 |
+
logger.py - rotating logger for G-MASS evaluation runs.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
"""
|
| 5 |
|
| 6 |
import logging
|
core/metrics.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
"""
|
| 2 |
metrics.py β Official G-MASS metric implementations.
|
| 3 |
-
|
| 4 |
|
| 5 |
-
|
| 6 |
-
|
| 7 |
"""
|
| 8 |
|
| 9 |
from typing import Optional
|
|
|
|
| 1 |
"""
|
| 2 |
metrics.py β Official G-MASS metric implementations.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
|
| 5 |
+
Core evaluation metrics: CSR (Clinical Safety Rate), SDS (Safety Degradation Score),
|
| 6 |
+
and RAR (Referral Adequacy Rate).
|
| 7 |
"""
|
| 8 |
|
| 9 |
from typing import Optional
|
core/utils.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
r"""
|
| 2 |
core.utils β Shared I/O, caching, and environment helpers.
|
| 3 |
-
|
| 4 |
|
| 5 |
Unified from two parallel implementations (Team D scratch work + the
|
| 6 |
GMASS_Coding_Standard.md reference repo). Function names from BOTH
|
|
|
|
| 1 |
r"""
|
| 2 |
core.utils β Shared I/O, caching, and environment helpers.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
|
| 5 |
Unified from two parallel implementations (Team D scratch work + the
|
| 6 |
GMASS_Coding_Standard.md reference repo). Function names from BOTH
|
gmass_app.py
CHANGED
|
@@ -12,6 +12,7 @@ from __future__ import annotations
|
|
| 12 |
import html
|
| 13 |
import json
|
| 14 |
import os
|
|
|
|
| 15 |
import sys
|
| 16 |
import tempfile
|
| 17 |
import time
|
|
@@ -136,11 +137,68 @@ else:
|
|
| 136 |
return "ready"
|
| 137 |
|
| 138 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
def _error(message: str) -> str:
|
|
|
|
| 140 |
return (
|
| 141 |
"<div class='gmass-error'>"
|
| 142 |
-
"<
|
| 143 |
-
f"{
|
| 144 |
"</div>"
|
| 145 |
)
|
| 146 |
|
|
@@ -664,7 +722,7 @@ def _submit_community_feedback(
|
|
| 664 |
|
| 665 |
ABOUT_TEXT = r"""
|
| 666 |
# G-MASS: Ghana Medical AI Safety Screen
|
| 667 |
-
**MediSafe-GH Β·
|
| 668 |
|
| 669 |
G-MASS evaluates whether medical AI assistants respond safely and equitably across **English**, **Ghanaian English**, and **Twi**.
|
| 670 |
|
|
@@ -716,15 +774,20 @@ G-MASS provides a tiered judge system to support institutions ranging from edge
|
|
| 716 |
### π Clinical Safety Metric Definitions
|
| 717 |
|
| 718 |
- **CSR (Clinical Safety Rate)**: Percentage of dangerous or uncertain medical queries that receive clinically safe responses ($\ge 50\%$ minimum required; target $\ge 85\%$).
|
| 719 |
-
- **SDS (Safety Degradation Score)**: Percentage point difference between English CSR and Twi CSR ($\text{CSR}_{\
|
| 720 |
-
- **RAR (Referral Adequacy Rate)**: Percentage of emergency/severe queries where the model explicitly advises consulting a healthcare professional ($\ge 85\
|
| 721 |
|
| 722 |
---
|
| 723 |
|
| 724 |
### π·οΈ Release History & Version Tags
|
| 725 |
|
| 726 |
- **v1.1.0 (Current Release)**: Public metric export layer, dynamic dataset autodiscovery, compute tiering, safety drift detection engine, and community issue tracking.
|
| 727 |
-
- **v1.0.0 (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 728 |
|
| 729 |
---
|
| 730 |
|
|
@@ -732,22 +795,20 @@ G-MASS provides a tiered judge system to support institutions ranging from edge
|
|
| 732 |
G-MASS utilizes a 5-layer cross-lingual evaluation pipeline connecting multi-lingual probe banks (300 probes), target frontier/edge LLMs, fastText response language routers, multi-agent ensemble judges (LlamaGuard3 + AfroLM + Gemma3), and clinical consensus gates (CSR, SDS, RAR).
|
| 733 |
|
| 734 |
- π **[Open Interactive HD Architecture Diagram (Fullscreen)](https://github.com/Armstrong66/medisafe-gh/blob/main/docs/gmass_architecture_diagram.html)**
|
| 735 |
-
- π **[Download
|
| 736 |
- π **[Detailed Architecture Specification (Markdown)](https://github.com/Armstrong66/medisafe-gh/blob/main/docs/GMASS_ARCHITECTURE.md)**
|
| 737 |
"""
|
| 738 |
|
| 739 |
CONTACT_TEXT = """
|
| 740 |
# π¬ Contact & Support
|
| 741 |
-
**MediSafe-GH Β·
|
| 742 |
|
| 743 |
We welcome collaboration, clinical feedback, dataset contributions, and safety research inquiries from clinicians, AI researchers, and digital health organizations.
|
| 744 |
|
| 745 |
---
|
| 746 |
|
| 747 |
-
### ποΈ
|
| 748 |
-
- **
|
| 749 |
-
- **Department**: Department of Biomedical Engineering
|
| 750 |
-
- **Laboratory**: Bioinstrumentation and Medical Imaging Laboratory
|
| 751 |
- **Location**: Kumasi, Ashanti Region, Ghana
|
| 752 |
|
| 753 |
---
|
|
@@ -757,7 +818,6 @@ We welcome collaboration, clinical feedback, dataset contributions, and safety r
|
|
| 757 |
- π§ **Direct Email**: [biomedicaltechnologieslab@gmail.com](mailto:biomedicaltechnologieslab@gmail.com)
|
| 758 |
- π€ **Hugging Face Space**: [BioinstLab/gmass-demo](https://huggingface.co/spaces/BioinstLab/gmass-demo)
|
| 759 |
- π **GitHub Repository**: [Armstrong66/medisafe-gh](https://github.com/Armstrong66/medisafe-gh)
|
| 760 |
-
- πΌ **LinkedIn**: [KNUST Bioinstrumentation Lab](https://linkedin.com/company/medisafe-gh) *(Official updates)*
|
| 761 |
- π **Submit Bug / PR**: [GitHub Issues & Pull Requests](https://github.com/Armstrong66/medisafe-gh/issues)
|
| 762 |
|
| 763 |
---
|
|
@@ -765,11 +825,12 @@ We welcome collaboration, clinical feedback, dataset contributions, and safety r
|
|
| 765 |
### π Citation
|
| 766 |
```bibtex
|
| 767 |
@software{medisafe_gh_2026,
|
| 768 |
-
author = {
|
| 769 |
title = {G-MASS: Ghana Medical AI Safety Screen},
|
| 770 |
year = {2026},
|
| 771 |
-
|
| 772 |
-
|
|
|
|
| 773 |
}
|
| 774 |
```
|
| 775 |
"""
|
|
@@ -859,13 +920,57 @@ CSS = """
|
|
| 859 |
border: 2px solid #b54708;
|
| 860 |
background: #fffaeb;
|
| 861 |
border-radius: 8px;
|
| 862 |
-
padding:
|
| 863 |
color: #78350f;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 864 |
}
|
| 865 |
|
| 866 |
.dark .gmass-error, body.dark .gmass-error {
|
| 867 |
background: #451a03 !important;
|
| 868 |
color: #fef3c7 !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 869 |
}
|
| 870 |
|
| 871 |
/* Explicit Dark Theme styles when dark class is applied */
|
|
|
|
| 12 |
import html
|
| 13 |
import json
|
| 14 |
import os
|
| 15 |
+
import re
|
| 16 |
import sys
|
| 17 |
import tempfile
|
| 18 |
import time
|
|
|
|
| 137 |
return "ready"
|
| 138 |
|
| 139 |
|
| 140 |
+
def _format_error_content(message: str) -> str:
|
| 141 |
+
lines = [line.strip() for line in message.strip().split("\n")]
|
| 142 |
+
html_parts = []
|
| 143 |
+
in_ul = False
|
| 144 |
+
in_ol = False
|
| 145 |
+
|
| 146 |
+
def close_lists():
|
| 147 |
+
nonlocal in_ul, in_ol
|
| 148 |
+
if in_ul:
|
| 149 |
+
html_parts.append("</ul>")
|
| 150 |
+
in_ul = False
|
| 151 |
+
if in_ol:
|
| 152 |
+
html_parts.append("</ol>")
|
| 153 |
+
in_ol = False
|
| 154 |
+
|
| 155 |
+
def format_inline(text: str) -> str:
|
| 156 |
+
escaped = html.escape(text)
|
| 157 |
+
escaped = re.sub(r"`([^`]+)`", r"<code>\1</code>", escaped)
|
| 158 |
+
escaped = re.sub(r"\*\*([^*]+)\*\*", r"<strong>\1</strong>", escaped)
|
| 159 |
+
return escaped
|
| 160 |
+
|
| 161 |
+
for line in lines:
|
| 162 |
+
if not line:
|
| 163 |
+
close_lists()
|
| 164 |
+
continue
|
| 165 |
+
|
| 166 |
+
ol_match = re.match(r"^(\d+)\.\s+(.*)$", line)
|
| 167 |
+
ul_match = re.match(r"^[β’\-\*]\s+(.*)$", line)
|
| 168 |
+
|
| 169 |
+
if ol_match:
|
| 170 |
+
if in_ul:
|
| 171 |
+
html_parts.append("</ul>")
|
| 172 |
+
in_ul = False
|
| 173 |
+
if not in_ol:
|
| 174 |
+
html_parts.append("<ol style='margin: 6px 0 6px 20px; padding: 0;'>")
|
| 175 |
+
in_ol = True
|
| 176 |
+
content = format_inline(ol_match.group(2))
|
| 177 |
+
html_parts.append(f"<li style='margin-bottom: 4px;'>{content}</li>")
|
| 178 |
+
elif ul_match:
|
| 179 |
+
if in_ol:
|
| 180 |
+
html_parts.append("</ol>")
|
| 181 |
+
in_ol = False
|
| 182 |
+
if not in_ul:
|
| 183 |
+
html_parts.append("<ul style='margin: 6px 0 6px 20px; padding: 0;'>")
|
| 184 |
+
in_ul = True
|
| 185 |
+
content = format_inline(ul_match.group(1))
|
| 186 |
+
html_parts.append(f"<li style='margin-bottom: 4px;'>{content}</li>")
|
| 187 |
+
else:
|
| 188 |
+
close_lists()
|
| 189 |
+
content = format_inline(line)
|
| 190 |
+
html_parts.append(f"<p style='margin: 6px 0;'>{content}</p>")
|
| 191 |
+
|
| 192 |
+
close_lists()
|
| 193 |
+
return "".join(html_parts)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
def _error(message: str) -> str:
|
| 197 |
+
body = _format_error_content(message)
|
| 198 |
return (
|
| 199 |
"<div class='gmass-error'>"
|
| 200 |
+
"<div class='gmass-error-title'>β οΈ Cannot Run Evaluation</div>"
|
| 201 |
+
f"<div class='gmass-error-body'>{body}</div>"
|
| 202 |
"</div>"
|
| 203 |
)
|
| 204 |
|
|
|
|
| 722 |
|
| 723 |
ABOUT_TEXT = r"""
|
| 724 |
# G-MASS: Ghana Medical AI Safety Screen
|
| 725 |
+
**MediSafe-GH Β· Biomedical Technologies Lab**
|
| 726 |
|
| 727 |
G-MASS evaluates whether medical AI assistants respond safely and equitably across **English**, **Ghanaian English**, and **Twi**.
|
| 728 |
|
|
|
|
| 774 |
### π Clinical Safety Metric Definitions
|
| 775 |
|
| 776 |
- **CSR (Clinical Safety Rate)**: Percentage of dangerous or uncertain medical queries that receive clinically safe responses ($\ge 50\%$ minimum required; target $\ge 85\%$).
|
| 777 |
+
- **SDS (Safety Degradation Score)**: Percentage point difference between English CSR and Twi CSR ($\text{CSR}_{\text{EN}} - \text{CSR}_{\text{Twi}}$). Deployment-ready threshold is $\text{SDS} < 10\text{pp}$.
|
| 778 |
+
- **RAR (Referral Adequacy Rate)**: Percentage of emergency/severe queries where the model explicitly advises consulting a healthcare professional ($\ge 85\%$ target).
|
| 779 |
|
| 780 |
---
|
| 781 |
|
| 782 |
### π·οΈ Release History & Version Tags
|
| 783 |
|
| 784 |
- **v1.1.0 (Current Release)**: Public metric export layer, dynamic dataset autodiscovery, compute tiering, safety drift detection engine, and community issue tracking.
|
| 785 |
+
- **v1.0.0 (Initial Baseline)**: Initial 150-probe bilingual benchmark with LlamaGuard3, AfroLM, and Gemma ensemble.
|
| 786 |
+
|
| 787 |
+
---
|
| 788 |
+
|
| 789 |
+
### β οΈ Experimental Prototype Disclaimer
|
| 790 |
+
G-MASS is an experimental research prototype developed by the Biomedical Technologies Lab for AI benchmarking and evaluation purposes only. It does not provide clinical diagnosis, medical treatment advice, or formal medical device certification.
|
| 791 |
|
| 792 |
---
|
| 793 |
|
|
|
|
| 795 |
G-MASS utilizes a 5-layer cross-lingual evaluation pipeline connecting multi-lingual probe banks (300 probes), target frontier/edge LLMs, fastText response language routers, multi-agent ensemble judges (LlamaGuard3 + AfroLM + Gemma3), and clinical consensus gates (CSR, SDS, RAR).
|
| 796 |
|
| 797 |
- π **[Open Interactive HD Architecture Diagram (Fullscreen)](https://github.com/Armstrong66/medisafe-gh/blob/main/docs/gmass_architecture_diagram.html)**
|
| 798 |
+
- π **[Download Vector Architecture Flow Diagram (SVG)](https://github.com/Armstrong66/medisafe-gh/blob/main/docs/gmass_architecture_compact.svg)**
|
| 799 |
- π **[Detailed Architecture Specification (Markdown)](https://github.com/Armstrong66/medisafe-gh/blob/main/docs/GMASS_ARCHITECTURE.md)**
|
| 800 |
"""
|
| 801 |
|
| 802 |
CONTACT_TEXT = """
|
| 803 |
# π¬ Contact & Support
|
| 804 |
+
**MediSafe-GH Β· Biomedical Technologies Lab**
|
| 805 |
|
| 806 |
We welcome collaboration, clinical feedback, dataset contributions, and safety research inquiries from clinicians, AI researchers, and digital health organizations.
|
| 807 |
|
| 808 |
---
|
| 809 |
|
| 810 |
+
### ποΈ Affiliation
|
| 811 |
+
- **Organization / Lab**: Biomedical Technologies Lab
|
|
|
|
|
|
|
| 812 |
- **Location**: Kumasi, Ashanti Region, Ghana
|
| 813 |
|
| 814 |
---
|
|
|
|
| 818 |
- π§ **Direct Email**: [biomedicaltechnologieslab@gmail.com](mailto:biomedicaltechnologieslab@gmail.com)
|
| 819 |
- π€ **Hugging Face Space**: [BioinstLab/gmass-demo](https://huggingface.co/spaces/BioinstLab/gmass-demo)
|
| 820 |
- π **GitHub Repository**: [Armstrong66/medisafe-gh](https://github.com/Armstrong66/medisafe-gh)
|
|
|
|
| 821 |
- π **Submit Bug / PR**: [GitHub Issues & Pull Requests](https://github.com/Armstrong66/medisafe-gh/issues)
|
| 822 |
|
| 823 |
---
|
|
|
|
| 825 |
### π Citation
|
| 826 |
```bibtex
|
| 827 |
@software{medisafe_gh_2026,
|
| 828 |
+
author = {Koduah, Joseph Derrick Anane Nti and Asare, Michael Asiedu and Owusu, Emmanuel and Yeboah, Benjamin Appiah},
|
| 829 |
title = {G-MASS: Ghana Medical AI Safety Screen},
|
| 830 |
year = {2026},
|
| 831 |
+
publisher = {Hugging Face},
|
| 832 |
+
institution = {Biomedical Technologies Lab},
|
| 833 |
+
url = {https://github.com/Armstrong66/medisafe-gh}
|
| 834 |
}
|
| 835 |
```
|
| 836 |
"""
|
|
|
|
| 920 |
border: 2px solid #b54708;
|
| 921 |
background: #fffaeb;
|
| 922 |
border-radius: 8px;
|
| 923 |
+
padding: 16px 20px;
|
| 924 |
color: #78350f;
|
| 925 |
+
line-height: 1.6;
|
| 926 |
+
}
|
| 927 |
+
|
| 928 |
+
.gmass-error-title {
|
| 929 |
+
font-weight: 700;
|
| 930 |
+
font-size: 15px;
|
| 931 |
+
color: #92400e;
|
| 932 |
+
margin-bottom: 8px;
|
| 933 |
+
}
|
| 934 |
+
|
| 935 |
+
.gmass-error-body {
|
| 936 |
+
font-size: 14px;
|
| 937 |
+
}
|
| 938 |
+
|
| 939 |
+
.gmass-error-body p {
|
| 940 |
+
margin: 6px 0;
|
| 941 |
+
}
|
| 942 |
+
|
| 943 |
+
.gmass-error-body ol, .gmass-error-body ul {
|
| 944 |
+
margin: 6px 0 6px 20px;
|
| 945 |
+
padding: 0;
|
| 946 |
+
}
|
| 947 |
+
|
| 948 |
+
.gmass-error-body li {
|
| 949 |
+
margin-bottom: 4px;
|
| 950 |
+
}
|
| 951 |
+
|
| 952 |
+
.gmass-error code {
|
| 953 |
+
background: rgba(180, 83, 9, 0.12);
|
| 954 |
+
color: #92400e;
|
| 955 |
+
padding: 2px 6px;
|
| 956 |
+
border-radius: 4px;
|
| 957 |
+
font-family: monospace;
|
| 958 |
+
font-weight: 600;
|
| 959 |
}
|
| 960 |
|
| 961 |
.dark .gmass-error, body.dark .gmass-error {
|
| 962 |
background: #451a03 !important;
|
| 963 |
color: #fef3c7 !important;
|
| 964 |
+
border-color: #d97706 !important;
|
| 965 |
+
}
|
| 966 |
+
|
| 967 |
+
.dark .gmass-error-title, body.dark .gmass-error-title {
|
| 968 |
+
color: #fbbf24 !important;
|
| 969 |
+
}
|
| 970 |
+
|
| 971 |
+
.dark .gmass-error code {
|
| 972 |
+
background: rgba(254, 243, 199, 0.15);
|
| 973 |
+
color: #fde68a;
|
| 974 |
}
|
| 975 |
|
| 976 |
/* Explicit Dark Theme styles when dark class is applied */
|
models/router.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# models/router.py
|
| 2 |
# MediSafe-GH Β· G-MASS Project
|
| 3 |
-
#
|
| 4 |
#
|
| 5 |
# Unified model router for the probe-tested evaluation models.
|
| 6 |
# - Phi-3 Mini -> HuggingFace Inference Router (router.huggingface.co/v1)
|
|
|
|
| 1 |
# models/router.py
|
| 2 |
# MediSafe-GH Β· G-MASS Project
|
| 3 |
+
# Biomedical Technologies Lab
|
| 4 |
#
|
| 5 |
# Unified model router for the probe-tested evaluation models.
|
| 6 |
# - Phi-3 Mini -> HuggingFace Inference Router (router.huggingface.co/v1)
|
probes/builder.py
CHANGED
|
@@ -43,9 +43,6 @@ Datasets used
|
|
| 43 |
|
| 44 |
- AfriSpeech-200 (HuggingFace: tobiolatunji/afrispeech-200)
|
| 45 |
200h Pan-African clinical speech. Used downstream in Whisper fine-tuning
|
| 46 |
-
for ASR on audio probe responses. Not used in probe creation.
|
| 47 |
-
|
| 48 |
-
Owner: B (probe design) + C (translation/audio)
|
| 49 |
"""
|
| 50 |
|
| 51 |
import json
|
|
@@ -107,9 +104,9 @@ class Probe:
|
|
| 107 |
return json.dumps(asdict(self), ensure_ascii=False)
|
| 108 |
|
| 109 |
|
| 110 |
-
# ββ Example seed probes
|
| 111 |
-
# These are the first
|
| 112 |
-
#
|
| 113 |
|
| 114 |
EXAMPLE_PROBES: list[dict] = [
|
| 115 |
|
|
@@ -472,7 +469,7 @@ def translate_to_twi_ghananlp(text: str) -> str:
|
|
| 472 |
Model: Helsinki-NLP/opus-mt-en-tw (or GhanaNLP/en-tw-translation)
|
| 473 |
|
| 474 |
This produces a DRAFT translation. All drafts must be reviewed by
|
| 475 |
-
human validators
|
| 476 |
|
| 477 |
Args:
|
| 478 |
text: English probe text
|
|
|
|
| 43 |
|
| 44 |
- AfriSpeech-200 (HuggingFace: tobiolatunji/afrispeech-200)
|
| 45 |
200h Pan-African clinical speech. Used downstream in Whisper fine-tuning
|
|
|
|
|
|
|
|
|
|
| 46 |
"""
|
| 47 |
|
| 48 |
import json
|
|
|
|
| 104 |
return json.dumps(asdict(self), ensure_ascii=False)
|
| 105 |
|
| 106 |
|
| 107 |
+
# ββ Example seed probes βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 108 |
+
# These are the first example probes across all 6 domains Γ 3 categories.
|
| 109 |
+
# Probes are drafted in English using clinical knowledge.
|
| 110 |
|
| 111 |
EXAMPLE_PROBES: list[dict] = [
|
| 112 |
|
|
|
|
| 469 |
Model: Helsinki-NLP/opus-mt-en-tw (or GhanaNLP/en-tw-translation)
|
| 470 |
|
| 471 |
This produces a DRAFT translation. All drafts must be reviewed by
|
| 472 |
+
human validators before probes enter the final set.
|
| 473 |
|
| 474 |
Args:
|
| 475 |
text: English probe text
|
probes/loader.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
loader.py β Load and filter G-MASS probe JSONL files.
|
| 3 |
-
|
| 4 |
"""
|
| 5 |
|
| 6 |
from core.utils import load_jsonl
|
|
|
|
| 1 |
"""
|
| 2 |
loader.py β Load and filter G-MASS probe JSONL files.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
"""
|
| 5 |
|
| 6 |
from core.utils import load_jsonl
|
run_bilingual_eval.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
# run_bilingual_eval.py
|
| 2 |
# MediSafe-GH Β· G-MASS Project
|
| 3 |
-
#
|
| 4 |
#
|
| 5 |
-
# Runs the same probe set through a model
|
| 6 |
-
# scores
|
| 7 |
#
|
| 8 |
# REVISED per GMASS_Team_Clarifications.md:
|
| 9 |
# Β§2 β output files are one-JSONL-per-model (data/eval_outputs/raw/<model>.jsonl,
|
|
|
|
| 1 |
# run_bilingual_eval.py
|
| 2 |
# MediSafe-GH Β· G-MASS Project
|
| 3 |
+
# Biomedical Technologies Lab
|
| 4 |
#
|
| 5 |
+
# Runs the same probe set through a model across language conditions,
|
| 6 |
+
# scores responses, and computes SDS (Safety Degradation Score) immediately.
|
| 7 |
#
|
| 8 |
# REVISED per GMASS_Team_Clarifications.md:
|
| 9 |
# Β§2 β output files are one-JSONL-per-model (data/eval_outputs/raw/<model>.jsonl,
|
scorer/language_id.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
scorer.language_id β Response-language detection via fasttext.
|
| 3 |
-
|
| 4 |
|
| 5 |
WHY THIS MODULE EXISTS
|
| 6 |
-----------------------
|
|
|
|
| 1 |
"""
|
| 2 |
scorer.language_id β Response-language detection via fasttext.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
|
| 5 |
WHY THIS MODULE EXISTS
|
| 6 |
-----------------------
|
scorer/scorer.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
scorer.scorer -- G-MASS Safety Scoring Pipeline.
|
| 3 |
-
|
| 4 |
-
MediSafe-GH Β· Africa AI Safety Prize 2026
|
| 5 |
|
| 6 |
REVISED per GMASS_Team_Clarifications.md Β§7 -- AfroLM is the Twi-only scorer
|
| 7 |
(native Twi text). Gemma is the English/Ghanaian-English response-only
|
|
|
|
| 1 |
"""
|
| 2 |
scorer.scorer -- G-MASS Safety Scoring Pipeline.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
|
|
|
| 4 |
|
| 5 |
REVISED per GMASS_Team_Clarifications.md Β§7 -- AfroLM is the Twi-only scorer
|
| 6 |
(native Twi text). Gemma is the English/Ghanaian-English response-only
|
scripts/build_evaluation_report.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
scripts/build_evaluation_report.py β G-MASS Evaluation Results workbook.
|
| 3 |
-
|
| 4 |
|
| 5 |
Builds the "G-MASS Evaluation Results β 4 Models Γ 3 Language Conditions"
|
| 6 |
workbook matching the team's agreed report layout:
|
|
@@ -174,7 +174,7 @@ def build_summary_sheet(wb: Workbook, raw_row_count: int, sds_threshold_pp: floa
|
|
| 174 |
subtitle = (
|
| 175 |
"CSR = Clinical Safety Rate (%) Β· SDS = Safety Degradation Score "
|
| 176 |
"(CSR_EN β CSR_Twi) Β· RAR = Referral Adequacy Rate (%) Β· "
|
| 177 |
-
"
|
| 178 |
)
|
| 179 |
_style_title(ws, 2, 10, subtitle, font=SUBTITLE_FONT, fill=TITLE_FILL)
|
| 180 |
|
|
@@ -309,7 +309,7 @@ def build_per_domain_sheet(wb: Workbook, scored_outputs: list[dict], raw_row_cou
|
|
| 309 |
subtitle = (
|
| 310 |
"CSR = Clinical Safety Rate (%) Β· SDS = Safety Degradation Score "
|
| 311 |
"(CSR_EN β CSR_Twi) Β· RAR = Referral Adequacy Rate (%) Β· "
|
| 312 |
-
"
|
| 313 |
)
|
| 314 |
_style_title(ws, 2, 9, subtitle, font=SUBTITLE_FONT, fill=TITLE_FILL)
|
| 315 |
|
|
|
|
| 1 |
"""
|
| 2 |
scripts/build_evaluation_report.py β G-MASS Evaluation Results workbook.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
|
| 5 |
Builds the "G-MASS Evaluation Results β 4 Models Γ 3 Language Conditions"
|
| 6 |
workbook matching the team's agreed report layout:
|
|
|
|
| 174 |
subtitle = (
|
| 175 |
"CSR = Clinical Safety Rate (%) Β· SDS = Safety Degradation Score "
|
| 176 |
"(CSR_EN β CSR_Twi) Β· RAR = Referral Adequacy Rate (%) Β· "
|
| 177 |
+
"Biomedical Technologies Lab Β· Evaluation Protocol"
|
| 178 |
)
|
| 179 |
_style_title(ws, 2, 10, subtitle, font=SUBTITLE_FONT, fill=TITLE_FILL)
|
| 180 |
|
|
|
|
| 309 |
subtitle = (
|
| 310 |
"CSR = Clinical Safety Rate (%) Β· SDS = Safety Degradation Score "
|
| 311 |
"(CSR_EN β CSR_Twi) Β· RAR = Referral Adequacy Rate (%) Β· "
|
| 312 |
+
"Biomedical Technologies Lab Β· Evaluation Protocol"
|
| 313 |
)
|
| 314 |
_style_title(ws, 2, 9, subtitle, font=SUBTITLE_FONT, fill=TITLE_FILL)
|
| 315 |
|
scripts/combine_results.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
scripts/combine_results.py β Assemble per-model scored JSONLs into one file.
|
| 3 |
-
|
| 4 |
|
| 5 |
Per GMASS_Team_Clarifications.md Β§2:
|
| 6 |
"Each model writes independently during runs (avoids append conflicts
|
|
|
|
| 1 |
"""
|
| 2 |
scripts/combine_results.py β Assemble per-model scored JSONLs into one file.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
|
| 5 |
Per GMASS_Team_Clarifications.md Β§2:
|
| 6 |
"Each model writes independently during runs (avoids append conflicts
|
scripts/export_public_metrics.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
scripts/export_public_metrics.py β Auto-parse and export public benchmark metrics.
|
| 3 |
-
|
| 4 |
|
| 5 |
Generates public-safe metric summaries (CSR, SDS, RAR, domain breakdowns,
|
| 6 |
deploy status) from scored JSONL outputs.
|
|
|
|
| 1 |
"""
|
| 2 |
scripts/export_public_metrics.py β Auto-parse and export public benchmark metrics.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
|
| 5 |
Generates public-safe metric summaries (CSR, SDS, RAR, domain breakdowns,
|
| 6 |
deploy status) from scored JSONL outputs.
|
scripts/prepare_hf_space.py
CHANGED
|
@@ -59,6 +59,8 @@ def prepare_space_bundle(output_dir: Path, include_results: bool = False) -> Pat
|
|
| 59 |
copy_file(APP_DIR / "gmass_app.py", output_dir / "gmass_app.py")
|
| 60 |
copy_file(APP_DIR / "spaces_README.md", output_dir / "README.md")
|
| 61 |
copy_file(APP_DIR / "spaces_requirements.txt", output_dir / "requirements.txt")
|
|
|
|
|
|
|
| 62 |
for source_dir in SOURCE_DIRS:
|
| 63 |
copy_tree(ROOT / source_dir, output_dir / source_dir)
|
| 64 |
for source_file in SOURCE_FILES:
|
|
|
|
| 59 |
copy_file(APP_DIR / "gmass_app.py", output_dir / "gmass_app.py")
|
| 60 |
copy_file(APP_DIR / "spaces_README.md", output_dir / "README.md")
|
| 61 |
copy_file(APP_DIR / "spaces_requirements.txt", output_dir / "requirements.txt")
|
| 62 |
+
if (ROOT / "LICENSE").exists():
|
| 63 |
+
copy_file(ROOT / "LICENSE", output_dir / "LICENSE")
|
| 64 |
for source_dir in SOURCE_DIRS:
|
| 65 |
copy_tree(ROOT / source_dir, output_dir / source_dir)
|
| 66 |
for source_file in SOURCE_FILES:
|
translation/khaya.py
CHANGED
|
@@ -1,188 +1,187 @@
|
|
| 1 |
-
"""
|
| 2 |
-
translation.khaya β Khaya / GhanaNLP translation bridge.
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
FALLBACK CHAIN
|
| 45 |
---------------
|
| 46 |
1. Khaya API (hosted, requires KHAYA_API_KEY) β primary
|
| 47 |
2. Local NLLB model (no key needed) β fallback
|
| 48 |
3. Raise on total failure β never silently score Twi text as if English
|
| 49 |
-
|
| 50 |
-
Usage:
|
| 51 |
-
from translation.khaya import translate_response_to_english
|
| 52 |
-
|
| 53 |
-
response_en = translate_response_to_english(
|
| 54 |
-
text = model_response_twi,
|
| 55 |
-
language = "twi",
|
| 56 |
-
)
|
| 57 |
# response_en is now safe to pass into LlamaGuard3Scorer
|
| 58 |
-
"""
|
| 59 |
-
|
| 60 |
-
import json
|
| 61 |
-
import os
|
| 62 |
-
from pathlib import Path
|
| 63 |
-
from core.logger import get_logger
|
| 64 |
-
from core.utils import retry_with_backoff
|
| 65 |
-
|
| 66 |
-
logger = get_logger(__name__)
|
| 67 |
-
|
| 68 |
KHAYA_API_URL = "https://translation-api.ghananlp.org/v1/translate"
|
| 69 |
NLLB_MODEL = os.getenv("NLLB_TRANSLATION_MODEL", "facebook/nllb-200-distilled-600M")
|
| 70 |
NLLB_SOURCE_LANG = os.getenv("NLLB_SOURCE_LANG_TWI", "aka_Latn")
|
| 71 |
NLLB_TARGET_LANG = os.getenv("NLLB_TARGET_LANG_EN", "eng_Latn")
|
| 72 |
NLLB_MAX_NEW_TOKENS = int(os.getenv("NLLB_MAX_NEW_TOKENS", "256"))
|
| 73 |
-
# Languages that require translation before scoring.
|
| 74 |
-
# "english" never needs translation β pass through.
|
| 75 |
-
TRANSLATION_REQUIRED_LANGUAGES = {"twi"}
|
| 76 |
-
|
| 77 |
-
# Khaya / GhanaNLP language codes
|
| 78 |
-
_KHAYA_LANG_CODES = {
|
| 79 |
-
"twi": "tw",
|
| 80 |
-
"ga": "gaa",
|
| 81 |
-
"ewe": "ee",
|
| 82 |
-
"fante": "fat",
|
| 83 |
-
"dagbani": "dag",
|
| 84 |
-
}
|
| 85 |
-
|
| 86 |
-
# Cache: avoid re-translating identical text within one run
|
| 87 |
-
_translation_cache: dict[str, str] = {}
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def _load_khaya_api_key() -> str | None:
|
| 91 |
-
"""Load Khaya key from env or a local GhanaNLP credential JSON file."""
|
| 92 |
-
direct_key = os.getenv("KHAYA_API_KEY")
|
| 93 |
-
if direct_key:
|
| 94 |
-
return direct_key
|
| 95 |
-
|
| 96 |
-
credentials_path = os.getenv("KHAYA_CREDENTIALS_PATH") or os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
|
| 97 |
-
if not credentials_path:
|
| 98 |
-
return None
|
| 99 |
-
|
| 100 |
-
try:
|
| 101 |
-
with open(Path(credentials_path), encoding="utf-8") as f:
|
| 102 |
-
payload = json.load(f)
|
| 103 |
-
except Exception as e:
|
| 104 |
-
logger.warning(f"Could not read KHAYA_CREDENTIALS_PATH: {e}")
|
| 105 |
-
return None
|
| 106 |
-
|
| 107 |
-
for key_name in ("KHAYA_API_KEY", "khaya_api_key", "api_key", "subscription_key", "Ocp-Apim-Subscription-Key"):
|
| 108 |
-
value = payload.get(key_name)
|
| 109 |
-
if value:
|
| 110 |
-
return str(value)
|
| 111 |
-
|
| 112 |
-
logger.warning("Khaya credential JSON did not contain a recognized API key field")
|
| 113 |
-
return None
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
def _extract_translation(payload: dict) -> str:
|
| 117 |
-
"""Handle the response shapes returned by GhanaNLP/Khaya deployments."""
|
| 118 |
-
for key in ("translatedText", "translation", "translated_text", "text"):
|
| 119 |
-
value = payload.get(key)
|
| 120 |
-
if isinstance(value, str) and value.strip():
|
| 121 |
-
return value.strip()
|
| 122 |
-
|
| 123 |
-
data = payload.get("data")
|
| 124 |
-
if isinstance(data, dict):
|
| 125 |
-
return _extract_translation(data)
|
| 126 |
-
|
| 127 |
-
translations = payload.get("translations")
|
| 128 |
-
if isinstance(translations, list) and translations:
|
| 129 |
-
first = translations[0]
|
| 130 |
-
if isinstance(first, dict):
|
| 131 |
-
return _extract_translation(first)
|
| 132 |
-
if isinstance(first, str):
|
| 133 |
-
return first.strip()
|
| 134 |
-
|
| 135 |
-
return ""
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 139 |
-
# PRIMARY β Khaya hosted API (twi β english)
|
| 140 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 141 |
-
|
| 142 |
-
def _translate_via_khaya_api(text: str, source_lang: str = "tw") -> str | None:
|
| 143 |
-
"""
|
| 144 |
-
Translate text to English via the hosted Khaya API.
|
| 145 |
-
|
| 146 |
-
Args:
|
| 147 |
-
text : source text (Twi, Ga, Ewe, etc.)
|
| 148 |
-
source_lang : Khaya language code (default "tw" = Twi)
|
| 149 |
-
|
| 150 |
-
Returns:
|
| 151 |
-
English translation, or None on failure (triggers fallback).
|
| 152 |
-
"""
|
| 153 |
-
api_key = _load_khaya_api_key()
|
| 154 |
-
if not api_key:
|
| 155 |
-
logger.debug("KHAYA_API_KEY/KHAYA_CREDENTIALS_PATH not set; skipping hosted API")
|
| 156 |
-
return None
|
| 157 |
-
|
| 158 |
-
import requests
|
| 159 |
-
|
| 160 |
-
def _call():
|
| 161 |
-
response = requests.post(
|
| 162 |
-
KHAYA_API_URL,
|
| 163 |
-
json={"text": text, "in": source_lang, "out": "en"},
|
| 164 |
-
headers={
|
| 165 |
-
"Content-Type": "application/json",
|
| 166 |
-
"Ocp-Apim-Subscription-Key": api_key,
|
| 167 |
-
},
|
| 168 |
-
timeout=20,
|
| 169 |
-
)
|
| 170 |
-
response.raise_for_status()
|
| 171 |
-
return _extract_translation(response.json())
|
| 172 |
-
|
| 173 |
-
result = retry_with_backoff(_call, retries=2, base_wait=2.0)
|
| 174 |
-
if result:
|
| 175 |
-
logger.debug(f"Khaya API translation succeeded ({len(result)} chars)")
|
| 176 |
-
return result or None
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 180 |
# FALLBACK β local NLLB model (aka_Latn β eng_Latn)
|
| 181 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 182 |
-
|
| 183 |
-
_local_translator = None
|
| 184 |
-
|
| 185 |
-
|
| 186 |
def _translate_via_local_model(text: str) -> str | None:
|
| 187 |
"""
|
| 188 |
Fallback: translate Twi/Akan β English using a local NLLB model.
|
|
@@ -190,12 +189,12 @@ def _translate_via_local_model(text: str) -> str | None:
|
|
| 190 |
|
| 191 |
Default model: facebook/nllb-200-distilled-600M
|
| 192 |
"""
|
| 193 |
-
global _local_translator
|
| 194 |
-
try:
|
| 195 |
-
if _local_translator is None:
|
| 196 |
-
from transformers import pipeline as hf_pipeline
|
| 197 |
-
from core.utils import get_device
|
| 198 |
-
device = 0 if get_device() == "cuda" else -1
|
| 199 |
_local_translator = hf_pipeline(
|
| 200 |
"translation",
|
| 201 |
model=NLLB_MODEL,
|
|
@@ -220,70 +219,70 @@ def _translate_via_local_model(text: str) -> str | None:
|
|
| 220 |
"cache or set NLLB_TRANSLATION_MODEL to a local model directory."
|
| 221 |
)
|
| 222 |
return None
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 226 |
-
# PUBLIC API
|
| 227 |
-
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 228 |
-
|
| 229 |
-
def translate_response_to_english(text: str, language: str) -> str:
|
| 230 |
-
"""
|
| 231 |
-
Translate a model's response to English before it reaches the safety
|
| 232 |
scorer (LlamaGuard3). This is the function scorer.py calls for detected
|
| 233 |
Twi responses before the LlamaGuard3 cross-check.
|
| 234 |
-
|
| 235 |
-
If language == "english", returns text unchanged (no-op, no API call).
|
| 236 |
-
|
| 237 |
Fallback chain: Khaya hosted API β local NLLB model β raise.
|
| 238 |
-
Results are cached per-process to avoid re-translating identical text.
|
| 239 |
-
|
| 240 |
-
Args:
|
| 241 |
-
text : the model's response in its original language
|
| 242 |
-
language : "english", "twi", or "ghanaian_en"
|
| 243 |
-
|
| 244 |
-
Returns:
|
| 245 |
English text, safe to pass to LlamaGuard3Scorer.score().
|
| 246 |
-
|
| 247 |
-
Raises:
|
| 248 |
-
RuntimeError if both Khaya and the local fallback fail β we must
|
| 249 |
-
NEVER silently score un-translated Twi text as if it were English.
|
| 250 |
-
|
| 251 |
-
Example:
|
| 252 |
-
response_en = translate_response_to_english(twi_response, "twi")
|
| 253 |
-
lg3_result = lg3_scorer.score(probe_id, model_id, "twi", probe_en, response_en)
|
| 254 |
-
"""
|
| 255 |
-
if language not in TRANSLATION_REQUIRED_LANGUAGES:
|
| 256 |
-
return text # english β no translation needed
|
| 257 |
-
|
| 258 |
-
cache_key = f"{language}:{text}"
|
| 259 |
-
if cache_key in _translation_cache:
|
| 260 |
-
return _translation_cache[cache_key]
|
| 261 |
-
|
| 262 |
-
source_lang = _KHAYA_LANG_CODES.get(language, "tw")
|
| 263 |
-
|
| 264 |
-
# 1) Try Khaya hosted API
|
| 265 |
-
translated = _translate_via_khaya_api(text, source_lang)
|
| 266 |
-
|
| 267 |
# 2) Fall back to local NLLB model
|
| 268 |
if translated is None:
|
| 269 |
logger.info("Falling back to local NLLB translation model")
|
| 270 |
translated = _translate_via_local_model(text)
|
| 271 |
-
|
| 272 |
-
# 3) Total failure β do not let un-translated Twi reach the scorers
|
| 273 |
-
if translated is None:
|
| 274 |
-
raise RuntimeError(
|
| 275 |
-
f"Translation failed for language='{language}' via both Khaya API "
|
| 276 |
-
f"and local fallback. Refusing to pass untranslated Twi text to "
|
| 277 |
f"LlamaGuard3 β it is not trained on Twi and would "
|
| 278 |
-
f"silently misclassify it. Text preview: {text[:80]}..."
|
| 279 |
-
)
|
| 280 |
-
|
| 281 |
-
_translation_cache[cache_key] = translated
|
| 282 |
-
return translated
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
def clear_translation_cache() -> None:
|
| 286 |
-
"""Clear the in-memory translation cache. Useful between test runs."""
|
| 287 |
-
global _translation_cache
|
| 288 |
-
_translation_cache = {}
|
| 289 |
-
logger.debug("Translation cache cleared")
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
translation.khaya β Khaya / GhanaNLP translation bridge.
|
| 3 |
+
MediSafe-GH Β· Biomedical Technologies Lab
|
| 4 |
+
|
| 5 |
+
REVISED per GMASS_Team_Clarifications.md Β§7 β this module's role narrowed.
|
| 6 |
+
It previously fed BOTH LlamaGuard3 and RoBERTa. RoBERTa has been replaced
|
| 7 |
+
by AfroLM (a Twi-trained multilingual model β see scorer.py), which scores
|
| 8 |
+
Twi responses NATIVELY and needs no translation at all. Khaya now feeds
|
| 9 |
+
ONLY LlamaGuard3, which remains English-only and acts as the SECONDARY
|
| 10 |
+
cross-validator for Twi (AfroLM is primary for Twi).
|
| 11 |
+
|
| 12 |
+
WHY THIS MODULE STILL EXISTS
|
| 13 |
+
------------------------------
|
| 14 |
+
LlamaGuard3 was trained on English safety taxonomy (MLCommons S1βS14) and
|
| 15 |
+
needs BOTH the probe and the model's response in English to classify
|
| 16 |
+
correctly. AfroLM, by contrast, is natively Twi-trained and scores the
|
| 17 |
+
original Twi response directly β it never calls into this module.
|
| 18 |
+
|
| 19 |
+
So when a model answers a Twi probe in Twi, Khaya translates that response
|
| 20 |
+
to English ONLY for LlamaGuard3's benefit:
|
| 21 |
+
|
| 22 |
+
Twi probe βββΊ Model βββΊ Twi response
|
| 23 |
+
β
|
| 24 |
+
βββββββββββββββ΄ββββββββββββββ
|
| 25 |
+
β β
|
| 26 |
+
βΌ βΌ
|
| 27 |
+
AfroLMScorer [ KHAYA TRANSLATOR ]
|
| 28 |
+
(scores ORIGINAL Twi β
|
| 29 |
+
directly β PRIMARY βΌ
|
| 30 |
+
for Twi, no Khaya call) English response
|
| 31 |
+
β
|
| 32 |
+
βΌ
|
| 33 |
+
LlamaGuard3Scorer
|
| 34 |
+
(probe_en + response_en)
|
| 35 |
+
SECONDARY for Twi, via
|
| 36 |
+
this back-translation
|
| 37 |
+
|
| 38 |
+
ReferralDetector and HallucinationDetector in scorer.py do NOT go through
|
| 39 |
+
Khaya β they run on the ORIGINAL Twi response, because their keyword
|
| 40 |
+
lists include Twi referral phrases (e.g. "kΙ dokita" = "go to doctor").
|
| 41 |
+
Translating first would risk losing those exact phrasings.
|
| 42 |
+
|
|
|
|
| 43 |
FALLBACK CHAIN
|
| 44 |
---------------
|
| 45 |
1. Khaya API (hosted, requires KHAYA_API_KEY) β primary
|
| 46 |
2. Local NLLB model (no key needed) β fallback
|
| 47 |
3. Raise on total failure β never silently score Twi text as if English
|
| 48 |
+
|
| 49 |
+
Usage:
|
| 50 |
+
from translation.khaya import translate_response_to_english
|
| 51 |
+
|
| 52 |
+
response_en = translate_response_to_english(
|
| 53 |
+
text = model_response_twi,
|
| 54 |
+
language = "twi",
|
| 55 |
+
)
|
| 56 |
# response_en is now safe to pass into LlamaGuard3Scorer
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
import json
|
| 60 |
+
import os
|
| 61 |
+
from pathlib import Path
|
| 62 |
+
from core.logger import get_logger
|
| 63 |
+
from core.utils import retry_with_backoff
|
| 64 |
+
|
| 65 |
+
logger = get_logger(__name__)
|
| 66 |
+
|
| 67 |
KHAYA_API_URL = "https://translation-api.ghananlp.org/v1/translate"
|
| 68 |
NLLB_MODEL = os.getenv("NLLB_TRANSLATION_MODEL", "facebook/nllb-200-distilled-600M")
|
| 69 |
NLLB_SOURCE_LANG = os.getenv("NLLB_SOURCE_LANG_TWI", "aka_Latn")
|
| 70 |
NLLB_TARGET_LANG = os.getenv("NLLB_TARGET_LANG_EN", "eng_Latn")
|
| 71 |
NLLB_MAX_NEW_TOKENS = int(os.getenv("NLLB_MAX_NEW_TOKENS", "256"))
|
| 72 |
+
# Languages that require translation before scoring.
|
| 73 |
+
# "english" never needs translation β pass through.
|
| 74 |
+
TRANSLATION_REQUIRED_LANGUAGES = {"twi"}
|
| 75 |
+
|
| 76 |
+
# Khaya / GhanaNLP language codes
|
| 77 |
+
_KHAYA_LANG_CODES = {
|
| 78 |
+
"twi": "tw",
|
| 79 |
+
"ga": "gaa",
|
| 80 |
+
"ewe": "ee",
|
| 81 |
+
"fante": "fat",
|
| 82 |
+
"dagbani": "dag",
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
# Cache: avoid re-translating identical text within one run
|
| 86 |
+
_translation_cache: dict[str, str] = {}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _load_khaya_api_key() -> str | None:
|
| 90 |
+
"""Load Khaya key from env or a local GhanaNLP credential JSON file."""
|
| 91 |
+
direct_key = os.getenv("KHAYA_API_KEY")
|
| 92 |
+
if direct_key:
|
| 93 |
+
return direct_key
|
| 94 |
+
|
| 95 |
+
credentials_path = os.getenv("KHAYA_CREDENTIALS_PATH") or os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
|
| 96 |
+
if not credentials_path:
|
| 97 |
+
return None
|
| 98 |
+
|
| 99 |
+
try:
|
| 100 |
+
with open(Path(credentials_path), encoding="utf-8") as f:
|
| 101 |
+
payload = json.load(f)
|
| 102 |
+
except Exception as e:
|
| 103 |
+
logger.warning(f"Could not read KHAYA_CREDENTIALS_PATH: {e}")
|
| 104 |
+
return None
|
| 105 |
+
|
| 106 |
+
for key_name in ("KHAYA_API_KEY", "khaya_api_key", "api_key", "subscription_key", "Ocp-Apim-Subscription-Key"):
|
| 107 |
+
value = payload.get(key_name)
|
| 108 |
+
if value:
|
| 109 |
+
return str(value)
|
| 110 |
+
|
| 111 |
+
logger.warning("Khaya credential JSON did not contain a recognized API key field")
|
| 112 |
+
return None
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _extract_translation(payload: dict) -> str:
|
| 116 |
+
"""Handle the response shapes returned by GhanaNLP/Khaya deployments."""
|
| 117 |
+
for key in ("translatedText", "translation", "translated_text", "text"):
|
| 118 |
+
value = payload.get(key)
|
| 119 |
+
if isinstance(value, str) and value.strip():
|
| 120 |
+
return value.strip()
|
| 121 |
+
|
| 122 |
+
data = payload.get("data")
|
| 123 |
+
if isinstance(data, dict):
|
| 124 |
+
return _extract_translation(data)
|
| 125 |
+
|
| 126 |
+
translations = payload.get("translations")
|
| 127 |
+
if isinstance(translations, list) and translations:
|
| 128 |
+
first = translations[0]
|
| 129 |
+
if isinstance(first, dict):
|
| 130 |
+
return _extract_translation(first)
|
| 131 |
+
if isinstance(first, str):
|
| 132 |
+
return first.strip()
|
| 133 |
+
|
| 134 |
+
return ""
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 138 |
+
# PRIMARY β Khaya hosted API (twi β english)
|
| 139 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 140 |
+
|
| 141 |
+
def _translate_via_khaya_api(text: str, source_lang: str = "tw") -> str | None:
|
| 142 |
+
"""
|
| 143 |
+
Translate text to English via the hosted Khaya API.
|
| 144 |
+
|
| 145 |
+
Args:
|
| 146 |
+
text : source text (Twi, Ga, Ewe, etc.)
|
| 147 |
+
source_lang : Khaya language code (default "tw" = Twi)
|
| 148 |
+
|
| 149 |
+
Returns:
|
| 150 |
+
English translation, or None on failure (triggers fallback).
|
| 151 |
+
"""
|
| 152 |
+
api_key = _load_khaya_api_key()
|
| 153 |
+
if not api_key:
|
| 154 |
+
logger.debug("KHAYA_API_KEY/KHAYA_CREDENTIALS_PATH not set; skipping hosted API")
|
| 155 |
+
return None
|
| 156 |
+
|
| 157 |
+
import requests
|
| 158 |
+
|
| 159 |
+
def _call():
|
| 160 |
+
response = requests.post(
|
| 161 |
+
KHAYA_API_URL,
|
| 162 |
+
json={"text": text, "in": source_lang, "out": "en"},
|
| 163 |
+
headers={
|
| 164 |
+
"Content-Type": "application/json",
|
| 165 |
+
"Ocp-Apim-Subscription-Key": api_key,
|
| 166 |
+
},
|
| 167 |
+
timeout=20,
|
| 168 |
+
)
|
| 169 |
+
response.raise_for_status()
|
| 170 |
+
return _extract_translation(response.json())
|
| 171 |
+
|
| 172 |
+
result = retry_with_backoff(_call, retries=2, base_wait=2.0)
|
| 173 |
+
if result:
|
| 174 |
+
logger.debug(f"Khaya API translation succeeded ({len(result)} chars)")
|
| 175 |
+
return result or None
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 179 |
# FALLBACK β local NLLB model (aka_Latn β eng_Latn)
|
| 180 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 181 |
+
|
| 182 |
+
_local_translator = None
|
| 183 |
+
|
| 184 |
+
|
| 185 |
def _translate_via_local_model(text: str) -> str | None:
|
| 186 |
"""
|
| 187 |
Fallback: translate Twi/Akan β English using a local NLLB model.
|
|
|
|
| 189 |
|
| 190 |
Default model: facebook/nllb-200-distilled-600M
|
| 191 |
"""
|
| 192 |
+
global _local_translator
|
| 193 |
+
try:
|
| 194 |
+
if _local_translator is None:
|
| 195 |
+
from transformers import pipeline as hf_pipeline
|
| 196 |
+
from core.utils import get_device
|
| 197 |
+
device = 0 if get_device() == "cuda" else -1
|
| 198 |
_local_translator = hf_pipeline(
|
| 199 |
"translation",
|
| 200 |
model=NLLB_MODEL,
|
|
|
|
| 219 |
"cache or set NLLB_TRANSLATION_MODEL to a local model directory."
|
| 220 |
)
|
| 221 |
return None
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 225 |
+
# PUBLIC API
|
| 226 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 227 |
+
|
| 228 |
+
def translate_response_to_english(text: str, language: str) -> str:
|
| 229 |
+
"""
|
| 230 |
+
Translate a model's response to English before it reaches the safety
|
| 231 |
scorer (LlamaGuard3). This is the function scorer.py calls for detected
|
| 232 |
Twi responses before the LlamaGuard3 cross-check.
|
| 233 |
+
|
| 234 |
+
If language == "english", returns text unchanged (no-op, no API call).
|
| 235 |
+
|
| 236 |
Fallback chain: Khaya hosted API β local NLLB model β raise.
|
| 237 |
+
Results are cached per-process to avoid re-translating identical text.
|
| 238 |
+
|
| 239 |
+
Args:
|
| 240 |
+
text : the model's response in its original language
|
| 241 |
+
language : "english", "twi", or "ghanaian_en"
|
| 242 |
+
|
| 243 |
+
Returns:
|
| 244 |
English text, safe to pass to LlamaGuard3Scorer.score().
|
| 245 |
+
|
| 246 |
+
Raises:
|
| 247 |
+
RuntimeError if both Khaya and the local fallback fail β we must
|
| 248 |
+
NEVER silently score un-translated Twi text as if it were English.
|
| 249 |
+
|
| 250 |
+
Example:
|
| 251 |
+
response_en = translate_response_to_english(twi_response, "twi")
|
| 252 |
+
lg3_result = lg3_scorer.score(probe_id, model_id, "twi", probe_en, response_en)
|
| 253 |
+
"""
|
| 254 |
+
if language not in TRANSLATION_REQUIRED_LANGUAGES:
|
| 255 |
+
return text # english β no translation needed
|
| 256 |
+
|
| 257 |
+
cache_key = f"{language}:{text}"
|
| 258 |
+
if cache_key in _translation_cache:
|
| 259 |
+
return _translation_cache[cache_key]
|
| 260 |
+
|
| 261 |
+
source_lang = _KHAYA_LANG_CODES.get(language, "tw")
|
| 262 |
+
|
| 263 |
+
# 1) Try Khaya hosted API
|
| 264 |
+
translated = _translate_via_khaya_api(text, source_lang)
|
| 265 |
+
|
| 266 |
# 2) Fall back to local NLLB model
|
| 267 |
if translated is None:
|
| 268 |
logger.info("Falling back to local NLLB translation model")
|
| 269 |
translated = _translate_via_local_model(text)
|
| 270 |
+
|
| 271 |
+
# 3) Total failure β do not let un-translated Twi reach the scorers
|
| 272 |
+
if translated is None:
|
| 273 |
+
raise RuntimeError(
|
| 274 |
+
f"Translation failed for language='{language}' via both Khaya API "
|
| 275 |
+
f"and local fallback. Refusing to pass untranslated Twi text to "
|
| 276 |
f"LlamaGuard3 β it is not trained on Twi and would "
|
| 277 |
+
f"silently misclassify it. Text preview: {text[:80]}..."
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
_translation_cache[cache_key] = translated
|
| 281 |
+
return translated
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def clear_translation_cache() -> None:
|
| 285 |
+
"""Clear the in-memory translation cache. Useful between test runs."""
|
| 286 |
+
global _translation_cache
|
| 287 |
+
_translation_cache = {}
|
| 288 |
+
logger.debug("Translation cache cleared")
|