Spaces:
Runtime error
Runtime error
RhutuTuvoc commited on
Commit ·
673bc41
0
Parent(s):
Initial submission: IT Mental Health OpenEnv
Browse files- .dockerignore +17 -0
- .env.example +5 -0
- .gitignore +33 -0
- Dockerfile +29 -0
- README.md +484 -0
- app.py +175 -0
- inference.py +164 -0
- it_mental_health_environment.py +430 -0
- models.py +66 -0
- openenv.yaml +68 -0
- requirements.txt +6 -0
- requirements_inference.txt +2 -0
- validate.py +139 -0
.dockerignore
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
.gitignore
|
| 3 |
+
.env
|
| 4 |
+
.env.*
|
| 5 |
+
!.env.example
|
| 6 |
+
venv/
|
| 7 |
+
.venv/
|
| 8 |
+
__pycache__/
|
| 9 |
+
*.py[cod]
|
| 10 |
+
.pytest_cache/
|
| 11 |
+
.mypy_cache/
|
| 12 |
+
.ruff_cache/
|
| 13 |
+
build/
|
| 14 |
+
dist/
|
| 15 |
+
*.egg-info/
|
| 16 |
+
Thumbs.db
|
| 17 |
+
.DS_Store
|
.env.example
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
API_BASE_URL=https://api-inference.huggingface.co/v1
|
| 2 |
+
MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct
|
| 3 |
+
HF_TOKEN=
|
| 4 |
+
LOCAL_IMAGE_NAME=
|
| 5 |
+
ENV_BASE_URL=http://localhost:7860
|
.gitignore
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Env files
|
| 2 |
+
.env
|
| 3 |
+
.env.*
|
| 4 |
+
!.env.example
|
| 5 |
+
|
| 6 |
+
# Python cache/build
|
| 7 |
+
__pycache__/
|
| 8 |
+
*.py[cod]
|
| 9 |
+
*.pyo
|
| 10 |
+
*.pyd
|
| 11 |
+
.pytest_cache/
|
| 12 |
+
.mypy_cache/
|
| 13 |
+
.ruff_cache/
|
| 14 |
+
|
| 15 |
+
# Virtual envs
|
| 16 |
+
.venv/
|
| 17 |
+
venv/
|
| 18 |
+
env/
|
| 19 |
+
|
| 20 |
+
# Coverage/build artifacts
|
| 21 |
+
.coverage
|
| 22 |
+
htmlcov/
|
| 23 |
+
build/
|
| 24 |
+
dist/
|
| 25 |
+
*.egg-info/
|
| 26 |
+
|
| 27 |
+
# IDE/editor
|
| 28 |
+
.vscode/
|
| 29 |
+
.idea/
|
| 30 |
+
|
| 31 |
+
# OS files
|
| 32 |
+
.DS_Store
|
| 33 |
+
Thumbs.db
|
Dockerfile
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# IT Mental Health OpenEnv Dockerfile
|
| 2 |
+
# Compatible with Hugging Face Spaces (port 7860)
|
| 3 |
+
|
| 4 |
+
FROM python:3.11-slim
|
| 5 |
+
|
| 6 |
+
WORKDIR /app
|
| 7 |
+
|
| 8 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 9 |
+
curl \
|
| 10 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 11 |
+
|
| 12 |
+
COPY requirements.txt /tmp/requirements.txt
|
| 13 |
+
RUN pip install --no-cache-dir -r /tmp/requirements.txt
|
| 14 |
+
|
| 15 |
+
COPY app.py /app/app.py
|
| 16 |
+
COPY it_mental_health_environment.py /app/it_mental_health_environment.py
|
| 17 |
+
COPY models.py /app/models.py
|
| 18 |
+
COPY inference.py /app/inference.py
|
| 19 |
+
COPY validate.py /app/validate.py
|
| 20 |
+
COPY openenv.yaml /app/openenv.yaml
|
| 21 |
+
COPY README.md /app/README.md
|
| 22 |
+
COPY .env.example /app/.env.example
|
| 23 |
+
|
| 24 |
+
EXPOSE 7860
|
| 25 |
+
|
| 26 |
+
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
| 27 |
+
CMD curl -f http://localhost:7860/health || exit 1
|
| 28 |
+
|
| 29 |
+
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860", "--workers", "1"]
|
README.md
ADDED
|
@@ -0,0 +1,484 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# IT Mental Health OpenEnv
|
| 2 |
+
|
| 3 |
+
An OpenEnv-compatible reinforcement learning environment for mental health assessment and intervention planning in the IT and software engineering sector.
|
| 4 |
+
|
| 5 |
+
Built for the [Scaler x Meta-PyTorch OpenEnv Hackathon 2026](https://www.scaler.com/school-of-technology/meta-pytorch-hackathon).
|
| 6 |
+
|
| 7 |
+
## Elevator Pitch
|
| 8 |
+
|
| 9 |
+
Modern IT teams operate under sprint pressure, on-call fatigue, remote isolation, and always-on expectations. Those stressors often appear long before a formal HR escalation, but most benchmark environments do not test whether an AI system can recognize burnout patterns, prioritize mental health risk, or propose realistic workplace interventions.
|
| 10 |
+
|
| 11 |
+
This project solves that gap by turning workplace mental health into a structured evaluation environment. Instead of asking an agent to solve toy problems, it asks the agent to:
|
| 12 |
+
|
| 13 |
+
1. detect burnout from an employee profile,
|
| 14 |
+
2. triage multiple employees by urgency, and
|
| 15 |
+
3. design a structured intervention plan for a team already showing systemic strain.
|
| 16 |
+
|
| 17 |
+
The result is a benchmark that is easier to pitch to judges because it is:
|
| 18 |
+
|
| 19 |
+
- socially meaningful,
|
| 20 |
+
- clearly scoped,
|
| 21 |
+
- grounded in recognizable workplace scenarios,
|
| 22 |
+
- measurable through rubric-based rewards, and
|
| 23 |
+
- extensible to future HR and wellbeing tasks.
|
| 24 |
+
|
| 25 |
+
## What Problem It Solves
|
| 26 |
+
|
| 27 |
+
Most LLM evaluation environments focus on navigation, coding, or generic QA. They do not measure whether an agent can reason through workplace mental health cases in a structured, actionable way.
|
| 28 |
+
|
| 29 |
+
This environment is designed to evaluate:
|
| 30 |
+
|
| 31 |
+
- recognition of burnout symptoms using the Maslach Burnout Inventory (MBI) framing,
|
| 32 |
+
- prioritization of urgent cases using tiered stress triage,
|
| 33 |
+
- generation of practical intervention plans rather than vague advice,
|
| 34 |
+
- structured communication quality,
|
| 35 |
+
- rewardable outputs for iterative benchmarking.
|
| 36 |
+
|
| 37 |
+
For a hackathon pitch, the core story is:
|
| 38 |
+
|
| 39 |
+
- burnout in IT is widespread,
|
| 40 |
+
- companies need earlier detection and more structured support,
|
| 41 |
+
- current AI benchmarks ignore this domain,
|
| 42 |
+
- this environment turns that real-world problem into a reproducible benchmark.
|
| 43 |
+
|
| 44 |
+
## Why This Is Useful in a Hackathon
|
| 45 |
+
|
| 46 |
+
This project works well for a benchmark-oriented hackathon because it gives judges three things at once:
|
| 47 |
+
|
| 48 |
+
- a novel domain,
|
| 49 |
+
- a technically runnable benchmark,
|
| 50 |
+
- a strong real-world narrative.
|
| 51 |
+
|
| 52 |
+
It is also practical for submission because:
|
| 53 |
+
|
| 54 |
+
- the API is simple and inspectable,
|
| 55 |
+
- the inference flow is reproducible,
|
| 56 |
+
- Docker packaging is supported,
|
| 57 |
+
- validation is built in,
|
| 58 |
+
- the benchmark returns normalized scores in `[0, 1]`.
|
| 59 |
+
|
| 60 |
+
## Functional Overview
|
| 61 |
+
|
| 62 |
+
The environment simulates three progressive tasks:
|
| 63 |
+
|
| 64 |
+
| Task ID | Difficulty | Goal |
|
| 65 |
+
|---|---|---|
|
| 66 |
+
| `burnout_detection` | Easy | Detect MBI dimensions, severity, red flags, and HR escalation need |
|
| 67 |
+
| `stress_triage` | Medium | Classify multiple employees by urgency and recommend immediate plus medium-term actions |
|
| 68 |
+
| `intervention_plan` | Hard | Build a 4-week recovery/intervention plan for a team in distress |
|
| 69 |
+
|
| 70 |
+
Each episode moves through those tasks in order. The agent receives a scenario, returns a text response, and gets:
|
| 71 |
+
|
| 72 |
+
- a reward,
|
| 73 |
+
- grader feedback,
|
| 74 |
+
- a score breakdown,
|
| 75 |
+
- and the next scenario until the episode ends.
|
| 76 |
+
|
| 77 |
+
## Benchmark Design
|
| 78 |
+
|
| 79 |
+
The benchmark is intentionally structured around realistic workplace mental health reasoning rather than open-ended chat.
|
| 80 |
+
|
| 81 |
+
### Task 1: Burnout Detection
|
| 82 |
+
|
| 83 |
+
The agent reviews a single employee profile and is expected to identify:
|
| 84 |
+
|
| 85 |
+
- MBI dimensions present,
|
| 86 |
+
- severity level,
|
| 87 |
+
- top red flags,
|
| 88 |
+
- whether immediate HR escalation is needed.
|
| 89 |
+
|
| 90 |
+
### Task 2: Stress Triage
|
| 91 |
+
|
| 92 |
+
The agent sees multiple employees and must:
|
| 93 |
+
|
| 94 |
+
- assign each a stress tier,
|
| 95 |
+
- identify likely primary stressor type,
|
| 96 |
+
- recommend one immediate action,
|
| 97 |
+
- recommend one medium-term support action,
|
| 98 |
+
- rank intervention priority.
|
| 99 |
+
|
| 100 |
+
### Task 3: Intervention Plan
|
| 101 |
+
|
| 102 |
+
The agent receives a team-wide burnout scenario and must produce:
|
| 103 |
+
|
| 104 |
+
- a 4-week phased plan,
|
| 105 |
+
- clear responsibilities,
|
| 106 |
+
- measurable outcomes,
|
| 107 |
+
- KPIs,
|
| 108 |
+
- a risk statement,
|
| 109 |
+
- a budget band.
|
| 110 |
+
|
| 111 |
+
## Architecture Diagram
|
| 112 |
+
|
| 113 |
+
```mermaid
|
| 114 |
+
flowchart TD
|
| 115 |
+
A[Client / Inference Runner] -->|POST /reset seed| B[FastAPI app.py]
|
| 116 |
+
B --> C[IT Mental Health Environment]
|
| 117 |
+
C --> D[Scenario Generator]
|
| 118 |
+
D -->|Task 1: Burnout Detection| E[Employee Profile]
|
| 119 |
+
D -->|Task 2: Stress Triage| F[Multi-Employee Cases]
|
| 120 |
+
D -->|Task 3: Intervention Plan| G[Team Scenario]
|
| 121 |
+
E --> H[Observation: scenario + task_id]
|
| 122 |
+
F --> H
|
| 123 |
+
G --> H
|
| 124 |
+
H --> A
|
| 125 |
+
A -->|POST /step response| B
|
| 126 |
+
B --> I{Grader}
|
| 127 |
+
I -->|Configured| J[LLM Judge]
|
| 128 |
+
I -->|Fallback| K[Heuristic Grader]
|
| 129 |
+
J --> L[Reward + Score Breakdown]
|
| 130 |
+
K --> L
|
| 131 |
+
L --> M[State Update<br/>cumulative_reward, tasks_completed]
|
| 132 |
+
M -->|done=false| D
|
| 133 |
+
M -->|done=true| N[Final Normalized Score 0-1]
|
| 134 |
+
N --> A
|
| 135 |
+
```
|
| 136 |
+
|
| 137 |
+
## How the Environment Works
|
| 138 |
+
|
| 139 |
+
At a high level:
|
| 140 |
+
|
| 141 |
+
1. `POST /reset` starts a new episode and returns the first scenario.
|
| 142 |
+
2. `POST /step` accepts the agent's answer and advances the benchmark.
|
| 143 |
+
3. The environment tracks reward, progress, and task order.
|
| 144 |
+
4. `GET /state` exposes current episode metadata.
|
| 145 |
+
5. `GET /tasks` and `GET /schema` help inspection and integration.
|
| 146 |
+
|
| 147 |
+
The environment uses randomized scenario generation so episodes are not identical across resets unless a seed is supplied.
|
| 148 |
+
|
| 149 |
+
## Reward and Scoring
|
| 150 |
+
|
| 151 |
+
The environment uses rubric-based grading. Depending on runtime configuration, it can use:
|
| 152 |
+
|
| 153 |
+
- an LLM judge, or
|
| 154 |
+
- a heuristic fallback grader.
|
| 155 |
+
|
| 156 |
+
The fallback heuristic is currently important because it allows the benchmark to run even without a configured external judge model.
|
| 157 |
+
|
| 158 |
+
Reward properties:
|
| 159 |
+
|
| 160 |
+
- each task returns a normalized reward in `[0.0, 1.0]`,
|
| 161 |
+
- the score breakdown reports sub-dimension performance,
|
| 162 |
+
- the inference script outputs step rewards and a final normalized score.
|
| 163 |
+
|
| 164 |
+
### Rubric Themes
|
| 165 |
+
|
| 166 |
+
The grader looks for factors such as:
|
| 167 |
+
|
| 168 |
+
- clinical accuracy,
|
| 169 |
+
- completeness,
|
| 170 |
+
- relevance to the scenario,
|
| 171 |
+
- prioritization quality,
|
| 172 |
+
- structure and clarity,
|
| 173 |
+
- intervention practicality.
|
| 174 |
+
|
| 175 |
+
## Technical Architecture
|
| 176 |
+
|
| 177 |
+
This repository uses a flat root layout:
|
| 178 |
+
|
| 179 |
+
```text
|
| 180 |
+
it_mental_health_env/
|
| 181 |
+
|-- app.py
|
| 182 |
+
|-- it_mental_health_environment.py
|
| 183 |
+
|-- inference.py
|
| 184 |
+
|-- validate.py
|
| 185 |
+
|-- models.py
|
| 186 |
+
|-- openenv.yaml
|
| 187 |
+
|-- requirements.txt
|
| 188 |
+
|-- requirements_inference.txt
|
| 189 |
+
|-- .env.example
|
| 190 |
+
|-- .gitignore
|
| 191 |
+
|-- Dockerfile
|
| 192 |
+
`-- README.md
|
| 193 |
+
```
|
| 194 |
+
|
| 195 |
+
### File Responsibilities
|
| 196 |
+
|
| 197 |
+
- `app.py`
|
| 198 |
+
FastAPI application exposing the benchmark endpoints.
|
| 199 |
+
- `it_mental_health_environment.py`
|
| 200 |
+
Core environment logic, scenario generation, rubrics, grader flow, and state transitions.
|
| 201 |
+
- `models.py`
|
| 202 |
+
Typed action, observation, and state models.
|
| 203 |
+
- `inference.py`
|
| 204 |
+
Benchmark runner using the OpenAI client with the required env-variable contract.
|
| 205 |
+
- `validate.py`
|
| 206 |
+
Pre-submission sanity checker for API health and scoring behavior.
|
| 207 |
+
- `openenv.yaml`
|
| 208 |
+
OpenEnv manifest for the environment.
|
| 209 |
+
- `Dockerfile`
|
| 210 |
+
Container packaging for local and HF Spaces style deployment.
|
| 211 |
+
|
| 212 |
+
## API Reference
|
| 213 |
+
|
| 214 |
+
### `GET /health`
|
| 215 |
+
|
| 216 |
+
Simple readiness check.
|
| 217 |
+
|
| 218 |
+
Example response:
|
| 219 |
+
|
| 220 |
+
```json
|
| 221 |
+
{
|
| 222 |
+
"status": "ok",
|
| 223 |
+
"env": "it_mental_health_env",
|
| 224 |
+
"version": "1.0.0"
|
| 225 |
+
}
|
| 226 |
+
```
|
| 227 |
+
|
| 228 |
+
### `POST /reset`
|
| 229 |
+
|
| 230 |
+
Starts a new episode.
|
| 231 |
+
|
| 232 |
+
Request body:
|
| 233 |
+
|
| 234 |
+
```json
|
| 235 |
+
{
|
| 236 |
+
"seed": 123
|
| 237 |
+
}
|
| 238 |
+
```
|
| 239 |
+
|
| 240 |
+
Notes:
|
| 241 |
+
|
| 242 |
+
- `seed` is optional.
|
| 243 |
+
- when `seed` is provided, the same episode scenario sequence can be reproduced.
|
| 244 |
+
|
| 245 |
+
### `POST /step`
|
| 246 |
+
|
| 247 |
+
Submits the agent's answer for the current task.
|
| 248 |
+
|
| 249 |
+
Request body:
|
| 250 |
+
|
| 251 |
+
```json
|
| 252 |
+
{
|
| 253 |
+
"response": "Your structured answer here",
|
| 254 |
+
"task_id": "burnout_detection",
|
| 255 |
+
"confidence": 0.9,
|
| 256 |
+
"metadata": {}
|
| 257 |
+
}
|
| 258 |
+
```
|
| 259 |
+
|
| 260 |
+
Notes:
|
| 261 |
+
|
| 262 |
+
- `metadata` is optional and can safely be `{}`.
|
| 263 |
+
- `task_id` should usually match the `task_id` returned by the latest observation.
|
| 264 |
+
- the environment itself advances task order internally.
|
| 265 |
+
|
| 266 |
+
### `GET /state`
|
| 267 |
+
|
| 268 |
+
Returns current episode progress.
|
| 269 |
+
|
| 270 |
+
### `GET /tasks`
|
| 271 |
+
|
| 272 |
+
Returns benchmark task metadata such as difficulty and description.
|
| 273 |
+
|
| 274 |
+
### `GET /schema`
|
| 275 |
+
|
| 276 |
+
Returns a human-readable action and observation schema.
|
| 277 |
+
|
| 278 |
+
## Request and Response Shapes
|
| 279 |
+
|
| 280 |
+
### Action
|
| 281 |
+
|
| 282 |
+
```json
|
| 283 |
+
{
|
| 284 |
+
"response": "string",
|
| 285 |
+
"task_id": "burnout_detection | stress_triage | intervention_plan",
|
| 286 |
+
"confidence": 0.9,
|
| 287 |
+
"metadata": {}
|
| 288 |
+
}
|
| 289 |
+
```
|
| 290 |
+
|
| 291 |
+
### Observation
|
| 292 |
+
|
| 293 |
+
```json
|
| 294 |
+
{
|
| 295 |
+
"scenario": "string",
|
| 296 |
+
"feedback": "string",
|
| 297 |
+
"reward": 0.655,
|
| 298 |
+
"done": false,
|
| 299 |
+
"score_breakdown": {
|
| 300 |
+
"example_dimension": 0.8
|
| 301 |
+
},
|
| 302 |
+
"task_id": "stress_triage",
|
| 303 |
+
"metadata": {}
|
| 304 |
+
}
|
| 305 |
+
```
|
| 306 |
+
|
| 307 |
+
### State
|
| 308 |
+
|
| 309 |
+
```json
|
| 310 |
+
{
|
| 311 |
+
"episode_id": "uuid",
|
| 312 |
+
"step_count": 1,
|
| 313 |
+
"current_task": "stress_triage",
|
| 314 |
+
"cumulative_reward": 0.655,
|
| 315 |
+
"tasks_completed": ["burnout_detection"]
|
| 316 |
+
}
|
| 317 |
+
```
|
| 318 |
+
|
| 319 |
+
## Running Locally
|
| 320 |
+
|
| 321 |
+
### 1. Install dependencies
|
| 322 |
+
|
| 323 |
+
```bash
|
| 324 |
+
pip install -r requirements.txt
|
| 325 |
+
pip install -r requirements_inference.txt
|
| 326 |
+
```
|
| 327 |
+
|
| 328 |
+
### 2. Start the API server
|
| 329 |
+
|
| 330 |
+
```bash
|
| 331 |
+
python -m uvicorn app:app --host 127.0.0.1 --port 7860
|
| 332 |
+
```
|
| 333 |
+
|
| 334 |
+
### 3. Explore the interactive docs
|
| 335 |
+
|
| 336 |
+
Open:
|
| 337 |
+
|
| 338 |
+
- `http://127.0.0.1:7860/docs`
|
| 339 |
+
- `http://127.0.0.1:7860/health`
|
| 340 |
+
- `http://127.0.0.1:7860/tasks`
|
| 341 |
+
|
| 342 |
+
### 4. Configure environment variables
|
| 343 |
+
|
| 344 |
+
Start from `.env.example`:
|
| 345 |
+
|
| 346 |
+
```env
|
| 347 |
+
API_BASE_URL=https://api-inference.huggingface.co/v1
|
| 348 |
+
MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct
|
| 349 |
+
HF_TOKEN=
|
| 350 |
+
LOCAL_IMAGE_NAME=
|
| 351 |
+
ENV_BASE_URL=http://localhost:7860
|
| 352 |
+
```
|
| 353 |
+
|
| 354 |
+
Important notes:
|
| 355 |
+
|
| 356 |
+
- `HF_TOKEN` is required for the inference script.
|
| 357 |
+
- `LOCAL_IMAGE_NAME` is optional and can stay empty in this repo's current HTTP-server workflow.
|
| 358 |
+
- `ENV_BASE_URL` should point to your local FastAPI server.
|
| 359 |
+
|
| 360 |
+
### 5. Run the inference script
|
| 361 |
+
|
| 362 |
+
```bash
|
| 363 |
+
python inference.py
|
| 364 |
+
```
|
| 365 |
+
|
| 366 |
+
## Inference Script Contract
|
| 367 |
+
|
| 368 |
+
The hackathon requires the inference runner to follow a specific pattern. This repository's `inference.py` is aligned to that requirement:
|
| 369 |
+
|
| 370 |
+
- it uses the OpenAI client for all LLM calls,
|
| 371 |
+
- it reads `API_BASE_URL`, `MODEL_NAME`, and `HF_TOKEN`,
|
| 372 |
+
- it includes optional `LOCAL_IMAGE_NAME`,
|
| 373 |
+
- it emits plain-text structured logs to stdout.
|
| 374 |
+
|
| 375 |
+
### Required stdout format
|
| 376 |
+
|
| 377 |
+
```text
|
| 378 |
+
[START] task=burnout_detection env=it_mental_health_env model=meta-llama/Llama-3.1-8B-Instruct
|
| 379 |
+
[STEP] step=1 action=... reward=0.72 done=false error=null
|
| 380 |
+
[STEP] step=2 action=... reward=0.68 done=false error=null
|
| 381 |
+
[END] success=true steps=2 score=0.467 rewards=0.72,0.68
|
| 382 |
+
```
|
| 383 |
+
|
| 384 |
+
Rules followed:
|
| 385 |
+
|
| 386 |
+
- exactly one `[START]` line,
|
| 387 |
+
- one `[STEP]` line per environment step,
|
| 388 |
+
- one `[END]` line even if execution fails,
|
| 389 |
+
- rewards formatted to two decimals in step logs,
|
| 390 |
+
- final score normalized to `[0, 1]`.
|
| 391 |
+
|
| 392 |
+
## Manual API Testing
|
| 393 |
+
|
| 394 |
+
If you want to test without the inference script:
|
| 395 |
+
|
| 396 |
+
1. call `POST /reset`,
|
| 397 |
+
2. copy the returned `scenario` and `task_id`,
|
| 398 |
+
3. write a structured response,
|
| 399 |
+
4. send it to `POST /step`,
|
| 400 |
+
5. inspect `reward`, `feedback`, and `score_breakdown`.
|
| 401 |
+
|
| 402 |
+
Example `POST /step` payload:
|
| 403 |
+
|
| 404 |
+
```json
|
| 405 |
+
{
|
| 406 |
+
"response": "1. MBI Dimensions\nExhaustion and Depersonalization are present.\n\n2. Severity\nHigh.\n\n3. Top 3 Red Flags\n- Long working hours\n- Extended period without vacation\n- Emotional detachment\n\n4. HR Escalation\nYes. The presentation suggests serious burnout risk and warrants prompt support.",
|
| 407 |
+
"task_id": "burnout_detection",
|
| 408 |
+
"confidence": 0.9,
|
| 409 |
+
"metadata": {}
|
| 410 |
+
}
|
| 411 |
+
```
|
| 412 |
+
|
| 413 |
+
## Validation
|
| 414 |
+
|
| 415 |
+
Run the pre-submission validator after the server is up:
|
| 416 |
+
|
| 417 |
+
```bash
|
| 418 |
+
python validate.py
|
| 419 |
+
```
|
| 420 |
+
|
| 421 |
+
What it checks:
|
| 422 |
+
|
| 423 |
+
- manifest validity,
|
| 424 |
+
- root `inference.py` presence,
|
| 425 |
+
- `/health`,
|
| 426 |
+
- `/reset`,
|
| 427 |
+
- `/step`,
|
| 428 |
+
- `/state`,
|
| 429 |
+
- reward range compliance.
|
| 430 |
+
|
| 431 |
+
## Docker Usage
|
| 432 |
+
|
| 433 |
+
Build:
|
| 434 |
+
|
| 435 |
+
```bash
|
| 436 |
+
docker build -t it-mental-health-env .
|
| 437 |
+
```
|
| 438 |
+
|
| 439 |
+
Run:
|
| 440 |
+
|
| 441 |
+
```bash
|
| 442 |
+
docker run -p 7860:7860 it-mental-health-env
|
| 443 |
+
```
|
| 444 |
+
|
| 445 |
+
Then visit:
|
| 446 |
+
|
| 447 |
+
- `http://127.0.0.1:7860/health`
|
| 448 |
+
- `http://127.0.0.1:7860/docs`
|
| 449 |
+
|
| 450 |
+
## Pitching Notes for Judges
|
| 451 |
+
|
| 452 |
+
If you are demoing this live, emphasize:
|
| 453 |
+
|
| 454 |
+
- **Domain relevance**
|
| 455 |
+
This benchmark addresses a real workplace problem with growing urgency.
|
| 456 |
+
- **Novel evaluation target**
|
| 457 |
+
Most environments do not evaluate burnout detection, triage, or intervention planning.
|
| 458 |
+
- **Structured benchmark**
|
| 459 |
+
The project is not just a chatbot; it is a measurable RL-style environment with rewards and step transitions.
|
| 460 |
+
- **Technical completeness**
|
| 461 |
+
It includes API endpoints, validation, an inference runner, Docker packaging, and an OpenEnv manifest.
|
| 462 |
+
- **Extensibility**
|
| 463 |
+
Future tasks could include return-to-work support, manager coaching, policy audits, absenteeism risk, or escalation workflows.
|
| 464 |
+
|
| 465 |
+
## Current Limitations
|
| 466 |
+
|
| 467 |
+
It is useful to be transparent about the current scope:
|
| 468 |
+
|
| 469 |
+
- the benchmark is primarily for structured evaluation, not clinical diagnosis,
|
| 470 |
+
- heuristic fallback grading can be blunt compared with a stronger external judge model,
|
| 471 |
+
- the current flow is benchmark-first rather than custom-case analysis first,
|
| 472 |
+
- duplicate names can occasionally appear in generated scenarios and affect grading clarity.
|
| 473 |
+
|
| 474 |
+
## Suggested Future Improvements
|
| 475 |
+
|
| 476 |
+
- add a custom scenario endpoint for real-world demos,
|
| 477 |
+
- improve scenario diversity and de-duplicate names within multi-person cases,
|
| 478 |
+
- strengthen grading with a more robust external LLM judge,
|
| 479 |
+
- add analytics dashboards for reward trends and failure modes,
|
| 480 |
+
- expand task coverage into HR escalation and manager support planning.
|
| 481 |
+
|
| 482 |
+
## License
|
| 483 |
+
|
| 484 |
+
MIT
|
app.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
IT Mental Health OpenEnv - FastAPI Server
|
| 3 |
+
Exposes /reset, /step, /state, /health, /tasks endpoints.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from typing import Any, Dict, Optional
|
| 7 |
+
|
| 8 |
+
import uvicorn
|
| 9 |
+
from fastapi import FastAPI
|
| 10 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 11 |
+
from pydantic import BaseModel
|
| 12 |
+
|
| 13 |
+
from it_mental_health_environment import ITMentalHealthEnvironment, TASK_ORDER
|
| 14 |
+
from models import MentalHealthAction
|
| 15 |
+
|
| 16 |
+
app = FastAPI(
|
| 17 |
+
title="IT Mental Health OpenEnv",
|
| 18 |
+
description="RL environment for mental health assessment in the IT sector.",
|
| 19 |
+
version="1.0.0",
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
app.add_middleware(
|
| 23 |
+
CORSMiddleware,
|
| 24 |
+
allow_origins=["*"],
|
| 25 |
+
allow_methods=["*"],
|
| 26 |
+
allow_headers=["*"],
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
# Global environment instance (single session for Spaces)
|
| 30 |
+
env = ITMentalHealthEnvironment()
|
| 31 |
+
|
| 32 |
+
TASK_METADATA = {
|
| 33 |
+
"burnout_detection": {
|
| 34 |
+
"difficulty": "easy",
|
| 35 |
+
"description": "Identify Maslach burnout dimensions, severity, red flags, and whether HR escalation is needed.",
|
| 36 |
+
},
|
| 37 |
+
"stress_triage": {
|
| 38 |
+
"difficulty": "medium",
|
| 39 |
+
"description": "Triage three IT employees by stress tier and recommend immediate and medium-term support.",
|
| 40 |
+
},
|
| 41 |
+
"intervention_plan": {
|
| 42 |
+
"difficulty": "hard",
|
| 43 |
+
"description": "Design a four-week intervention plan for a software team facing systemic burnout.",
|
| 44 |
+
},
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
class ResetRequest(BaseModel):
|
| 49 |
+
seed: Optional[int] = None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class StepRequest(BaseModel):
|
| 53 |
+
response: str
|
| 54 |
+
task_id: Optional[str] = "burnout_detection"
|
| 55 |
+
confidence: Optional[float] = 1.0
|
| 56 |
+
metadata: Optional[Dict[str, Any]] = {}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class ObservationResponse(BaseModel):
|
| 60 |
+
scenario: str
|
| 61 |
+
feedback: str
|
| 62 |
+
reward: float
|
| 63 |
+
done: bool
|
| 64 |
+
score_breakdown: Dict[str, float] = {}
|
| 65 |
+
task_id: str
|
| 66 |
+
metadata: Dict[str, Any] = {}
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
class StateResponse(BaseModel):
|
| 70 |
+
episode_id: Optional[str]
|
| 71 |
+
step_count: int
|
| 72 |
+
current_task: str
|
| 73 |
+
cumulative_reward: float
|
| 74 |
+
tasks_completed: list
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
@app.get("/")
|
| 78 |
+
def root():
|
| 79 |
+
return {
|
| 80 |
+
"name": "IT Mental Health OpenEnv",
|
| 81 |
+
"version": "1.0.0",
|
| 82 |
+
"docs": "/docs",
|
| 83 |
+
"health": "/health",
|
| 84 |
+
"tasks": "/tasks",
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@app.get("/health")
|
| 89 |
+
def health():
|
| 90 |
+
return {"status": "ok", "env": "it_mental_health_env", "version": "1.0.0"}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
@app.post("/reset", response_model=ObservationResponse)
|
| 94 |
+
def reset(req: ResetRequest = ResetRequest()):
|
| 95 |
+
obs = env.reset(seed=req.seed)
|
| 96 |
+
return ObservationResponse(
|
| 97 |
+
scenario=obs.scenario,
|
| 98 |
+
feedback=obs.feedback,
|
| 99 |
+
reward=obs.reward,
|
| 100 |
+
done=obs.done,
|
| 101 |
+
score_breakdown=obs.score_breakdown,
|
| 102 |
+
task_id=obs.task_id,
|
| 103 |
+
metadata=obs.metadata,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
@app.post("/step", response_model=ObservationResponse)
|
| 108 |
+
def step(req: StepRequest):
|
| 109 |
+
action = MentalHealthAction(
|
| 110 |
+
response=req.response,
|
| 111 |
+
task_id=req.task_id or "burnout_detection",
|
| 112 |
+
confidence=req.confidence or 1.0,
|
| 113 |
+
metadata=req.metadata or {},
|
| 114 |
+
)
|
| 115 |
+
obs = env.step(action)
|
| 116 |
+
return ObservationResponse(
|
| 117 |
+
scenario=obs.scenario,
|
| 118 |
+
feedback=obs.feedback,
|
| 119 |
+
reward=obs.reward,
|
| 120 |
+
done=obs.done,
|
| 121 |
+
score_breakdown=obs.score_breakdown,
|
| 122 |
+
task_id=obs.task_id,
|
| 123 |
+
metadata=obs.metadata,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
@app.get("/state", response_model=StateResponse)
|
| 128 |
+
def state():
|
| 129 |
+
s = env.state
|
| 130 |
+
return StateResponse(
|
| 131 |
+
episode_id=s.episode_id,
|
| 132 |
+
step_count=s.step_count,
|
| 133 |
+
current_task=s.current_task,
|
| 134 |
+
cumulative_reward=s.cumulative_reward,
|
| 135 |
+
tasks_completed=s.tasks_completed,
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
@app.get("/tasks")
|
| 140 |
+
def list_tasks():
|
| 141 |
+
"""List all tasks with metadata used by grader validation."""
|
| 142 |
+
return {
|
| 143 |
+
"tasks": [
|
| 144 |
+
{
|
| 145 |
+
"task_id": tid,
|
| 146 |
+
"difficulty": TASK_METADATA[tid]["difficulty"],
|
| 147 |
+
"description": TASK_METADATA[tid]["description"],
|
| 148 |
+
}
|
| 149 |
+
for tid in TASK_ORDER
|
| 150 |
+
]
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
@app.get("/schema")
|
| 155 |
+
def schema():
|
| 156 |
+
return {
|
| 157 |
+
"action": {
|
| 158 |
+
"response": "str - agent's textual answer",
|
| 159 |
+
"task_id": "str - one of: burnout_detection, stress_triage, intervention_plan",
|
| 160 |
+
"confidence": "float 0.0-1.0",
|
| 161 |
+
"metadata": "dict - optional",
|
| 162 |
+
},
|
| 163 |
+
"observation": {
|
| 164 |
+
"scenario": "str - current scenario text",
|
| 165 |
+
"feedback": "str - grader feedback",
|
| 166 |
+
"reward": "float 0.0-1.0",
|
| 167 |
+
"done": "bool",
|
| 168 |
+
"score_breakdown": "dict - partial scores per rubric dimension",
|
| 169 |
+
"task_id": "str",
|
| 170 |
+
},
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
if __name__ == "__main__":
|
| 175 |
+
uvicorn.run("app:app", host="0.0.0.0", port=7860, reload=False)
|
inference.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Inference script for IT Mental Health OpenEnv.
|
| 3 |
+
|
| 4 |
+
Required environment variables in the inference configuration:
|
| 5 |
+
API_BASE_URL The API endpoint for the LLM.
|
| 6 |
+
MODEL_NAME The model identifier to use for inference.
|
| 7 |
+
HF_TOKEN Your Hugging Face / API key.
|
| 8 |
+
|
| 9 |
+
Optional:
|
| 10 |
+
LOCAL_IMAGE_NAME Present for compatibility with environments created from
|
| 11 |
+
docker images.
|
| 12 |
+
|
| 13 |
+
Defaults are set only for API_BASE_URL and MODEL_NAME.
|
| 14 |
+
The script uses the OpenAI client for all LLM calls.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
from typing import List, Optional
|
| 20 |
+
|
| 21 |
+
import requests
|
| 22 |
+
from openai import OpenAI
|
| 23 |
+
|
| 24 |
+
API_BASE_URL = os.getenv("API_BASE_URL", "https://api-inference.huggingface.co/v1")
|
| 25 |
+
MODEL_NAME = os.getenv("MODEL_NAME", "meta-llama/Llama-3.1-8B-Instruct")
|
| 26 |
+
HF_TOKEN = os.getenv("HF_TOKEN")
|
| 27 |
+
LOCAL_IMAGE_NAME = os.getenv("LOCAL_IMAGE_NAME")
|
| 28 |
+
ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:7860")
|
| 29 |
+
|
| 30 |
+
BENCHMARK = "it_mental_health_env"
|
| 31 |
+
MAX_STEPS = 3
|
| 32 |
+
TEMPERATURE = 0.3
|
| 33 |
+
MAX_TOKENS = 800
|
| 34 |
+
SUCCESS_SCORE_THRESHOLD = 0.6
|
| 35 |
+
|
| 36 |
+
SYSTEM_PROMPT = """You are an expert occupational psychologist and workplace mental health consultant
|
| 37 |
+
specialising in the IT/software engineering sector. You have deep knowledge of:
|
| 38 |
+
- Maslach Burnout Inventory (MBI) dimensions
|
| 39 |
+
- Stress triage frameworks (GREEN/AMBER/RED/CRITICAL)
|
| 40 |
+
- Evidence-based workplace mental health interventions
|
| 41 |
+
- Employee Assistance Programmes (EAP) and HR escalation protocols
|
| 42 |
+
|
| 43 |
+
Respond in clear, structured format with headings and concrete recommendations.
|
| 44 |
+
Be specific, actionable, and professional. Aim for 200-600 words per response."""
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def clean_field(value: Optional[str]) -> str:
|
| 48 |
+
if value is None:
|
| 49 |
+
return "null"
|
| 50 |
+
return " ".join(str(value).split())
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def log_start(task: str, env: str, model: str) -> None:
|
| 54 |
+
print(f"[START] task={clean_field(task)} env={clean_field(env)} model={clean_field(model)}", flush=True)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]) -> None:
|
| 58 |
+
print(
|
| 59 |
+
f"[STEP] step={step} action={clean_field(action)} reward={reward:.2f} "
|
| 60 |
+
f"done={str(done).lower()} error={clean_field(error)}",
|
| 61 |
+
flush=True,
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:
|
| 66 |
+
rewards_str = ",".join(f"{reward:.2f}" for reward in rewards)
|
| 67 |
+
print(
|
| 68 |
+
f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}",
|
| 69 |
+
flush=True,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def call_env(endpoint: str, payload: Optional[dict] = None, method: str = "POST") -> dict:
|
| 74 |
+
url = f"{ENV_BASE_URL}/{endpoint}"
|
| 75 |
+
if method == "GET":
|
| 76 |
+
response = requests.get(url, timeout=30)
|
| 77 |
+
else:
|
| 78 |
+
response = requests.post(url, json=payload or {}, timeout=30)
|
| 79 |
+
response.raise_for_status()
|
| 80 |
+
return response.json()
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def get_model_response(client: OpenAI, scenario: str) -> str:
|
| 84 |
+
completion = client.chat.completions.create(
|
| 85 |
+
model=MODEL_NAME,
|
| 86 |
+
messages=[
|
| 87 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 88 |
+
{"role": "user", "content": scenario},
|
| 89 |
+
],
|
| 90 |
+
temperature=TEMPERATURE,
|
| 91 |
+
max_tokens=MAX_TOKENS,
|
| 92 |
+
stream=False,
|
| 93 |
+
)
|
| 94 |
+
content = completion.choices[0].message.content or ""
|
| 95 |
+
return content.strip() or "Provide a structured assessment and immediate next steps."
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def main() -> None:
|
| 99 |
+
rewards: List[float] = []
|
| 100 |
+
steps_taken = 0
|
| 101 |
+
score = 0.0
|
| 102 |
+
success = False
|
| 103 |
+
current_task = "unknown"
|
| 104 |
+
start_logged = False
|
| 105 |
+
|
| 106 |
+
try:
|
| 107 |
+
if not HF_TOKEN:
|
| 108 |
+
raise RuntimeError("HF_TOKEN must be set for inference.")
|
| 109 |
+
|
| 110 |
+
client = OpenAI(base_url=API_BASE_URL, api_key=HF_TOKEN)
|
| 111 |
+
observation = call_env("reset", {})
|
| 112 |
+
current_task = observation.get("task_id", "unknown")
|
| 113 |
+
log_start(task=current_task, env=BENCHMARK, model=MODEL_NAME)
|
| 114 |
+
start_logged = True
|
| 115 |
+
|
| 116 |
+
for step in range(1, MAX_STEPS + 1):
|
| 117 |
+
if observation.get("done", False):
|
| 118 |
+
break
|
| 119 |
+
|
| 120 |
+
current_task = observation.get("task_id", "unknown")
|
| 121 |
+
action_text = get_model_response(client, observation["scenario"])
|
| 122 |
+
|
| 123 |
+
error = None
|
| 124 |
+
done = False
|
| 125 |
+
reward = 0.0
|
| 126 |
+
|
| 127 |
+
try:
|
| 128 |
+
observation = call_env(
|
| 129 |
+
"step",
|
| 130 |
+
{
|
| 131 |
+
"response": action_text,
|
| 132 |
+
"task_id": current_task,
|
| 133 |
+
"confidence": 0.85,
|
| 134 |
+
"metadata": {},
|
| 135 |
+
},
|
| 136 |
+
)
|
| 137 |
+
reward = float(observation.get("reward", 0.0) or 0.0)
|
| 138 |
+
done = bool(observation.get("done", False))
|
| 139 |
+
except Exception as exc:
|
| 140 |
+
error = str(exc)
|
| 141 |
+
done = True
|
| 142 |
+
|
| 143 |
+
rewards.append(reward)
|
| 144 |
+
steps_taken = step
|
| 145 |
+
log_step(step=step, action=action_text, reward=reward, done=done, error=error)
|
| 146 |
+
|
| 147 |
+
if error or done:
|
| 148 |
+
break
|
| 149 |
+
|
| 150 |
+
score = sum(rewards) / MAX_STEPS if MAX_STEPS > 0 else 0.0
|
| 151 |
+
score = max(0.0, min(1.0, score))
|
| 152 |
+
success = score >= SUCCESS_SCORE_THRESHOLD
|
| 153 |
+
finally:
|
| 154 |
+
if not start_logged:
|
| 155 |
+
log_start(task=current_task, env=BENCHMARK, model=MODEL_NAME)
|
| 156 |
+
log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
if __name__ == "__main__":
|
| 160 |
+
try:
|
| 161 |
+
main()
|
| 162 |
+
except Exception as exc:
|
| 163 |
+
print(f"Inference failed: {exc}", file=sys.stderr)
|
| 164 |
+
sys.exit(1)
|
it_mental_health_environment.py
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
IT Mental Health OpenEnv - Environment Logic (v2 - improved)
|
| 3 |
+
|
| 4 |
+
FIX 1: LLM-as-judge grader replaces keyword matching.
|
| 5 |
+
FIX 2: Randomised episode generation — every reset() is unique.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import os
|
| 9 |
+
import uuid
|
| 10 |
+
import random
|
| 11 |
+
import json
|
| 12 |
+
from typing import Optional
|
| 13 |
+
from openai import OpenAI
|
| 14 |
+
|
| 15 |
+
try:
|
| 16 |
+
from models import MentalHealthAction, MentalHealthObservation, MentalHealthState
|
| 17 |
+
except ImportError:
|
| 18 |
+
try:
|
| 19 |
+
from server.models import MentalHealthAction, MentalHealthObservation, MentalHealthState
|
| 20 |
+
except ImportError:
|
| 21 |
+
import sys
|
| 22 |
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 23 |
+
from models import MentalHealthAction, MentalHealthObservation, MentalHealthState
|
| 24 |
+
|
| 25 |
+
TASK_ORDER = ["burnout_detection", "stress_triage", "intervention_plan"]
|
| 26 |
+
|
| 27 |
+
# ── LLM Judge client ──────────────────────────────────────────────────────────
|
| 28 |
+
_llm_client: Optional[OpenAI] = None
|
| 29 |
+
|
| 30 |
+
def _get_llm_client() -> Optional[OpenAI]:
|
| 31 |
+
global _llm_client
|
| 32 |
+
if _llm_client is None:
|
| 33 |
+
api_key = os.environ.get("HF_TOKEN", "")
|
| 34 |
+
base_url = os.environ.get("API_BASE_URL", "https://api-inference.huggingface.co/v1")
|
| 35 |
+
if api_key:
|
| 36 |
+
_llm_client = OpenAI(api_key=api_key, base_url=base_url)
|
| 37 |
+
return _llm_client
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# ── Randomised scenario data pools ───────────────────────────────────────────
|
| 41 |
+
NAMES = ["Alex","Jordan","Sam","Riley","Morgan","Casey","Taylor","Drew","Jamie","Avery"]
|
| 42 |
+
ROLES = ["Software Engineer","DevOps Engineer","QA Engineer","Data Scientist",
|
| 43 |
+
"Backend Developer","Frontend Developer","ML Engineer","Platform Engineer"]
|
| 44 |
+
YEARS_EXP = [1,2,3,4,5,7,8,10]
|
| 45 |
+
|
| 46 |
+
EXHAUSTION_SYMPTOMS = [
|
| 47 |
+
"sleeping only 4-5 hours/night",
|
| 48 |
+
"skipping meals regularly",
|
| 49 |
+
"frequent headaches and migraines",
|
| 50 |
+
"working 65-75 hours/week for 3+ months",
|
| 51 |
+
"no vacation in over 12 months",
|
| 52 |
+
"falling asleep at desk",
|
| 53 |
+
"constant tiredness even after weekends",
|
| 54 |
+
]
|
| 55 |
+
DEPERSONALIZATION_SYMPTOMS = [
|
| 56 |
+
"says 'I don't care about outcomes anymore'",
|
| 57 |
+
"becoming cynical about project goals",
|
| 58 |
+
"stopped attending team standups",
|
| 59 |
+
"avoids messages for hours",
|
| 60 |
+
"emotionally detached during sprint reviews",
|
| 61 |
+
"sarcastic about the company mission",
|
| 62 |
+
]
|
| 63 |
+
REDUCED_ACCOMPLISHMENT_SYMPTOMS = [
|
| 64 |
+
"submitting incomplete pull requests",
|
| 65 |
+
"missing sprint commitments regularly",
|
| 66 |
+
"says 'nothing I do matters anyway'",
|
| 67 |
+
"code review quality has declined sharply",
|
| 68 |
+
"stopped proposing solutions in meetings",
|
| 69 |
+
"productivity metrics down 40% vs last quarter",
|
| 70 |
+
]
|
| 71 |
+
|
| 72 |
+
SEVERITY_MAP = {
|
| 73 |
+
"Low": {"hours_range":(42,50), "vacation_months":(1,4), "dimensions":1},
|
| 74 |
+
"Moderate": {"hours_range":(50,60), "vacation_months":(4,8), "dimensions":2},
|
| 75 |
+
"High": {"hours_range":(60,68), "vacation_months":(8,14), "dimensions":2},
|
| 76 |
+
"Critical": {"hours_range":(68,80), "vacation_months":(14,24),"dimensions":3},
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
STRESS_TIER_TEMPLATES = {
|
| 80 |
+
"GREEN": ["Feeling slightly overwhelmed with the sprint but I think I'll manage."],
|
| 81 |
+
"AMBER": ["Haven't slept great — around 6 hours most nights. Starting to affect focus."],
|
| 82 |
+
"RED": ["I snapped at a teammate yesterday. I'm scared of how I'm feeling lately."],
|
| 83 |
+
"CRITICAL": ["I've been having chest tightness every time I get a pager alert. Three weeks. Haven't told anyone."],
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def generate_burnout_scenario(rng):
|
| 88 |
+
name = rng.choice(NAMES)
|
| 89 |
+
role = rng.choice(ROLES)
|
| 90 |
+
exp = rng.choice(YEARS_EXP)
|
| 91 |
+
severity_label, cfg = rng.choice(list(SEVERITY_MAP.items()))
|
| 92 |
+
hours = rng.randint(*cfg["hours_range"])
|
| 93 |
+
vacation = rng.randint(*cfg["vacation_months"])
|
| 94 |
+
n_dims = cfg["dimensions"]
|
| 95 |
+
|
| 96 |
+
all_dims = ["exhaustion","depersonalization","reduced_accomplishment"]
|
| 97 |
+
active_dims = rng.sample(all_dims, n_dims)
|
| 98 |
+
symptoms = []
|
| 99 |
+
ground_truth_dims = []
|
| 100 |
+
if "exhaustion" in active_dims:
|
| 101 |
+
symptoms += rng.sample(EXHAUSTION_SYMPTOMS, 2)
|
| 102 |
+
ground_truth_dims.append("Exhaustion")
|
| 103 |
+
if "depersonalization" in active_dims:
|
| 104 |
+
symptoms += rng.sample(DEPERSONALIZATION_SYMPTOMS, 2)
|
| 105 |
+
ground_truth_dims.append("Depersonalization")
|
| 106 |
+
if "reduced_accomplishment" in active_dims:
|
| 107 |
+
symptoms += rng.sample(REDUCED_ACCOMPLISHMENT_SYMPTOMS, 2)
|
| 108 |
+
ground_truth_dims.append("Reduced Personal Accomplishment")
|
| 109 |
+
rng.shuffle(symptoms)
|
| 110 |
+
sym_text = "\n".join(f"- {s}" for s in symptoms)
|
| 111 |
+
|
| 112 |
+
scenario = f"""[TASK: Burnout Detection — EASY]
|
| 113 |
+
|
| 114 |
+
You are an occupational psychologist reviewing an IT employee profile.
|
| 115 |
+
|
| 116 |
+
Employee Profile:
|
| 117 |
+
- Name: {name} ({role}, {exp} years experience)
|
| 118 |
+
- Working hours: ~{hours} hrs/week for the past 3 months
|
| 119 |
+
- Last vacation: {vacation} months ago
|
| 120 |
+
- Observed symptoms:
|
| 121 |
+
{sym_text}
|
| 122 |
+
|
| 123 |
+
Your task:
|
| 124 |
+
1. Identify which MBI dimensions are present: Exhaustion / Depersonalization / Reduced Personal Accomplishment
|
| 125 |
+
2. Rate severity: Low / Moderate / High / Critical
|
| 126 |
+
3. List TOP 3 red-flag signals from the profile
|
| 127 |
+
4. State whether immediate HR escalation is needed (Yes/No) and why
|
| 128 |
+
|
| 129 |
+
Respond with clear headings."""
|
| 130 |
+
|
| 131 |
+
gt = {"active_dimensions":ground_truth_dims, "severity":severity_label,
|
| 132 |
+
"escalation_needed":severity_label in ("High","Critical"),
|
| 133 |
+
"name":name, "hours":hours, "vacation_months":vacation}
|
| 134 |
+
return scenario, gt
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def generate_stress_triage_scenario(rng):
|
| 138 |
+
tiers = ["CRITICAL","RED",rng.choice(["GREEN","AMBER"])]
|
| 139 |
+
rng.shuffle(tiers)
|
| 140 |
+
selected_names = rng.sample(NAMES, len(tiers))
|
| 141 |
+
employees = []
|
| 142 |
+
for idx, tier in enumerate(tiers):
|
| 143 |
+
employees.append({
|
| 144 |
+
"name": selected_names[idx],
|
| 145 |
+
"role": rng.choice(ROLES),
|
| 146 |
+
"tier": tier,
|
| 147 |
+
"quote": rng.choice(STRESS_TIER_TEMPLATES[tier]),
|
| 148 |
+
})
|
| 149 |
+
tier_order = {"CRITICAL":0,"RED":1,"AMBER":2,"GREEN":3}
|
| 150 |
+
priority_order = sorted(range(3), key=lambda i: tier_order[employees[i]["tier"]])
|
| 151 |
+
|
| 152 |
+
cases = "".join(
|
| 153 |
+
f'{i+1}. {e["name"]} ({e["role"]}): "{e["quote"]}"\n'
|
| 154 |
+
for i,e in enumerate(employees)
|
| 155 |
+
)
|
| 156 |
+
scenario = f"""[TASK: Stress Triage — MEDIUM]
|
| 157 |
+
|
| 158 |
+
You are reviewing urgent mental health flags from an IT pulse survey.
|
| 159 |
+
Triage these 3 employees and prioritise them.
|
| 160 |
+
|
| 161 |
+
Cases:
|
| 162 |
+
{cases}
|
| 163 |
+
For each case:
|
| 164 |
+
a) Assign: GREEN / AMBER / RED / CRITICAL
|
| 165 |
+
b) Primary stressor: Workload / Physiological / Relationship / Cognitive
|
| 166 |
+
c) ONE immediate action (within 24 hours)
|
| 167 |
+
d) ONE medium-term support (within 2 weeks)
|
| 168 |
+
|
| 169 |
+
Rank the 3 cases by intervention priority (1 = most urgent)."""
|
| 170 |
+
|
| 171 |
+
gt = {
|
| 172 |
+
"employees": employees,
|
| 173 |
+
"correct_tiers": {e["name"]:e["tier"] for e in employees},
|
| 174 |
+
"priority_order": [employees[i]["name"] for i in priority_order],
|
| 175 |
+
"names": [e["name"] for e in employees],
|
| 176 |
+
}
|
| 177 |
+
return scenario, gt
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def generate_intervention_plan_scenario(rng):
|
| 181 |
+
team_size = rng.choice([8,10,12,15,18,20])
|
| 182 |
+
affected = int(team_size * rng.choice([0.4,0.5,0.6,0.7]))
|
| 183 |
+
overtime = rng.choice([10,15,18,20,25])
|
| 184 |
+
oncall = rng.choice([2,3,4,5])
|
| 185 |
+
no_tb = rng.choice([6,9,12,15,18,24])
|
| 186 |
+
no_1on1 = rng.choice([3,4,5,6,8])
|
| 187 |
+
on_leave = rng.randint(1,3)
|
| 188 |
+
hr_c = rng.randint(1,4)
|
| 189 |
+
|
| 190 |
+
scenario = f"""[TASK: Intervention Plan — HARD]
|
| 191 |
+
|
| 192 |
+
You are a workplace mental health consultant. Design a 4-week intervention for a
|
| 193 |
+
software team with systemic burnout ({affected} of {team_size} members affected).
|
| 194 |
+
|
| 195 |
+
Team Context:
|
| 196 |
+
- Overtime: {overtime} hrs/week above contract
|
| 197 |
+
- On-call: 1 person every {oncall} days (24/7)
|
| 198 |
+
- No team-building in {no_tb} months
|
| 199 |
+
- No manager 1:1s in {no_1on1} months
|
| 200 |
+
- {hr_c} formal HR complaints about workload
|
| 201 |
+
- {on_leave} members on anxiety-related medical leave
|
| 202 |
+
|
| 203 |
+
Design a 4-week plan:
|
| 204 |
+
Week 1: Immediate stabilisation
|
| 205 |
+
Week 2: Assessment & listening
|
| 206 |
+
Week 3: Process reforms (on-call, overtime, workload)
|
| 207 |
+
Week 4: Sustainable systems
|
| 208 |
+
|
| 209 |
+
For each week: 2+ concrete actions, responsible party (HR/Manager/EAP), measurable outcome.
|
| 210 |
+
|
| 211 |
+
Also include:
|
| 212 |
+
- 3 measurable KPIs for 90-day tracking
|
| 213 |
+
- 1 key risk if plan is NOT executed
|
| 214 |
+
- Budget: Low (<$500) / Medium ($500-$5000) / High (>$5000)"""
|
| 215 |
+
|
| 216 |
+
gt = {"team_size":team_size,"affected":affected,"overtime":overtime,
|
| 217 |
+
"oncall_days":oncall,"on_leave":on_leave,"hr_complaints":hr_c}
|
| 218 |
+
return scenario, gt
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# ── Clinical rubrics ──────────────────────────────────────────────────────────
|
| 222 |
+
RUBRICS = {
|
| 223 |
+
"burnout_detection": {
|
| 224 |
+
"dimensions_identified": {"max":3.0,"desc":"1pt per correct MBI dimension identified (0-3)"},
|
| 225 |
+
"severity_accuracy": {"max":2.0,"desc":"2=exact match, 1=adjacent, 0=wrong"},
|
| 226 |
+
"red_flags_quality": {"max":2.0,"desc":"Flags are specific and drawn from the profile (0-2)"},
|
| 227 |
+
"escalation_reasoning": {"max":2.0,"desc":"Correct Yes/No + sound clinical reasoning (0-2)"},
|
| 228 |
+
"structure_clarity": {"max":1.0,"desc":"Clear headings, professional format (0-1)"},
|
| 229 |
+
},
|
| 230 |
+
"stress_triage": {
|
| 231 |
+
"tier_accuracy": {"max":3.0,"desc":"1pt per correct tier assigned (0-3)"},
|
| 232 |
+
"priority_ranking": {"max":2.0,"desc":"2=correct, 1=partially correct, 0=wrong"},
|
| 233 |
+
"immediate_actions": {"max":2.0,"desc":"Specific 24h actions proportionate to tier (0-2)"},
|
| 234 |
+
"medium_term_support": {"max":2.0,"desc":"Realistic 2-week support for each case (0-2)"},
|
| 235 |
+
"completeness": {"max":1.0,"desc":"All 3 employees addressed with all 4 elements (0-1)"},
|
| 236 |
+
},
|
| 237 |
+
"intervention_plan": {
|
| 238 |
+
"four_week_structure": {"max":3.0,"desc":"All 4 weeks with distinct focus (0-3)"},
|
| 239 |
+
"action_concreteness": {"max":2.0,"desc":"Actions specific, not vague (0-2)"},
|
| 240 |
+
"responsibility": {"max":1.0,"desc":"Responsible party per action (0-1)"},
|
| 241 |
+
"kpis_quality": {"max":2.0,"desc":"3 measurable, relevant KPIs (0-2)"},
|
| 242 |
+
"risk_and_budget": {"max":1.0,"desc":"Risk realistic + budget category present (0-1)"},
|
| 243 |
+
"proportionality": {"max":1.0,"desc":"Plan addresses this team's specific numbers (0-1)"},
|
| 244 |
+
},
|
| 245 |
+
}
|
| 246 |
+
|
| 247 |
+
JUDGE_SYSTEM = """You are a clinical rubric evaluator for an AI mental health RL environment.
|
| 248 |
+
Score the agent's response against the rubric. Award marks based on reasoning quality and accuracy — NOT keyword presence alone.
|
| 249 |
+
Return ONLY valid JSON, no markdown fences:
|
| 250 |
+
{"scores": {"dim_name": float, ...}, "feedback": "2-3 sentence critique"}"""
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _llm_judge_grade(task_id, scenario, response, ground_truth):
|
| 254 |
+
rubric = RUBRICS[task_id]
|
| 255 |
+
total_max = sum(v["max"] for v in rubric.values())
|
| 256 |
+
rubric_text = "\n".join(f' "{k}": max={v["max"]} — {v["desc"]}' for k,v in rubric.items())
|
| 257 |
+
|
| 258 |
+
prompt = f"""Task: {task_id}
|
| 259 |
+
Scenario: {scenario}
|
| 260 |
+
Ground truth: {json.dumps(ground_truth)}
|
| 261 |
+
Agent response: {response}
|
| 262 |
+
Rubric:
|
| 263 |
+
{rubric_text}
|
| 264 |
+
Score now. JSON only."""
|
| 265 |
+
|
| 266 |
+
client = _get_llm_client()
|
| 267 |
+
if client is None:
|
| 268 |
+
return _heuristic_grade(task_id, response, ground_truth)
|
| 269 |
+
|
| 270 |
+
model = os.environ.get("MODEL_NAME","meta-llama/Llama-3.1-8B-Instruct")
|
| 271 |
+
try:
|
| 272 |
+
resp = client.chat.completions.create(
|
| 273 |
+
model=model,
|
| 274 |
+
messages=[{"role":"system","content":JUDGE_SYSTEM},
|
| 275 |
+
{"role":"user","content":prompt}],
|
| 276 |
+
max_tokens=400, temperature=0.0,
|
| 277 |
+
)
|
| 278 |
+
raw = resp.choices[0].message.content.strip().replace("```json","").replace("```","").strip()
|
| 279 |
+
parsed = json.loads(raw)
|
| 280 |
+
scores = parsed.get("scores",{})
|
| 281 |
+
feedback = parsed.get("feedback","")
|
| 282 |
+
breakdown, total = {}, 0.0
|
| 283 |
+
for dim, cfg in rubric.items():
|
| 284 |
+
s = max(0.0, min(cfg["max"], float(scores.get(dim, 0.0))))
|
| 285 |
+
breakdown[dim] = round(s/cfg["max"], 3)
|
| 286 |
+
total += s
|
| 287 |
+
reward = round(max(0.0, min(1.0, total/total_max)), 3)
|
| 288 |
+
return reward, breakdown, f"[LLM Judge] {feedback}"
|
| 289 |
+
except Exception as e:
|
| 290 |
+
r,b,f = _heuristic_grade(task_id, response, ground_truth)
|
| 291 |
+
return r, b, f"[Fallback — judge error: {e}] {f}"
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def _heuristic_grade(task_id, response, ground_truth):
|
| 295 |
+
"""Ground-truth-aware heuristic fallback (better than pure keyword matching)."""
|
| 296 |
+
r = response.lower()
|
| 297 |
+
breakdown, score = {}, 0.0
|
| 298 |
+
|
| 299 |
+
if task_id == "burnout_detection":
|
| 300 |
+
dims = ground_truth.get("active_dimensions",[])
|
| 301 |
+
# Match on a distinctive token per dimension, not just the first word.
|
| 302 |
+
dim_keys = {
|
| 303 |
+
"Exhaustion": ["exhaustion","exhausted"],
|
| 304 |
+
"Depersonalization": ["depersonalization","depersonalisation","cynicism","cynical"],
|
| 305 |
+
"Reduced Personal Accomplishment": ["reduced personal accomplishment","reduced accomplishment","inefficacy","personal accomplishment"],
|
| 306 |
+
}
|
| 307 |
+
dim_hits = sum(1 for d in dims if any(k in r for k in dim_keys.get(d, [d.lower()])))
|
| 308 |
+
breakdown["dimensions_identified"] = dim_hits/max(len(dims),1)
|
| 309 |
+
# Severity: exact match = 1.0, adjacent tier = 0.5, otherwise 0.
|
| 310 |
+
sev_order = ["low","moderate","high","critical"]
|
| 311 |
+
sev = ground_truth.get("severity","High").lower()
|
| 312 |
+
mentioned = [s for s in sev_order if s in r]
|
| 313 |
+
if sev in mentioned:
|
| 314 |
+
breakdown["severity_accuracy"] = 1.0
|
| 315 |
+
elif mentioned and sev in sev_order:
|
| 316 |
+
gt_idx = sev_order.index(sev)
|
| 317 |
+
best = min(abs(sev_order.index(m) - gt_idx) for m in mentioned)
|
| 318 |
+
breakdown["severity_accuracy"] = 0.5 if best == 1 else 0.0
|
| 319 |
+
else:
|
| 320 |
+
breakdown["severity_accuracy"] = 0.0
|
| 321 |
+
breakdown["red_flags_quality"] = min(1.0, len(response)/300)
|
| 322 |
+
esc = ground_truth.get("escalation_needed",True)
|
| 323 |
+
esc_hit = "yes" in r or "escalat" in r or "immediate" in r
|
| 324 |
+
breakdown["escalation_reasoning"] = 1.0 if esc==esc_hit else 0.3
|
| 325 |
+
breakdown["structure_clarity"] = 1.0 if any(h in response for h in ["##","**","1.","1:"]) else 0.3
|
| 326 |
+
|
| 327 |
+
elif task_id == "stress_triage":
|
| 328 |
+
tiers = ground_truth.get("correct_tiers",{})
|
| 329 |
+
tier_hits = sum(1 for nm,t in tiers.items() if nm.lower() in r and t.lower() in r)
|
| 330 |
+
breakdown["tier_accuracy"] = tier_hits/max(len(tiers),1)
|
| 331 |
+
priority = ground_truth.get("priority_order",[])
|
| 332 |
+
breakdown["priority_ranking"] = 1.0 if (priority and priority[0].lower() in r[:500]) else 0.4
|
| 333 |
+
breakdown["immediate_actions"] = min(1.0, r.count("24")*0.3 + r.count("immediat")*0.4)
|
| 334 |
+
breakdown["medium_term_support"] = 1.0 if ("week" in r or "fortnight" in r) else 0.3
|
| 335 |
+
names = ground_truth.get("names",[])
|
| 336 |
+
breakdown["completeness"] = 1.0 if all(n.lower() in r for n in names) else 0.4
|
| 337 |
+
|
| 338 |
+
elif task_id == "intervention_plan":
|
| 339 |
+
weeks = sum(1 for w in ["week 1","week 2","week 3","week 4"] if w in r)
|
| 340 |
+
breakdown["four_week_structure"] = weeks/4.0
|
| 341 |
+
breakdown["action_concreteness"] = min(1.0, len(response)/600)
|
| 342 |
+
breakdown["responsibility"] = 1.0 if any(x in r for x in ["hr","manager","eap"]) else 0.0
|
| 343 |
+
kpi_hits = sum(1 for k in ["kpi","metric","measure","indicator"] if k in r)
|
| 344 |
+
breakdown["kpis_quality"] = min(1.0, kpi_hits*0.4)
|
| 345 |
+
breakdown["risk_and_budget"] = 1.0 if ("risk" in r and any(b in r for b in ["low","medium","high","$"])) else 0.3
|
| 346 |
+
gt_size = str(ground_truth.get("team_size",""))
|
| 347 |
+
breakdown["proportionality"] = 1.0 if gt_size in response else 0.5
|
| 348 |
+
|
| 349 |
+
score = sum(breakdown.values())/max(len(breakdown),1)
|
| 350 |
+
reward = round(max(0.0, min(1.0, score)), 3)
|
| 351 |
+
return reward, breakdown, f"[Heuristic] {task_id} = {reward:.2f}. Breakdown: {breakdown}"
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
# ── Environment class ─────────────────────────────────────────────────────────
|
| 355 |
+
class ITMentalHealthEnvironment:
|
| 356 |
+
"""
|
| 357 |
+
OpenEnv RL environment — IT sector mental health.
|
| 358 |
+
Every reset() is unique (randomised profiles/scenarios).
|
| 359 |
+
Graded by LLM-as-judge against clinical rubric.
|
| 360 |
+
"""
|
| 361 |
+
def __init__(self):
|
| 362 |
+
self._state = MentalHealthState()
|
| 363 |
+
self._task_index = 0
|
| 364 |
+
self._rng = random.Random()
|
| 365 |
+
self._current_scenarios: dict = {}
|
| 366 |
+
|
| 367 |
+
def _generate_all(self):
|
| 368 |
+
self._current_scenarios = {
|
| 369 |
+
"burnout_detection": generate_burnout_scenario(self._rng),
|
| 370 |
+
"stress_triage": generate_stress_triage_scenario(self._rng),
|
| 371 |
+
"intervention_plan": generate_intervention_plan_scenario(self._rng),
|
| 372 |
+
}
|
| 373 |
+
|
| 374 |
+
def reset(self, seed=None):
|
| 375 |
+
actual_seed = seed if seed is not None else random.randint(0, 2**31)
|
| 376 |
+
self._rng = random.Random(actual_seed)
|
| 377 |
+
self._task_index = 0
|
| 378 |
+
self._generate_all()
|
| 379 |
+
self._state = MentalHealthState(
|
| 380 |
+
episode_id=str(uuid.uuid4()), step_count=0,
|
| 381 |
+
current_task=TASK_ORDER[0], cumulative_reward=0.0, tasks_completed=[],
|
| 382 |
+
)
|
| 383 |
+
task = TASK_ORDER[0]
|
| 384 |
+
scenario_text, _ = self._current_scenarios[task]
|
| 385 |
+
return MentalHealthObservation(
|
| 386 |
+
scenario=scenario_text, feedback=f"New episode (seed={actual_seed}).",
|
| 387 |
+
reward=0.0, done=False, score_breakdown={}, task_id=task,
|
| 388 |
+
metadata={"seed": actual_seed},
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
def step(self, action):
|
| 392 |
+
# Guard: refuse to grade once the episode is finished.
|
| 393 |
+
if self._task_index >= len(TASK_ORDER):
|
| 394 |
+
return MentalHealthObservation(
|
| 395 |
+
scenario="Episode already finished. Call /reset to start a new one.",
|
| 396 |
+
feedback="No-op: episode is done.",
|
| 397 |
+
reward=0.0, done=True, score_breakdown={},
|
| 398 |
+
task_id=self._state.current_task,
|
| 399 |
+
)
|
| 400 |
+
|
| 401 |
+
self._state.step_count += 1
|
| 402 |
+
task = self._state.current_task
|
| 403 |
+
scenario_text, ground_truth = self._current_scenarios.get(task, ("", {}))
|
| 404 |
+
|
| 405 |
+
reward, breakdown, feedback = _llm_judge_grade(
|
| 406 |
+
task_id=task, scenario=scenario_text,
|
| 407 |
+
response=action.response, ground_truth=ground_truth,
|
| 408 |
+
)
|
| 409 |
+
self._state.cumulative_reward += reward
|
| 410 |
+
self._state.tasks_completed.append(task)
|
| 411 |
+
|
| 412 |
+
self._task_index += 1
|
| 413 |
+
if self._task_index >= len(TASK_ORDER):
|
| 414 |
+
done = True
|
| 415 |
+
next_task = task
|
| 416 |
+
next_scenario = "All tasks complete. Episode finished."
|
| 417 |
+
else:
|
| 418 |
+
done = False
|
| 419 |
+
next_task = TASK_ORDER[self._task_index]
|
| 420 |
+
self._state.current_task = next_task
|
| 421 |
+
next_scenario, _ = self._current_scenarios[next_task]
|
| 422 |
+
|
| 423 |
+
return MentalHealthObservation(
|
| 424 |
+
scenario=next_scenario, feedback=feedback,
|
| 425 |
+
reward=reward, done=done, score_breakdown=breakdown, task_id=next_task,
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
@property
|
| 429 |
+
def state(self):
|
| 430 |
+
return self._state
|
models.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
IT Mental Health OpenEnv - Typed Models
|
| 3 |
+
Action / Observation / State for the IT Burnout & Mental Wellness environment.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from dataclasses import dataclass, field
|
| 7 |
+
from typing import Optional, Dict, Any, List
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@dataclass
|
| 11 |
+
class MentalHealthAction:
|
| 12 |
+
"""
|
| 13 |
+
Action submitted by the agent for each mental health scenario step.
|
| 14 |
+
|
| 15 |
+
Fields:
|
| 16 |
+
response (str): The agent's textual response / intervention recommendation.
|
| 17 |
+
task_id (str): Which task is being attempted: 'burnout_detection',
|
| 18 |
+
'stress_triage', or 'intervention_plan'.
|
| 19 |
+
confidence (float): Agent's self-reported confidence 0.0–1.0.
|
| 20 |
+
metadata (dict): Optional extra fields (reasoning chain, flags, etc.)
|
| 21 |
+
"""
|
| 22 |
+
response: str
|
| 23 |
+
task_id: str = "burnout_detection"
|
| 24 |
+
confidence: float = 1.0
|
| 25 |
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@dataclass
|
| 29 |
+
class MentalHealthObservation:
|
| 30 |
+
"""
|
| 31 |
+
Observation returned after each step.
|
| 32 |
+
|
| 33 |
+
Fields:
|
| 34 |
+
scenario (str): The current scenario description shown to the agent.
|
| 35 |
+
feedback (str): Evaluator feedback on the last action.
|
| 36 |
+
reward (float): Reward for the last step (0.0 – 1.0).
|
| 37 |
+
done (bool): Whether the episode is finished.
|
| 38 |
+
score_breakdown (dict): Partial scores by rubric dimension.
|
| 39 |
+
task_id (str): Current task identifier.
|
| 40 |
+
"""
|
| 41 |
+
scenario: str
|
| 42 |
+
feedback: str
|
| 43 |
+
reward: float
|
| 44 |
+
done: bool
|
| 45 |
+
score_breakdown: Dict[str, float] = field(default_factory=dict)
|
| 46 |
+
task_id: str = "burnout_detection"
|
| 47 |
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@dataclass
|
| 51 |
+
class MentalHealthState:
|
| 52 |
+
"""
|
| 53 |
+
Episode state / metadata.
|
| 54 |
+
|
| 55 |
+
Fields:
|
| 56 |
+
episode_id (str): UUID for the current episode.
|
| 57 |
+
step_count (int): How many steps have elapsed.
|
| 58 |
+
current_task (str): Active task identifier.
|
| 59 |
+
cumulative_reward (float): Total reward accumulated so far.
|
| 60 |
+
tasks_completed (list): List of task IDs already completed.
|
| 61 |
+
"""
|
| 62 |
+
episode_id: Optional[str] = None
|
| 63 |
+
step_count: int = 0
|
| 64 |
+
current_task: str = "burnout_detection"
|
| 65 |
+
cumulative_reward: float = 0.0
|
| 66 |
+
tasks_completed: List[str] = field(default_factory=list)
|
openenv.yaml
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
spec_version: 1
|
| 2 |
+
|
| 3 |
+
name: it_mental_health_env
|
| 4 |
+
version: "1.0.0"
|
| 5 |
+
description: >
|
| 6 |
+
An RL environment for mental health assessment and intervention planning
|
| 7 |
+
in the IT/software engineering sector. An AI agent faces three progressively
|
| 8 |
+
harder scenarios: detecting burnout signals (easy), triaging employee stress
|
| 9 |
+
cases (medium), and designing a structured team-wide intervention plan (hard).
|
| 10 |
+
Reward is 0.0–1.0 per task based on rubric coverage, structured format, and
|
| 11 |
+
clinical accuracy.
|
| 12 |
+
|
| 13 |
+
type: typed # uses typed Action / Observation / State models
|
| 14 |
+
|
| 15 |
+
runtime:
|
| 16 |
+
language: python
|
| 17 |
+
version: "3.11"
|
| 18 |
+
|
| 19 |
+
app:
|
| 20 |
+
module: app
|
| 21 |
+
object: app
|
| 22 |
+
|
| 23 |
+
port: 7860
|
| 24 |
+
|
| 25 |
+
tasks:
|
| 26 |
+
- id: burnout_detection
|
| 27 |
+
difficulty: easy
|
| 28 |
+
description: Identify Maslach burnout dimensions and severity from an employee profile
|
| 29 |
+
reward_range: [0.0, 1.0]
|
| 30 |
+
|
| 31 |
+
- id: stress_triage
|
| 32 |
+
difficulty: medium
|
| 33 |
+
description: Triage 3 IT employees by stress tier and recommend immediate actions
|
| 34 |
+
reward_range: [0.0, 1.0]
|
| 35 |
+
|
| 36 |
+
- id: intervention_plan
|
| 37 |
+
difficulty: hard
|
| 38 |
+
description: Design a 4-week mental health intervention plan for a burned-out IT team
|
| 39 |
+
reward_range: [0.0, 1.0]
|
| 40 |
+
|
| 41 |
+
action_space:
|
| 42 |
+
type: text
|
| 43 |
+
fields:
|
| 44 |
+
response: str
|
| 45 |
+
task_id: str
|
| 46 |
+
confidence: float
|
| 47 |
+
metadata: dict
|
| 48 |
+
|
| 49 |
+
observation_space:
|
| 50 |
+
type: text
|
| 51 |
+
fields:
|
| 52 |
+
scenario: str
|
| 53 |
+
feedback: str
|
| 54 |
+
reward: float
|
| 55 |
+
done: bool
|
| 56 |
+
score_breakdown: dict
|
| 57 |
+
task_id: str
|
| 58 |
+
|
| 59 |
+
endpoints:
|
| 60 |
+
reset: POST /reset
|
| 61 |
+
step: POST /step
|
| 62 |
+
state: GET /state
|
| 63 |
+
health: GET /health
|
| 64 |
+
tasks: GET /tasks
|
| 65 |
+
schema: GET /schema
|
| 66 |
+
|
| 67 |
+
author: "IT Mental Health OpenEnv — Scaler x Meta-PyTorch Hackathon 2026"
|
| 68 |
+
license: MIT
|
requirements.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi
|
| 2 |
+
uvicorn
|
| 3 |
+
pydantic
|
| 4 |
+
requests
|
| 5 |
+
PyYAML
|
| 6 |
+
openai
|
requirements_inference.txt
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
requests
|
| 2 |
+
openai
|
validate.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
validate.py - Pre-submission validator for IT Mental Health OpenEnv
|
| 3 |
+
Run this before submitting: python validate.py
|
| 4 |
+
|
| 5 |
+
Checks:
|
| 6 |
+
1. openenv.yaml is present and has required fields
|
| 7 |
+
2. /health returns 200
|
| 8 |
+
3. /reset responds correctly
|
| 9 |
+
4. /step responds correctly (all 3 tasks)
|
| 10 |
+
5. /state responds correctly
|
| 11 |
+
6. Rewards are in 0.0-1.0 range
|
| 12 |
+
7. inference.py exists at repo root
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
import sys
|
| 17 |
+
|
| 18 |
+
import requests
|
| 19 |
+
|
| 20 |
+
BASE_URL = "http://localhost:7860"
|
| 21 |
+
|
| 22 |
+
PASS = "[PASS]"
|
| 23 |
+
FAIL = "[FAIL]"
|
| 24 |
+
results = []
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def check(name: str, ok: bool, detail: str = ""):
|
| 28 |
+
status = PASS if ok else FAIL
|
| 29 |
+
msg = f"{status} {name}"
|
| 30 |
+
if detail:
|
| 31 |
+
msg += f" - {detail}"
|
| 32 |
+
print(msg)
|
| 33 |
+
results.append(ok)
|
| 34 |
+
return ok
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def run():
|
| 38 |
+
print("\n" + "=" * 60)
|
| 39 |
+
print(" IT Mental Health OpenEnv - Pre-Submission Validator")
|
| 40 |
+
print("=" * 60 + "\n")
|
| 41 |
+
|
| 42 |
+
try:
|
| 43 |
+
import yaml
|
| 44 |
+
|
| 45 |
+
with open("openenv.yaml", encoding="utf-8") as f:
|
| 46 |
+
cfg = yaml.safe_load(f)
|
| 47 |
+
required = ["spec_version", "name", "type", "runtime", "app", "port", "tasks"]
|
| 48 |
+
missing = [k for k in required if k not in cfg]
|
| 49 |
+
check(
|
| 50 |
+
"openenv.yaml present & valid",
|
| 51 |
+
not missing,
|
| 52 |
+
f"missing: {missing}" if missing else f"name={cfg['name']}",
|
| 53 |
+
)
|
| 54 |
+
task_count = len(cfg.get("tasks", []))
|
| 55 |
+
check("3+ tasks in openenv.yaml", task_count >= 3, f"{task_count} tasks found")
|
| 56 |
+
except Exception as e:
|
| 57 |
+
check("openenv.yaml", False, str(e))
|
| 58 |
+
|
| 59 |
+
check("inference.py at root", os.path.exists("inference.py"))
|
| 60 |
+
|
| 61 |
+
try:
|
| 62 |
+
r = requests.get(f"{BASE_URL}/health", timeout=10)
|
| 63 |
+
check("/health returns 200", r.status_code == 200, f"status={r.status_code}")
|
| 64 |
+
except Exception as e:
|
| 65 |
+
check("/health", False, str(e))
|
| 66 |
+
|
| 67 |
+
try:
|
| 68 |
+
r = requests.post(f"{BASE_URL}/reset", json={}, timeout=10)
|
| 69 |
+
ok = r.status_code == 200
|
| 70 |
+
data = r.json() if ok else {}
|
| 71 |
+
check("/reset returns 200", ok)
|
| 72 |
+
check("/reset has 'scenario'", "scenario" in data)
|
| 73 |
+
check("/reset has 'reward'", "reward" in data)
|
| 74 |
+
check("/reset has 'done'", "done" in data)
|
| 75 |
+
except Exception as e:
|
| 76 |
+
check("/reset", False, str(e))
|
| 77 |
+
|
| 78 |
+
try:
|
| 79 |
+
requests.post(f"{BASE_URL}/reset", json={}, timeout=10)
|
| 80 |
+
task_ids = ["burnout_detection", "stress_triage", "intervention_plan"]
|
| 81 |
+
sample_response = (
|
| 82 |
+
"The employee shows Exhaustion, Depersonalization, and Reduced Personal Accomplishment. "
|
| 83 |
+
"Severity: High/Critical. Red flags: 70hr weeks, 14 months no leave, disconnection. "
|
| 84 |
+
"Immediate HR escalation: Yes - employee is at critical burnout risk.\n\n"
|
| 85 |
+
"Jordan: RED/CRITICAL tier. Riley: RED tier (chest tightness = physiological alarm). "
|
| 86 |
+
"Sam: AMBER tier. Priority ranking: 1=Riley, 2=Jordan, 3=Sam.\n\n"
|
| 87 |
+
"Week 1: Immediate on-call freeze, HR 1:1s for all affected. Responsible: HR Manager. "
|
| 88 |
+
"Week 2: Anonymous team survey and individual 30min check-ins. Responsible: HR. "
|
| 89 |
+
"Week 3: On-call rotation reform, overtime cap policy. Responsible: Manager. "
|
| 90 |
+
"Week 4: Bi-weekly 1:1s, EAP enrollment, manager training. Responsible: HR + External EAP. "
|
| 91 |
+
"KPIs: overtime hours, eNPS score, sick leave frequency. "
|
| 92 |
+
"Risk: further attrition and possible disability claims. Budget: Medium ($500-$5000)."
|
| 93 |
+
)
|
| 94 |
+
for tid in task_ids:
|
| 95 |
+
r = requests.post(
|
| 96 |
+
f"{BASE_URL}/step",
|
| 97 |
+
json={
|
| 98 |
+
"response": sample_response,
|
| 99 |
+
"task_id": tid,
|
| 100 |
+
"confidence": 0.9,
|
| 101 |
+
},
|
| 102 |
+
timeout=30,
|
| 103 |
+
)
|
| 104 |
+
ok = r.status_code == 200
|
| 105 |
+
check(f"/step task={tid} returns 200", ok)
|
| 106 |
+
if ok:
|
| 107 |
+
d = r.json()
|
| 108 |
+
reward = d.get("reward", -1)
|
| 109 |
+
valid = 0.0 <= reward <= 1.0
|
| 110 |
+
check(f"reward in [0,1] for {tid}", valid, f"reward={reward:.3f}")
|
| 111 |
+
except Exception as e:
|
| 112 |
+
check("/step", False, str(e))
|
| 113 |
+
|
| 114 |
+
try:
|
| 115 |
+
r = requests.get(f"{BASE_URL}/state", timeout=10)
|
| 116 |
+
ok = r.status_code == 200
|
| 117 |
+
check("/state returns 200", ok)
|
| 118 |
+
if ok:
|
| 119 |
+
d = r.json()
|
| 120 |
+
check("/state has episode_id", "episode_id" in d)
|
| 121 |
+
check("/state has step_count", "step_count" in d)
|
| 122 |
+
except Exception as e:
|
| 123 |
+
check("/state", False, str(e))
|
| 124 |
+
|
| 125 |
+
total = len(results)
|
| 126 |
+
passed = sum(results)
|
| 127 |
+
print(f"\n{'=' * 60}")
|
| 128 |
+
print(f" Result: {passed}/{total} checks passed")
|
| 129 |
+
if passed == total:
|
| 130 |
+
print(" ALL CHECKS PASSED - Ready to submit!")
|
| 131 |
+
else:
|
| 132 |
+
print(" Fix failing checks before submitting.")
|
| 133 |
+
print("=" * 60 + "\n")
|
| 134 |
+
|
| 135 |
+
sys.exit(0 if passed == total else 1)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
if __name__ == "__main__":
|
| 139 |
+
run()
|