solar_eval/cli/commands/eval.py

"""Evaluation result commands — local JSONL or remote API."""

import json

import click

from solar_eval.cli.config import config
from solar_eval.cli.commands.projects import _resolve_project_id
from solar_eval.cli.formatters import format_evaluation_detail


@click.group("eval")
def eval_group() -> None:
    """View evaluation results."""


@eval_group.command("show")
@click.argument("run_ref")
@click.pass_context
def show_evaluation(ctx: click.Context, run_ref: str) -> None:
    """Show evaluation summary. Local: run dir name. Remote: 'project/run_id'."""
    cfg = ctx.obj["config"]

    if cfg.is_remote:
        from solar_eval.cli.client import EvalClient
        client = EvalClient(cfg.remote_url, cfg.timeout)
        parts = run_ref.split("/")
        if len(parts) != 2:
            raise click.ClickException("Remote: use 'project-name/run-id' format")
        project_id = _resolve_project_id(client, parts[0])
        evaluation = client.get(f"/api/projects/{project_id}/runs/{parts[1]}/evaluation")
    else:
        parts = run_ref.split("/")
        if len(parts) != 2:
            raise click.ClickException("Local mode: use 'project-name/run-name' format")
        eval_file = cfg.artifacts_dir(parts[0]) / parts[1] / "evaluation.json"
        if not eval_file.exists():
            raise click.ClickException(f"Evaluation not found: {eval_file}")
        evaluation = json.loads(eval_file.read_text())

    click.secho(format_evaluation_detail(evaluation))


@eval_group.command("details")
@click.argument("run_ref")
@click.option("--limit", "-n", default=None, type=int, help="Limit results")
@click.pass_context
def show_eval_details(ctx: click.Context, run_ref: str, limit: int | None) -> None:
    """Show per-sample evaluation details."""
    cfg = ctx.obj["config"]

    if cfg.is_remote:
        from solar_eval.cli.client import EvalClient
        client = EvalClient(cfg.remote_url, cfg.timeout)
        parts = run_ref.split("/")
        if len(parts) != 2:
            raise click.ClickException("Remote: use 'project-name/run-id' format")
        project_id = _resolve_project_id(client, parts[0])
        details = client.get(f"/api/projects/{project_id}/runs/{parts[1]}/evaluation/details")
    else:
        parts = run_ref.split("/")
        if len(parts) != 2:
            raise click.ClickException("Local mode: use 'project-name/run-name' format")
        details_file = cfg.artifacts_dir(parts[0]) / parts[1] / "eval_details.jsonl"
        if not details_file.exists():
            raise click.ClickException(f"Details not found: {details_file}")
        details = [json.loads(line) for line in details_file.read_text().strip().split("\n") if line]

    if not details:
        click.secho("No evaluation details found.", fg="yellow")
        return
    if limit:
        details = details[:limit]
    for d in details:
        click.secho(f"Sample #{d.get('sample_idx', '?')}", fg="blue", bold=True)
        if d.get("category_scores"):
            for cat, score in d["category_scores"].items():
                click.echo(f"  {cat}: {score:.2f}")
        if d.get("severity"):
            click.echo(f"  Severity: {d['severity']}")
        click.echo()
    click.echo(f"Showing {len(details)} sample(s)")