| from __future__ import annotations |
|
|
| from pathlib import Path |
|
|
| import yaml |
|
|
| from prompt_injection_framework.orchestration.config import load_config |
| from prompt_injection_framework.orchestration.runner import run_experiment |
| from prompt_injection_framework.utils.io import read_jsonl |
|
|
|
|
| def test_runner_executes_ordered_turns_and_saves_turn_artifacts(tmp_path: Path) -> None: |
| dataset_path = tmp_path / "attack_cases.jsonl" |
| dataset_path.write_text( |
| ( |
| '{"case_id":"case_mt_1","source_name":"Test","source_type":"unit","attack_family":"adaptive_attack",' |
| '"attack_subtype":"follow_up","attack_surface":"adaptive","prompt":"ignored","context":"system",' |
| '"expected_target":"hello","success_criteria":"contains hello",' |
| '"turns":[{"role":"user","prompt":"first harmless turn"},{"role":"user","prompt":"second turn say hello"}]}\n' |
| ), |
| encoding="utf-8", |
| ) |
|
|
| config_path = tmp_path / "config.yaml" |
| config_path.write_text( |
| yaml.safe_dump( |
| { |
| "experiment": { |
| "name": "multiturn_test", |
| "description": "mock multi-turn pipeline test", |
| "seed": 7, |
| "sample_size": 1, |
| "output_dir": str(tmp_path / "results"), |
| }, |
| "dataset": {"path": str(dataset_path)}, |
| "model": { |
| "provider": "mock", |
| "name": "mock-model", |
| "max_tokens": 32, |
| "temperature": 0.0, |
| "execution_mode": "plain", |
| }, |
| "mitigation": {"enabled": False, "strategy": "none"}, |
| } |
| ), |
| encoding="utf-8", |
| ) |
|
|
| config, raw_config = load_config(config_path) |
| summary = run_experiment(config, raw_config) |
| case_results = read_jsonl(Path(summary["result_paths"]["case_results"])) |
|
|
| assert len(case_results) == 1 |
| assert case_results[0]["has_multi_turn_payload"] is True |
| assert case_results[0]["turn_count"] == 2 |
| assert len(case_results[0]["turn_results"]) == 2 |
| assert case_results[0]["turn_results"][0]["turn_index"] == 1 |
| assert case_results[0]["turn_results"][1]["turn_index"] == 2 |
|
|
