import json from pathlib import Path import subprocess import sys import tempfile import unittest from unittest.mock import patch from scripts.fleet_status import query_candidate class FleetStatusTests(unittest.TestCase): def test_partial_logs_explicit_best_step_and_exact_child_identity(self): with tempfile.TemporaryDirectory() as directory: campaign=Path(directory) training=campaign/'training' training.mkdir() command=[sys.executable,'-c','import time; time.sleep(60)'] child=subprocess.Popen(command) try: state={'status':'running','stage':'training','child_pid':child.pid,'child_command':command} (campaign/'state.json').write_text(json.dumps(state)) (training/'training.jsonl').write_text('{"step":19}\n{"step":20}\n{"step":') metric='crossfit_temperature_nll_v1' (training/'validation.jsonl').write_text(json.dumps({'step':10,'improved':True, 'selection':{'metric':metric,'score':0.2}})+'\n{"step":') selected={'metric':metric,'score':0.2,'accuracy':0.875} (training/'best_validation_selection.json').write_text(json.dumps(selected)) candidate={'name':'fixture','host':'local','project':str(campaign), 'python':sys.executable,'campaign':str(campaign),'training':str(training)} result=query_candidate(candidate) self.assertEqual(result['latest_logged_step'],20) self.assertEqual(result['selected_step'],10) self.assertEqual(result['selected_accuracy'],0.875) self.assertTrue(result['processes']['trainer']) self.assertEqual(len(result['warnings']),2) # A new selection can be published before its validation log row; # no previous/current training step may be invented as its step. (training/'best_validation_selection.json').write_text(json.dumps({**selected,'score':0.15})) (campaign/'state.json').write_text(json.dumps({**state,'child_command':command+['mismatch']})) result=query_candidate(candidate) self.assertIsNone(result['selected_step']) self.assertFalse(result['processes']['trainer']) self.assertIsNone(child.poll()) finally: child.terminate() child.wait(timeout=5) def test_timeout_and_missing_remote_state_become_candidate_errors(self): candidate={'name':'offline','host':'andy@192.168.8.204','project':'/project', 'python':'/python','campaign':'/campaign','training':'/training'} with patch('scripts.fleet_status.subprocess.run',side_effect=subprocess.TimeoutExpired('large-inline-reader',15)): result=query_candidate(candidate) self.assertEqual(result['status'],'unavailable') self.assertEqual(result['error'],'read timed out after 15s') self.assertNotIn('large-inline-reader',result['error']) with patch('scripts.fleet_status.subprocess.run',return_value=subprocess.CompletedProcess([],1,'','ValueError: Missing state.json')): result=query_candidate(candidate) self.assertEqual(result['status'],'unavailable') self.assertIn('Missing state.json',result['error']) if __name__=='__main__': unittest.main()