| { |
| "gui_run": "/home/andy/ai/opensysone/runs/20260917T032753Z-playground", |
| "gui_pid": 1467162, |
| "gui_source_commit": "e94e58ee311ab10ebe26fc46d4a5dde2280679f6", |
| "trainer_campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h", |
| "purpose": "Read-only coexistence observation after browser tests switched three real frozen models", |
| "observation_method": "Two process/memory/status samples about45seconds apart; GET /api/status only; no scoring/model requests from observer", |
| "launch_preflight_mem_available_bytes": 106415841280, |
| "samples": [ |
| { |
| "observed_utc": "2026-09-17T03:35:55.926185+00:00", |
| "free_b": { |
| "exit_code": 0, |
| "stdout": "total used free shared buff/cache available\nMem: 130594160640 43303702528 29233373184 164196352 36601188352 87290458112\nSwap: 0 0 0" |
| }, |
| "gpu_processes": { |
| "exit_code": 0, |
| "stdout": "4674, /home/andy/ai/apps/llama.cpp/build/bin/llama-server, 170 MiB\n1096505, /home/andy/ai/envs/opensysone/bin/python, 16315 MiB\n1467162, /home/andy/ai/envs/opensysone/bin/python, 15655 MiB" |
| }, |
| "gpu_status": { |
| "exit_code": 0, |
| "stdout": "71, 93 %, 44.89 W" |
| }, |
| "processes": [ |
| { |
| "role": "gui", |
| "pid": 1467162, |
| "command_matches": true, |
| "oom_score_adj": "0", |
| "status": { |
| "State": "S (sleeping)", |
| "VmHWM": "32281892 kB", |
| "VmRSS": "24261080 kB", |
| "Threads": "56" |
| } |
| }, |
| { |
| "role": "trainer", |
| "pid": 1096505, |
| "command_matches": true, |
| "oom_score_adj": "0", |
| "status": { |
| "State": "S (sleeping)", |
| "VmHWM": "24653352 kB", |
| "VmRSS": "3402476 kB", |
| "Threads": "57" |
| } |
| } |
| ], |
| "gui_status": { |
| "http_status": 200, |
| "body": { |
| "status": "ready", |
| "loaded_model_id": "gx10-4b" |
| } |
| }, |
| "gui_error_marker_counts": { |
| "model_load_failed": 0, |
| "model_score_failed": 0, |
| "Traceback": 0, |
| "OutOfMemoryError": 0 |
| }, |
| "training": { |
| "last_step": 3063, |
| "last_update": { |
| "step": 3063, |
| "loss": 0.014626714604673907, |
| "gradient_norm": 0.30683499574661255, |
| "seconds": 8.4302611759922, |
| "decisions": 4, |
| "branches": 12, |
| "actual_branch_tokens": 1218, |
| "padded_branch_tokens": 1218, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.977023980559572 |
| }, |
| "recent100_finite": true, |
| "recent100_median_step_seconds": 9.043510261006304, |
| "recent100_mean_step_seconds": 9.17212467954203, |
| "recent100_max_step_seconds": 11.7679517439974, |
| "recent4_updates": [ |
| { |
| "step": 3060, |
| "loss": 0.019744501798413694, |
| "gradient_norm": 0.9511153101921082, |
| "seconds": 11.7679517439974, |
| "decisions": 4, |
| "branches": 13, |
| "actual_branch_tokens": 1523, |
| "padded_branch_tokens": 1523, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9770701241265155 |
| }, |
| { |
| "step": 3061, |
| "loss": 0.3342024694720731, |
| "gradient_norm": 11.301324844360352, |
| "seconds": 11.187965510995127, |
| "decisions": 4, |
| "branches": 11, |
| "actual_branch_tokens": 1641, |
| "padded_branch_tokens": 1641, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.977054747970101 |
| }, |
| { |
| "step": 3062, |
| "loss": 0.02495433943113312, |
| "gradient_norm": 0.9082462191581726, |
| "seconds": 9.814839319995372, |
| "decisions": 4, |
| "branches": 14, |
| "actual_branch_tokens": 1375, |
| "padded_branch_tokens": 1375, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9770393667810657 |
| }, |
| { |
| "step": 3063, |
| "loss": 0.014626714604673907, |
| "gradient_norm": 0.30683499574661255, |
| "seconds": 8.4302611759922, |
| "decisions": 4, |
| "branches": 12, |
| "actual_branch_tokens": 1218, |
| "padded_branch_tokens": 1218, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.977023980559572 |
| } |
| ], |
| "peak_cuda_allocated_bytes": 16776178176, |
| "checkpoint_age_seconds": 585.4080212116241, |
| "heartbeat_utc": "2026-09-17T03:35:52.527546+00:00" |
| } |
| }, |
| { |
| "observed_utc": "2026-09-17T03:36:41.021859+00:00", |
| "free_b": { |
| "exit_code": 0, |
| "stdout": "total used free shared buff/cache available\nMem: 130594160640 43292459008 29244313600 164196352 36601479168 87301701632\nSwap: 0 0 0" |
| }, |
| "gpu_processes": { |
| "exit_code": 0, |
| "stdout": "4674, /home/andy/ai/apps/llama.cpp/build/bin/llama-server, 170 MiB\n1096505, /home/andy/ai/envs/opensysone/bin/python, 16315 MiB\n1467162, /home/andy/ai/envs/opensysone/bin/python, 15655 MiB" |
| }, |
| "gpu_status": { |
| "exit_code": 0, |
| "stdout": "71, 93 %, 45.40 W" |
| }, |
| "processes": [ |
| { |
| "role": "gui", |
| "pid": 1467162, |
| "command_matches": true, |
| "oom_score_adj": "0", |
| "status": { |
| "State": "S (sleeping)", |
| "VmHWM": "32281892 kB", |
| "VmRSS": "24261080 kB", |
| "Threads": "56" |
| } |
| }, |
| { |
| "role": "trainer", |
| "pid": 1096505, |
| "command_matches": true, |
| "oom_score_adj": "0", |
| "status": { |
| "State": "S (sleeping)", |
| "VmHWM": "24653352 kB", |
| "VmRSS": "3402596 kB", |
| "Threads": "57" |
| } |
| } |
| ], |
| "gui_status": { |
| "http_status": 200, |
| "body": { |
| "status": "ready", |
| "loaded_model_id": "gx10-4b" |
| } |
| }, |
| "gui_error_marker_counts": { |
| "model_load_failed": 0, |
| "model_score_failed": 0, |
| "Traceback": 0, |
| "OutOfMemoryError": 0 |
| }, |
| "training": { |
| "last_step": 3068, |
| "last_update": { |
| "step": 3068, |
| "loss": 0.001936116958859202, |
| "gradient_norm": 0.10404492169618607, |
| "seconds": 8.810512408002978, |
| "decisions": 4, |
| "branches": 12, |
| "actual_branch_tokens": 1407, |
| "padded_branch_tokens": 1407, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769469739709091 |
| }, |
| "recent100_finite": true, |
| "recent100_median_step_seconds": 9.03708774750703, |
| "recent100_mean_step_seconds": 9.169983857631742, |
| "recent100_max_step_seconds": 11.7679517439974, |
| "recent4_updates": [ |
| { |
| "step": 3065, |
| "loss": 0.15298267499747453, |
| "gradient_norm": 6.990951061248779, |
| "seconds": 8.907861668005353, |
| "decisions": 4, |
| "branches": 10, |
| "actual_branch_tokens": 1726, |
| "padded_branch_tokens": 1726, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769931930198583 |
| }, |
| { |
| "step": 3066, |
| "loss": 0.0036661182439274853, |
| "gradient_norm": 0.1209888607263565, |
| "seconds": 8.316543419990921, |
| "decisions": 4, |
| "branches": 11, |
| "actual_branch_tokens": 1353, |
| "padded_branch_tokens": 1353, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769777917019633 |
| }, |
| { |
| "step": 3067, |
| "loss": 0.1977057716858326, |
| "gradient_norm": 11.540794372558594, |
| "seconds": 9.568666854989715, |
| "decisions": 4, |
| "branches": 11, |
| "actual_branch_tokens": 1853, |
| "padded_branch_tokens": 1853, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769623853522593 |
| }, |
| { |
| "step": 3068, |
| "loss": 0.001936116958859202, |
| "gradient_norm": 0.10404492169618607, |
| "seconds": 8.810512408002978, |
| "decisions": 4, |
| "branches": 12, |
| "actual_branch_tokens": 1407, |
| "padded_branch_tokens": 1407, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769469739709091 |
| } |
| ], |
| "peak_cuda_allocated_bytes": 16776178176, |
| "checkpoint_age_seconds": 630.5036952495575, |
| "heartbeat_utc": "2026-09-17T03:36:37.538019+00:00" |
| } |
| } |
| ], |
| "observation_window": { |
| "elapsed_seconds": 45.095674, |
| "optimizer_updates_completed": 5, |
| "steps_per_minute": 6.6525228118333475, |
| "new_update_rows": [ |
| { |
| "step": 3064, |
| "loss": 0.008182858524378389, |
| "gradient_norm": 0.19264547526836395, |
| "seconds": 9.048552577005466, |
| "decisions": 4, |
| "branches": 13, |
| "actual_branch_tokens": 1311, |
| "padded_branch_tokens": 1311, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.977008589305782 |
| }, |
| { |
| "step": 3065, |
| "loss": 0.15298267499747453, |
| "gradient_norm": 6.990951061248779, |
| "seconds": 8.907861668005353, |
| "decisions": 4, |
| "branches": 10, |
| "actual_branch_tokens": 1726, |
| "padded_branch_tokens": 1726, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769931930198583 |
| }, |
| { |
| "step": 3066, |
| "loss": 0.0036661182439274853, |
| "gradient_norm": 0.1209888607263565, |
| "seconds": 8.316543419990921, |
| "decisions": 4, |
| "branches": 11, |
| "actual_branch_tokens": 1353, |
| "padded_branch_tokens": 1353, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769777917019633 |
| }, |
| { |
| "step": 3067, |
| "loss": 0.1977057716858326, |
| "gradient_norm": 11.540794372558594, |
| "seconds": 9.568666854989715, |
| "decisions": 4, |
| "branches": 11, |
| "actual_branch_tokens": 1853, |
| "padded_branch_tokens": 1853, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769623853522593 |
| }, |
| { |
| "step": 3068, |
| "loss": 0.001936116958859202, |
| "gradient_norm": 0.10404492169618607, |
| "seconds": 8.810512408002978, |
| "decisions": 4, |
| "branches": 12, |
| "actual_branch_tokens": 1407, |
| "padded_branch_tokens": 1407, |
| "peak_cuda_allocated_bytes": 16776178176, |
| "lr_factor": 0.9769469739709091 |
| } |
| ], |
| "new_loss_and_gradients_all_finite": true, |
| "mean_update_seconds": 8.930427385598886, |
| "median_update_seconds": 8.907861668005353, |
| "max_update_seconds": 9.568666854989715 |
| }, |
| "limitations": [ |
| "No controlled baseline; recent100-step durations may include GUI activity.", |
| "GPU process memory from nvidia-smi includes context/cache overhead and is not the PyTorch allocated-memory cap.", |
| "Short observation verifies continued finite training and measured coexistence, not zero slowdown or long-term safety." |
| ], |
| "observer_loaded_models": false, |
| "observer_changed_processes": false, |
| "observer_accessed_credentials": false, |
| "reserved_predictions_accessed": false |
| } |
|
|
