Preserve crop experiment runs/crop384-20260909-instance-50427963/control

runs/crop384-20260909-instance-50427963/control/budget.json

@@ -1,20 +1,26 @@
 {
   "experiment": "sutrack",
-  "instance_id": 50427963,
-  "profile": "sutrack-crop384",
-  "workspace": "/workspace/balltrack-remote/sutrack-crop384-50427963",
+  "instance_id": 50514292,
+  "profile": "sutrack-resume-20260910",
+  "workspace": "/workspace/balltrack-remote/sutrack-resume-20260910-50514292",
   "cap_usd": 15,
-  "storage_budget_bytes": 15600000000,
-  "rented_at": 1788985983.344,
-  "hourly_rate": 0.5781481481481481,
+  "storage_budget_bytes": 96693887914,
+  "rented_at": 1789063925.8874364,
+  "hourly_rate": 0.5477777777777778,
   "transfer_reserve_usd": 3,
-  "hard_deadline": 1789060404.6764793,
-  "soft_deadline": 1789055004.6764793,
-  "eval_deadline": 1789059204.6764793,
+  "hard_deadline": 1789142489.9847994,
+  "soft_deadline": 1789137089.9847994,
+  "eval_deadline": 1789141289.9847994,
   "training_config": "configs/sutrack-crop384.json",
   "training_config_sha256": "202764be6e164f73056bfdd56bad9bfc4d2bc8cba2b191069c86b7ee6377df7d",
   "output": "artifacts/crop384/sutrack-20260909",
   "hf_repo": "bambezius/sutrack-soccernet-ball",
   "hf_prefix": "runs/crop384-20260909-instance-50427963",
-  "dataset_revision": "26512d1339d1b4655fff72c556e415c2d2cda967"
+  "dataset_revision": "26512d1339d1b4655fff72c556e415c2d2cda967",
+  "resume_source": {
+    "revision": "15e55dd79bd9091e51063f4ccef9e2109b367794",
+    "prefix": "runs/crop384-20260909-instance-50427963/final",
+    "epoch": 4,
+    "bad_checks": 2
+  }
 }

runs/crop384-20260909-instance-50427963/control/child.json

@@ -1,4 +1,4 @@
 {
   "stage": "test",
-  "pid": 2989
+  "pid": 2832
 }

runs/crop384-20260909-instance-50427963/control/status.json

@@ -1,7 +1,7 @@
 {
   "state": "complete",
-  "training": "storage_budget_stopped",
+  "training": "early_stopped",
   "evaluation": "completed",
-  "instance_id": 50427963,
+  "instance_id": 50514292,
   "destroy_requested": true
 }

runs/crop384-20260909-instance-50427963/control/test.log

@@ -1,17 +1,17 @@
-/workspace/balltrack-remote/sutrack-crop384-50427963/repo/.venv/lib/python3.11/site-packages/clip/clip.py:6: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
+/workspace/balltrack-remote/sutrack-resume-20260910-50514292/repo/.venv/lib/python3.11/site-packages/clip/clip.py:6: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
   from pkg_resources import packaging
 SUTrack fused SDPA enabled on 40 attention modules
 wandb: Currently logged in as: anthonyceponis (mvp-project) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
 wandb: setting up run 9kcrmh6u
 wandb: Tracking run with wandb version 0.23.1
-wandb: Run data is saved locally in /workspace/balltrack-remote/sutrack-crop384-50427963/repo/wandb/run-20260910_015008-9kcrmh6u
+wandb: Run data is saved locally in /workspace/balltrack-remote/sutrack-resume-20260910-50514292/repo/wandb/run-20260910_193333-9kcrmh6u
 wandb: Run `wandb offline` to turn off syncing.
 wandb: Resuming run sutrack-20260909
 wandb: ⭐️ View project at https://wandb.ai/anthonyceponis-university-of-cambridge/football-ball-tracking
 wandb: 🚀 View run at https://wandb.ai/anthonyceponis-university-of-cambridge/football-ball-tracking/runs/9kcrmh6u
 wandb: uploading summary; updating run metadata
-wandb: uploading requirements.txt; uploading wandb-summary.json
-wandb: uploading history steps 938-938, summary
+wandb: uploading wandb-metadata.json; uploading requirements.txt; uploading wandb-summary.json
+wandb: uploading history steps 1172-1172, summary
 wandb: 
 wandb: Run history:
 wandb:  test/center_error/count ▁
@@ -27,20 +27,20 @@ wandb:            test/eligible ▁
 wandb:                      +20 ...
 wandb: 
 wandb: Run summary:
-wandb:               bad_checks 2
+wandb:               bad_checks 3
 wandb:                     best 390.26756
-wandb:                    epoch 4
+wandb:                    epoch 5
 wandb:          partial_dataset False
-wandb:         selection_metric 751.70663
+wandb:         selection_metric 651.5754
 wandb:                    smoke False
 wandb:  test/center_error/count 9122
-wandb:   test/center_error/mean 357.16883
-wandb: test/center_error/median 2.87772
+wandb:   test/center_error/mean 354.70591
+wandb: test/center_error/median 2.8853
 wandb:    test/center_error/p95 2482.53573
 wandb:                     +138 ...
 wandb: 
 wandb: 🚀 View run sutrack-20260909 at: https://wandb.ai/anthonyceponis-university-of-cambridge/football-ball-tracking/runs/9kcrmh6u
 wandb: ⭐️ View project at: https://wandb.ai/anthonyceponis-university-of-cambridge/football-ball-tracking
 wandb: Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
-wandb: Find logs at: ./wandb/run-20260910_015008-9kcrmh6u/logs
-{"frames": 11965, "eligible": 9122, "in_crop": 7374, "unlabelled": 2843, "crop_misses": 1748, "crop_coverage": 0.8083753562815172, "center_error": {"count": 9122, "mean": 357.16882506490236, "median": 2.877716064453125, "p95": 2482.5357299804523}, "radius_error": {"count": 9122, "mean": 2.2424405797796534, "median": 1.0859375, "p95": 12.515625}, "within_5px": 0.6454724841043631, "within_20px": 0.7757070817803113, "losses": {"loss": 1.3118762254865959, "location": 0.4665638133856209, "l1": 0.01204897800443714, "giou": 0.39253376150857827, "center": 0.0044754733278232735, "radius": 0.007573504679842663}, "data_seconds": 21.61831338392949, "compute_seconds": 155.1539453319965, "seconds": 176.82690785399973, "frames_per_second": 67.66504117053901, "steps": 1498, "gpu_max_allocated_bytes": 6318884864, "gpu_max_reserved_bytes": 6989807616, "checkpoint_epoch": 2, "state": "completed"}
+wandb: Find logs at: ./wandb/run-20260910_193333-9kcrmh6u/logs
+{"frames": 11965, "eligible": 9122, "in_crop": 7374, "unlabelled": 2843, "crop_misses": 1748, "crop_coverage": 0.8083753562815172, "center_error": {"count": 9122, "mean": 354.7059118448118, "median": 2.8853044509887695, "p95": 2482.5357299804523}, "radius_error": {"count": 9122, "mean": 2.269953049632756, "median": 1.078125, "p95": 12.515625}, "within_5px": 0.6460206095154571, "within_20px": 0.7752685814514361, "losses": {"loss": 1.3117444160493505, "location": 0.46589546384665764, "l1": 0.01214504628883284, "giou": 0.3925618612891722, "center": 0.004491665123334892, "radius": 0.00765338117421442}, "data_seconds": 14.15910452697426, "compute_seconds": 97.58808584976941, "seconds": 111.78405178710818, "frames_per_second": 107.03673564085193, "steps": 1498, "gpu_max_allocated_bytes": 6318884864, "gpu_max_reserved_bytes": 6989807616, "checkpoint_epoch": 2, "state": "completed"}

runs/crop384-20260909-instance-50427963/control/transfer-manifest.json

@@ -1,22 +1,22 @@
 {
   "budget.json": {
-    "sha256": "1199961a5c128bceb21dac3f3b44c413d87bd843dc77986d4b0328c168c825d2",
-    "bytes": 802
+    "sha256": "c4c537d9fc7bb6dfac2ad11c6d744a24a798f17d37e03971092e317fbc375efa",
+    "bytes": 1007
   },
   "train.log": {
-    "sha256": "c0f92ebdfd2e42e7c069b05595d20ac818054f1935f4037ef7b67eff53b073c6",
-    "bytes": 1028431
+    "sha256": "ddfd52fec1d0e6812b94f0d3489e63e20ee1e529c3a4edbcbc8a3eeb426bf77a",
+    "bytes": 220692
   },
   "child.json": {
-    "sha256": "100816b04a3e984d04f236b8b82e8f875ebfd7bbab8ab33568421d7c02d40ba0",
+    "sha256": "cb91a3fba7cc98e12225621692a38131e4e623a63edcd7b25a6ea249ff339ac6",
     "bytes": 37
   },
   "status.json": {
-    "sha256": "011a784083e7d47ab903ae380a452b90228b0b6cd4d2ab4233157e7060235f71",
-    "bytes": 151
+    "sha256": "058349a1d5e0697f8478c017bc88ddb0cbffcad906bb83c11babc29b5d22bfc6",
+    "bytes": 142
   },
   "test.log": {
-    "sha256": "0cdf22ae282560ddd3e124115ae2c1ef88256220461957459e5f0269fd7ff6d4",
-    "bytes": 3417
+    "sha256": "237d1ad1f5d5635e6471bab1ffec4c43fb4d806d927a70a94068d33edb3c3cea",
+    "bytes": 3461
   }
 }