Upload train_log.txt to L56-D1920-qwen_gdn_qwen2-e1-nh6-hd64-nvh30-A0-S512-step1-rand2b-nolearn-o-bsz16x8-token1B
Browse files- train_log.txt +91 -0
train_log.txt
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
NEW RUN 2025-10-12-18-14-38
|
| 2 |
+
{}
|
| 3 |
+
{'zero_allow_untested_optimizer': True, 'zero_optimization': {'stage': 1, 'contiguous_gradients': True, 'overlap_comm': True, 'allgather_partitions': True, 'reduce_scatter': True, 'allgather_bucket_size': 200000000, 'reduce_bucket_size': 200000000, 'sub_group_size': 1000000000000}, 'activation_checkpointing': {'partition_activations': False, 'cpu_checkpointing': False, 'contiguous_memory_optimization': False, 'synchronize_checkpoint_boundary': False}, 'aio': {'block_size': 1048576, 'queue_depth': 8, 'single_submit': False, 'overlap_events': True, 'thread_count': 1}, 'gradient_accumulation_steps': 8, 'train_micro_batch_size_per_gpu': 16, 'gradient_clipping': 1.0, 'bf16': {'enabled': True}}
|
| 4 |
+
0 0.088185 1.0922 0.00078220 2025-10-12 18:24:37.466070 0
|
| 5 |
+
1 0.067483 1.0698 0.00099992 2025-10-12 18:28:38.392342 1
|
| 6 |
+
2 0.057017 1.0587 0.00099952 2025-10-12 18:32:30.735586 2
|
| 7 |
+
3 0.053236 1.0547 0.00099880 2025-10-12 18:36:24.519734 3
|
| 8 |
+
4 0.050668 1.0520 0.00099776 2025-10-12 18:40:25.384391 4
|
| 9 |
+
5 0.048811 1.0500 0.00099640 2025-10-12 18:44:18.284755 5
|
| 10 |
+
6 0.047172 1.0483 0.00099472 2025-10-12 18:48:11.076374 6
|
| 11 |
+
7 0.045947 1.0470 0.00099272 2025-10-12 18:52:04.829686 7
|
| 12 |
+
8 0.044851 1.0459 0.00099039 2025-10-12 18:55:57.888876 8
|
| 13 |
+
9 0.043912 1.0449 0.00098775 2025-10-12 19:00:00.533446 9
|
| 14 |
+
10 0.043137 1.0441 0.00098479 2025-10-12 19:03:53.334902 10
|
| 15 |
+
11 0.042417 1.0433 0.00098151 2025-10-12 19:07:47.854476 11
|
| 16 |
+
12 0.041706 1.0426 0.00097791 2025-10-12 19:11:41.035394 12
|
| 17 |
+
13 0.041123 1.0420 0.00097400 2025-10-12 19:15:35.657656 13
|
| 18 |
+
14 0.040560 1.0414 0.00096977 2025-10-12 19:19:36.910027 14
|
| 19 |
+
15 0.039951 1.0408 0.00096524 2025-10-12 19:23:30.610162 15
|
| 20 |
+
16 0.039517 1.0403 0.00096039 2025-10-12 19:27:23.805968 16
|
| 21 |
+
17 0.039054 1.0398 0.00095523 2025-10-12 19:31:18.661836 17
|
| 22 |
+
18 0.038640 1.0394 0.00094976 2025-10-12 19:35:11.907078 18
|
| 23 |
+
19 0.038318 1.0391 0.00094399 2025-10-12 19:39:15.685737 19
|
| 24 |
+
20 0.037963 1.0387 0.00093791 2025-10-12 19:43:08.754221 20
|
| 25 |
+
21 0.037658 1.0384 0.00093153 2025-10-12 19:47:01.799881 21
|
| 26 |
+
22 0.037253 1.0380 0.00092485 2025-10-12 19:50:55.759101 22
|
| 27 |
+
23 0.036941 1.0376 0.00091787 2025-10-12 19:54:49.039343 23
|
| 28 |
+
24 0.036736 1.0374 0.00091060 2025-10-12 19:58:52.690273 24
|
| 29 |
+
25 0.036360 1.0370 0.00090303 2025-10-12 20:02:45.570820 25
|
| 30 |
+
26 0.036218 1.0369 0.00089518 2025-10-12 20:06:39.283475 26
|
| 31 |
+
27 0.035966 1.0366 0.00088703 2025-10-12 20:10:32.583625 27
|
| 32 |
+
28 0.035722 1.0364 0.00087860 2025-10-12 20:14:26.258040 28
|
| 33 |
+
29 0.035559 1.0362 0.00086988 2025-10-12 20:18:28.101747 29
|
| 34 |
+
30 0.035290 1.0359 0.00086089 2025-10-12 20:22:22.473810 30
|
| 35 |
+
31 0.035084 1.0357 0.00085161 2025-10-12 20:26:15.646385 31
|
| 36 |
+
32 0.034956 1.0356 0.00084207 2025-10-12 20:30:10.617841 32
|
| 37 |
+
33 0.034754 1.0354 0.00083225 2025-10-12 20:34:03.442247 33
|
| 38 |
+
34 0.034605 1.0352 0.00082216 2025-10-12 20:38:06.222611 34
|
| 39 |
+
35 0.034420 1.0350 0.00081181 2025-10-12 20:41:58.695683 35
|
| 40 |
+
36 0.034309 1.0349 0.00080120 2025-10-12 20:45:52.902556 36
|
| 41 |
+
37 0.034189 1.0348 0.00079033 2025-10-12 20:49:46.234191 37
|
| 42 |
+
38 0.034013 1.0346 0.00077921 2025-10-12 20:53:40.136697 38
|
| 43 |
+
39 0.033854 1.0344 0.00076784 2025-10-12 20:57:41.434580 39
|
| 44 |
+
40 0.033729 1.0343 0.00075621 2025-10-12 21:01:35.280189 40
|
| 45 |
+
41 0.033612 1.0342 0.00074435 2025-10-12 21:05:27.913848 41
|
| 46 |
+
42 0.033542 1.0341 0.00073225 2025-10-12 21:09:21.615176 42
|
| 47 |
+
43 0.033351 1.0339 0.00071991 2025-10-12 21:13:13.916389 43
|
| 48 |
+
44 0.033330 1.0339 0.00070734 2025-10-12 21:17:16.055980 44
|
| 49 |
+
45 0.033142 1.0337 0.00069454 2025-10-12 21:21:08.810244 45
|
| 50 |
+
46 0.033068 1.0336 0.00068152 2025-10-12 21:25:02.072593 46
|
| 51 |
+
47 0.033015 1.0336 0.00066828 2025-10-12 21:28:54.306996 47
|
| 52 |
+
48 0.032923 1.0335 0.00065483 2025-10-12 21:32:48.056751 48
|
| 53 |
+
49 0.032800 1.0333 0.00064116 2025-10-12 21:36:49.941006 49
|
| 54 |
+
50 0.032666 1.0332 0.00062730 2025-10-12 21:40:44.307345 50
|
| 55 |
+
51 0.032616 1.0332 0.00061323 2025-10-12 21:44:37.195076 51
|
| 56 |
+
52 0.032536 1.0331 0.00059896 2025-10-12 21:48:31.894675 52
|
| 57 |
+
53 0.032449 1.0330 0.00058450 2025-10-12 21:52:24.991386 53
|
| 58 |
+
54 0.032363 1.0329 0.00056985 2025-10-12 21:56:27.720707 54
|
| 59 |
+
55 0.032380 1.0329 0.00055503 2025-10-12 22:00:20.476952 55
|
| 60 |
+
56 0.032247 1.0328 0.00054002 2025-10-12 22:04:14.793889 56
|
| 61 |
+
57 0.032231 1.0328 0.00052484 2025-10-12 22:08:07.887558 57
|
| 62 |
+
58 0.032050 1.0326 0.00050950 2025-10-12 22:12:01.724034 58
|
| 63 |
+
59 0.032101 1.0326 0.00049399 2025-10-12 22:16:03.979487 59
|
| 64 |
+
60 0.032043 1.0326 0.00047832 2025-10-12 22:19:57.956574 60
|
| 65 |
+
61 0.031954 1.0325 0.00046250 2025-10-12 22:23:51.966872 61
|
| 66 |
+
62 0.031851 1.0324 0.00044653 2025-10-12 22:27:46.826122 62
|
| 67 |
+
63 0.031908 1.0324 0.00043042 2025-10-12 22:31:41.333706 63
|
| 68 |
+
64 0.031767 1.0323 0.00041418 2025-10-12 22:35:44.251780 64
|
| 69 |
+
65 0.031779 1.0323 0.00039780 2025-10-12 22:39:38.871052 65
|
| 70 |
+
66 0.031769 1.0323 0.00038130 2025-10-12 22:43:32.248311 66
|
| 71 |
+
67 0.031724 1.0322 0.00036467 2025-10-12 22:47:27.407618 67
|
| 72 |
+
68 0.031613 1.0321 0.00034793 2025-10-12 22:51:21.709044 68
|
| 73 |
+
69 0.031604 1.0321 0.00033108 2025-10-12 22:55:26.244175 69
|
| 74 |
+
70 0.031665 1.0322 0.00031413 2025-10-12 22:59:20.409124 70
|
| 75 |
+
71 0.031451 1.0320 0.00029707 2025-10-12 23:03:14.910649 71
|
| 76 |
+
72 0.031458 1.0320 0.00027992 2025-10-12 23:07:08.488378 72
|
| 77 |
+
73 0.031506 1.0320 0.00026269 2025-10-12 23:11:03.840030 73
|
| 78 |
+
74 0.031475 1.0320 0.00024537 2025-10-12 23:15:06.344406 74
|
| 79 |
+
75 0.031369 1.0319 0.00022798 2025-10-12 23:19:01.048131 75
|
| 80 |
+
76 0.031338 1.0318 0.00021051 2025-10-12 23:22:54.496329 76
|
| 81 |
+
77 0.031373 1.0319 0.00019298 2025-10-12 23:26:49.700414 77
|
| 82 |
+
78 0.031358 1.0319 0.00017539 2025-10-12 23:30:43.033555 78
|
| 83 |
+
79 0.031310 1.0318 0.00015774 2025-10-12 23:34:46.771486 79
|
| 84 |
+
80 0.031293 1.0318 0.00014005 2025-10-12 23:38:40.764542 80
|
| 85 |
+
81 0.031283 1.0318 0.00012232 2025-10-12 23:42:35.469801 81
|
| 86 |
+
82 0.031257 1.0318 0.00010454 2025-10-12 23:46:29.163932 82
|
| 87 |
+
83 0.031270 1.0318 0.00008674 2025-10-12 23:50:24.248419 83
|
| 88 |
+
84 0.031299 1.0318 0.00006891 2025-10-12 23:54:25.770080 84
|
| 89 |
+
85 0.031219 1.0317 0.00005107 2025-10-12 23:58:19.924574 85
|
| 90 |
+
86 0.031255 1.0317 0.00003321 2025-10-13 00:02:13.334868 86
|
| 91 |
+
87 0.031277 1.0318 0.00001534 2025-10-13 00:06:16.958393 87
|