Upload train_log.txt to L56-D1920-qwen_gdn_qwen2-e1-nh6-hd64-nvh30-A0-S512-step1-rand2b-token2B
Browse files- train_log.txt +92 -0
train_log.txt
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
NEW RUN 2025-10-08-22-20-59
|
| 2 |
+
{}
|
| 3 |
+
{'zero_allow_untested_optimizer': True, 'zero_optimization': {'stage': 1, 'contiguous_gradients': True, 'overlap_comm': True, 'allgather_partitions': True, 'reduce_scatter': True, 'allgather_bucket_size': 200000000, 'reduce_bucket_size': 200000000, 'sub_group_size': 1000000000000}, 'activation_checkpointing': {'partition_activations': False, 'cpu_checkpointing': False, 'contiguous_memory_optimization': False, 'synchronize_checkpoint_boundary': False}, 'aio': {'block_size': 1048576, 'queue_depth': 8, 'single_submit': False, 'overlap_events': True, 'thread_count': 1}, 'gradient_accumulation_steps': 1, 'train_micro_batch_size_per_gpu': 4, 'gradient_clipping': 1.0, 'bf16': {'enabled': True}}
|
| 4 |
+
0 0.038676 1.0394 0.00099986 2025-10-08 22:50:12.925455 0
|
| 5 |
+
1 0.030150 1.0306 0.00099942 2025-10-08 22:59:45.506212 1
|
| 6 |
+
2 0.028209 1.0286 0.00099867 2025-10-08 23:09:22.359172 2
|
| 7 |
+
3 0.027128 1.0275 0.00099762 2025-10-08 23:18:59.261216 3
|
| 8 |
+
4 0.026392 1.0267 0.00099626 2025-10-08 23:28:47.334830 4
|
| 9 |
+
5 0.025925 1.0263 0.00099461 2025-10-08 23:38:19.871736 5
|
| 10 |
+
6 0.025495 1.0258 0.00099265 2025-10-08 23:47:55.249558 6
|
| 11 |
+
7 0.025215 1.0255 0.00099038 2025-10-08 23:57:25.394593 7
|
| 12 |
+
8 0.024964 1.0253 0.00098782 2025-10-09 00:06:54.965994 8
|
| 13 |
+
9 0.024723 1.0250 0.00098496 2025-10-09 00:16:38.450381 9
|
| 14 |
+
10 0.024623 1.0249 0.00098180 2025-10-09 00:26:08.672460 10
|
| 15 |
+
11 0.024482 1.0248 0.00097834 2025-10-09 00:35:44.462440 11
|
| 16 |
+
12 0.024310 1.0246 0.00097458 2025-10-09 00:45:23.876874 12
|
| 17 |
+
13 0.024253 1.0245 0.00097052 2025-10-09 00:55:06.679360 13
|
| 18 |
+
14 0.024129 1.0244 0.00096618 2025-10-09 01:05:00.949446 14
|
| 19 |
+
15 0.024018 1.0243 0.00096153 2025-10-09 01:14:40.550984 15
|
| 20 |
+
16 0.023953 1.0242 0.00095660 2025-10-09 01:24:19.359306 16
|
| 21 |
+
17 0.023873 1.0242 0.00095137 2025-10-09 01:33:56.120126 17
|
| 22 |
+
18 0.023817 1.0241 0.00094586 2025-10-09 01:43:42.846741 18
|
| 23 |
+
19 0.023787 1.0241 0.00094005 2025-10-09 01:53:35.984485 19
|
| 24 |
+
20 0.023710 1.0240 0.00093397 2025-10-09 02:03:11.592462 20
|
| 25 |
+
21 0.023749 1.0240 0.00092760 2025-10-09 02:12:50.429359 21
|
| 26 |
+
22 0.023641 1.0239 0.00092094 2025-10-09 02:22:40.196286 22
|
| 27 |
+
23 0.023569 1.0238 0.00091401 2025-10-09 02:32:22.413915 23
|
| 28 |
+
24 0.023562 1.0238 0.00090680 2025-10-09 02:42:14.694074 24
|
| 29 |
+
25 0.023508 1.0238 0.00089931 2025-10-09 02:51:50.624703 25
|
| 30 |
+
26 0.023448 1.0237 0.00089155 2025-10-09 03:01:28.087448 26
|
| 31 |
+
27 0.023456 1.0237 0.00088352 2025-10-09 03:11:05.325197 27
|
| 32 |
+
28 0.023405 1.0237 0.00087522 2025-10-09 03:20:42.477645 28
|
| 33 |
+
29 0.023373 1.0236 0.00086666 2025-10-09 03:30:32.603740 29
|
| 34 |
+
30 0.023372 1.0236 0.00085783 2025-10-09 03:40:04.621355 30
|
| 35 |
+
31 0.023282 1.0236 0.00084875 2025-10-09 03:49:42.685071 31
|
| 36 |
+
32 0.023324 1.0236 0.00083940 2025-10-09 03:59:18.233877 32
|
| 37 |
+
33 0.023212 1.0235 0.00082980 2025-10-09 04:08:53.676655 33
|
| 38 |
+
34 0.023266 1.0235 0.00081995 2025-10-09 04:18:39.209565 34
|
| 39 |
+
35 0.023209 1.0235 0.00080985 2025-10-09 04:28:14.905327 35
|
| 40 |
+
36 0.023207 1.0235 0.00079951 2025-10-09 04:37:48.753121 36
|
| 41 |
+
37 0.023155 1.0234 0.00078892 2025-10-09 04:47:25.793392 37
|
| 42 |
+
38 0.023163 1.0234 0.00077809 2025-10-09 04:57:01.747924 38
|
| 43 |
+
39 0.023119 1.0234 0.00076703 2025-10-09 05:06:50.269126 39
|
| 44 |
+
40 0.023091 1.0234 0.00075573 2025-10-09 05:16:25.317023 40
|
| 45 |
+
41 0.023079 1.0233 0.00074421 2025-10-09 05:26:01.753882 41
|
| 46 |
+
42 0.023015 1.0233 0.00073246 2025-10-09 05:35:38.249444 42
|
| 47 |
+
43 0.023084 1.0234 0.00072048 2025-10-09 05:45:18.540850 43
|
| 48 |
+
44 0.022972 1.0232 0.00070829 2025-10-09 05:55:06.130397 44
|
| 49 |
+
45 0.023000 1.0233 0.00069589 2025-10-09 06:04:43.919403 45
|
| 50 |
+
46 0.022996 1.0233 0.00068327 2025-10-09 06:14:25.132540 46
|
| 51 |
+
47 0.022945 1.0232 0.00067045 2025-10-09 06:24:06.442154 47
|
| 52 |
+
48 0.022954 1.0232 0.00065743 2025-10-09 06:33:43.343356 48
|
| 53 |
+
49 0.022999 1.0233 0.00064420 2025-10-09 06:43:33.187019 49
|
| 54 |
+
50 0.022877 1.0231 0.00063078 2025-10-09 06:53:07.178664 50
|
| 55 |
+
51 0.022862 1.0231 0.00061718 2025-10-09 07:02:38.383074 51
|
| 56 |
+
52 0.022860 1.0231 0.00060338 2025-10-09 07:12:13.516101 52
|
| 57 |
+
53 0.022844 1.0231 0.00058940 2025-10-09 07:21:47.218802 53
|
| 58 |
+
54 0.022819 1.0231 0.00057525 2025-10-09 07:31:31.698811 54
|
| 59 |
+
55 0.022831 1.0231 0.00056092 2025-10-09 07:41:04.739960 55
|
| 60 |
+
56 0.022826 1.0231 0.00054642 2025-10-09 07:50:39.112172 56
|
| 61 |
+
57 0.022767 1.0230 0.00053176 2025-10-09 08:00:16.526788 57
|
| 62 |
+
58 0.022778 1.0230 0.00051693 2025-10-09 08:09:55.021485 58
|
| 63 |
+
59 0.022692 1.0230 0.00050195 2025-10-09 08:19:42.691967 59
|
| 64 |
+
60 0.022770 1.0230 0.00048682 2025-10-09 08:29:19.797085 60
|
| 65 |
+
61 0.022748 1.0230 0.00047155 2025-10-09 08:38:55.509870 61
|
| 66 |
+
62 0.022704 1.0230 0.00045613 2025-10-09 08:48:33.828990 62
|
| 67 |
+
63 0.022659 1.0229 0.00044057 2025-10-09 08:58:15.838011 63
|
| 68 |
+
64 0.022714 1.0230 0.00042488 2025-10-09 09:07:58.914572 64
|
| 69 |
+
65 0.022623 1.0229 0.00040907 2025-10-09 09:17:35.610780 65
|
| 70 |
+
66 0.022654 1.0229 0.00039313 2025-10-09 09:27:17.916423 66
|
| 71 |
+
67 0.022680 1.0229 0.00037708 2025-10-09 09:36:52.202673 67
|
| 72 |
+
68 0.022628 1.0229 0.00036091 2025-10-09 09:46:32.035349 68
|
| 73 |
+
69 0.022611 1.0229 0.00034463 2025-10-09 09:56:23.507004 69
|
| 74 |
+
70 0.022574 1.0228 0.00032825 2025-10-09 10:05:58.789601 70
|
| 75 |
+
71 0.022665 1.0229 0.00031178 2025-10-09 10:15:39.950298 71
|
| 76 |
+
72 0.022508 1.0228 0.00029521 2025-10-09 10:25:20.161186 72
|
| 77 |
+
73 0.022521 1.0228 0.00027855 2025-10-09 10:34:57.935531 73
|
| 78 |
+
74 0.022547 1.0228 0.00026181 2025-10-09 10:44:50.541680 74
|
| 79 |
+
75 0.022542 1.0228 0.00024500 2025-10-09 10:54:29.665370 75
|
| 80 |
+
76 0.022452 1.0227 0.00022811 2025-10-09 11:04:18.983033 76
|
| 81 |
+
77 0.022479 1.0227 0.00021115 2025-10-09 11:14:03.632076 77
|
| 82 |
+
78 0.022433 1.0227 0.00019414 2025-10-09 11:23:43.466159 78
|
| 83 |
+
79 0.022490 1.0227 0.00017706 2025-10-09 11:33:35.091928 79
|
| 84 |
+
80 0.022436 1.0227 0.00015994 2025-10-09 11:43:25.921993 80
|
| 85 |
+
81 0.022395 1.0226 0.00014277 2025-10-09 11:53:21.005149 81
|
| 86 |
+
82 0.022402 1.0227 0.00012556 2025-10-09 12:03:10.995048 82
|
| 87 |
+
83 0.022395 1.0226 0.00010831 2025-10-09 12:13:00.847028 83
|
| 88 |
+
84 0.022348 1.0226 0.00009103 2025-10-09 12:22:57.006329 84
|
| 89 |
+
85 0.022379 1.0226 0.00007373 2025-10-09 12:32:36.634279 85
|
| 90 |
+
86 0.022354 1.0226 0.00005641 2025-10-09 12:42:10.906295 86
|
| 91 |
+
87 0.022317 1.0226 0.00003907 2025-10-09 12:51:41.145553 87
|
| 92 |
+
88 0.022350 1.0226 0.00002172 2025-10-09 13:01:27.144282 88
|