diff --git "a/log.txt" "b/log.txt" --- "a/log.txt" +++ "b/log.txt" @@ -66,7 +66,7 @@ Running 1 job }, "unload_text_encoder": false, "cache_text_embeddings": false, - "lr": 0.0001, + "lr": 0.0002, "ema_config": { "use_ema": false, "ema_decay": 0.99 @@ -89,12 +89,12 @@ Running 1 job "name_or_path": "ai-toolkit/Wan2.2-T2V-A14B-Diffusers-bf16", "quantize": false, "qtype": "qfloat8", - "quantize_te": false, + "quantize_te": true, "qtype_te": "qfloat8", "arch": "wan22_14b:t2v", "low_vram": false, "model_kwargs": { - "train_high_noise": false, + "train_high_noise": true, "train_low_noise": true }, "layer_offloading": false, @@ -108,61 +108,94 @@ Running 1 job "height": 1024, "samples": [ { - "prompt": "l4n4lux, woman with red hair, playing chess at the park, bomb going off in the background" + "prompt": "l4n4lux" } ], "neg": "", "seed": 42, "walk_seed": true, "guidance_scale": 4, - "sample_steps": 12, + "sample_steps": 25, "num_frames": 1, "fps": 1 } } Using SQLite database at /root/lab/ai-toolkit/aitk_db.db -Job ID: "765fbd95-c627-460c-a232-147504194a44" +Job ID: "e805981e-3cd8-4a28-9d3a-f4b28b40907a" ############################################# -# Running job: lana-lora-wan2.2 +# Running job: lana_lora_wan2.2 ############################################# Running 1 process Loading Wan model Loading transformer 1 - Loading checkpoint shards: 0%| | 0/3 [00:00 - File "/root/lab/ai-toolkit/run.py", line 120, in - main()main() - File "/root/lab/ai-toolkit/run.py", line 108, in main - File "/root/lab/ai-toolkit/run.py", line 108, in main - raise eraise e - File "/root/lab/ai-toolkit/run.py", line 96, in main - File "/root/lab/ai-toolkit/run.py", line 96, in main - job.run()job.run() + 96%|###################################################################################################8 | 24/25 [00:16<00:00, 1.50it/s] 96%|###################################################################################################8 | 24/25 [00:16<00:00, 1.50it/s] - File "/root/lab/ai-toolkit/jobs/ExtensionJob.py", line 22, in run - File "/root/lab/ai-toolkit/jobs/ExtensionJob.py", line 22, in run - process.run()process.run() - File "/root/lab/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run - File "/root/lab/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run - self.end_step_hook()self.end_step_hook() - File "/root/lab/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook - File "/root/lab/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook - self.maybe_stop()self.maybe_stop() + 100%|########################################################################################################| 25/25 [00:16<00:00, 1.50it/s] 100%|########################################################################################################| 25/25 [00:16<00:00, 1.50it/s] 100%|########################################################################################################| 25/25 [00:16<00:00, 1.49it/s] 100%|########################################################################################################| 25/25 [00:16<00:00, 1.49it/s] - File "/root/lab/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop - File "/root/lab/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop - raise Exception("Job stopped")raise Exception("Job stopped") -ExceptionException: : Job stoppedJob stopped - lana-lora-wan2.2: 81%|##########################################7 | 4035/5000 [51:25<38:37, 2.40s/it, lr: 1.0e-04 loss: 6.642e-02] lana-lora-wan2.2: 81%|##########################################7 | 4035/5000 [51:25<38:37, 2.40s/it, lr: 1.0e-04 loss: 6.642e-02] + Generating Images: 100%|#######################################################################################| 1/1 [00:17<00:00, 17.43s/it] Generating Images: 100%|#######################################################################################| 1/1 [00:17<00:00, 17.43s/it] +  lana_lora_wan2.2: 5%|##6 | 250/5000 [10:25<3:18:13, 2.50s/it, lr: 2.0e-04 loss: 5.012e-02] lana_lora_wan2.2: 5%|##6 | 250/5000 [10:25<3:18:13, 2.50s/it, lr: 2.0e-04 loss: 5.012e-02] lana_lora_wan2.2: 5%|##6 | 251/5000 [10:29<3:18:33, 2.51s/it, lr: 2.0e-04 loss: 4.005e-02] lana_lora_wan2.2: 5%|##6 | 251/5000 [10:29<3:18:33, 2.51s/it, lr: 2.0e-04 loss: 4.005e-02] lana_lora_wan2.2: 5%|##6 | 252/5000 [10:33<3:18:52, 2.51s/it, lr: 2.0e-04 loss: 3.868e-02] lana_lora_wan2.2: 5%|##6 | 252/5000 [10:33<3:18:52, 2.51s/it, lr: 2.0e-04 loss: 3.868e-02] lana_lora_wan2.2: 5%|##6 | 253/5000 [10:35<3:18:44, 2.51s/it, lr: 2.0e-04 loss: 4.959e-02] lana_lora_wan2.2: 5%|##6 | 253/5000 [10:35<3:18:44, 2.51s/it, lr: 2.0e-04 loss: 4.959e-02] lana_lora_wan2.2: 5%|##6 | 254/5000 [10:39<3:19:03, 2.52s/it, lr: 2.0e-04 loss: 5.056e-02] lana_lora_wan2.2: 5%|##6 | 254/5000 [10:39<3:19:03, 2.52s/it, lr: 2.0e-04 loss: 5.056e-02] lana_lora_wan2.2: 5%|##6 | 255/5000 [10:40<3:18:45, 2.51s/it, lr: 2.0e-04 loss: 7.666e-02] lana_lora_wan2.2: 5%|##6 | 255/5000 [10:40<3:18:45, 2.51s/it, lr: 2.0e-04 loss: 7.666e-02] lana_lora_wan2.2: 5%|##6 | 256/5000 [10:43<3:18:37, 2.51s/it, lr: 2.0e-04 loss: 5.034e-02] lana_lora_wan2.2: 5%|##6 | 256/5000 [10:43<3:18:37, 2.51s/it, lr: 2.0e-04 loss: 5.034e-02] lana_lora_wan2.2: 5%|##6 | 257/5000 [10:45<3:18:28, 2.51s/it, lr: 2.0e-04 loss: 4.239e-02] lana_lora_wan2.2: 5%|##6 | 257/5000 [10:45<3:18:28, 2.51s/it, lr: 2.0e-04 loss: 4.239e-02] lana_lora_wan2.2: 5%|##6 | 258/5000 [10:46<3:18:05, 2.51s/it, lr: 2.0e-04 loss: 2.135e-01] lana_lora_wan2.2: 5%|##6 | 258/5000 [10:46<3:18:05, 2.51s/it, lr: 2.0e-04 loss: 2.135e-01] lana_lora_wan2.2: 5%|##6 | 259/5000 [10:48<3:17:43, 2.50s/it, lr: 2.0e-04 loss: 6.766e-02] lana_lora_wan2.2: 5%|##6 | 259/5000 [10:48<3:17:43, 2.50s/it, lr: 2.0e-04 loss: 6.766e-02] lana_lora_wan2.2: 5%|##7 | 260/5000 [10:51<3:18:02, 2.51s/it, lr: 2.0e-04 loss: 5.707e-02] lana_lora_wan2.2: 5%|##7 | 260/5000 [10:51<3:18:02, 2.51s/it, lr: 2.0e-04 loss: 5.707e-02] lana_lora_wan2.2: 5%|##7 | 261/5000 [10:53<3:17:44, 2.50s/it, lr: 2.0e-04 loss: 7.334e-02] lana_lora_wan2.2: 5%|##7 | 261/5000 [10:53<3:17:44, 2.50s/it, lr: 2.0e-04 loss: 7.334e-02] lana_lora_wan2.2: 5%|##7 | 262/5000 [10:57<3:18:03, 2.51s/it, lr: 2.0e-04 loss: 5.068e-02] lana_lora_wan2.2: 5%|##7 | 262/5000 [10:57<3:18:03, 2.51s/it, lr: 2.0e-04 loss: 5.068e-02] lana_lora_wan2.2: 5%|##7 | 263/5000 [10:58<3:17:39, 2.50s/it, lr: 2.0e-04 loss: 9.065e-02] lana_lora_wan2.2: 5%|##7 | 263/5000 [10:58<3:17:39, 2.50s/it, lr: 2.0e-04 loss: 9.065e-02] lana_lora_wan2.2: 5%|##7 | 264/5000 [10:59<3:17:17, 2.50s/it, lr: 2.0e-04 loss: 6.212e-02] lana_lora_wan2.2: 5%|##7 | 264/5000 [10:59<3:17:17, 2.50s/it, lr: 2.0e-04 loss: 6.212e-02] lana_lora_wan2.2: 5%|##7 | 265/5000 [11:03<3:17:36, 2.50s/it, lr: 2.0e-04 loss: 6.106e-02] lana_lora_wan2.2: 5%|##7 | 265/5000 [11:03<3:17:36, 2.50s/it, lr: 2.0e-04 loss: 6.106e-02] lana_lora_wan2.2: 5%|##7 | 266/5000 [11:04<3:17:12, 2.50s/it, lr: 2.0e-04 loss: 4.350e-02] lana_lora_wan2.2: 5%|##7 | 266/5000 [11:04<3:17:12, 2.50s/it, lr: 2.0e-04 loss: 4.350e-02] lana_lora_wan2.2: 5%|##7 | 267/5000 [11:08<3:17:35, 2.50s/it, lr: 2.0e-04 loss: 4.135e-02] lana_lora_wan2.2: 5%|##7 | 267/5000 [11:08<3:17:35, 2.50s/it, lr: 2.0e-04 loss: 4.135e-02] lana_lora_wan2.2: 5%|##7 | 268/5000 [11:12<3:17:54, 2.51s/it, lr: 2.0e-04 loss: 1.573e-01] lana_lora_wan2.2: 5%|##7 | 268/5000 [11:12<3:17:54, 2.51s/it, lr: 2.0e-04 loss: 1.573e-01] lana_lora_wan2.2: 5%|##7 | 269/5000 [11:13<3:17:31, 2.51s/it, lr: 2.0e-04 loss: 7.498e-02] lana_lora_wan2.2: 5%|##7 | 269/5000 [11:13<3:17:31, 2.51s/it, lr: 2.0e-04 loss: 7.498e-02] lana_lora_wan2.2: 5%|##8 | 270/5000 [11:17<3:17:49, 2.51s/it, lr: 2.0e-04 loss: 3.909e-02] lana_lora_wan2.2: 5%|##8 | 270/5000 [11:17<3:17:49, 2.51s/it, lr: 2.0e-04 loss: 3.909e-02] lana_lora_wan2.2: 5%|##8 | 271/5000 [11:19<3:17:42, 2.51s/it, lr: 2.0e-04 loss: 4.902e-02] lana_lora_wan2.2: 5%|##8 | 271/5000 [11:19<3:17:42, 2.51s/it, lr: 2.0e-04 loss: 4.902e-02] lana_lora_wan2.2: 5%|##8 | 272/5000 [11:23<3:18:00, 2.51s/it, lr: 2.0e-04 loss: 2.974e-02] lana_lora_wan2.2: 5%|##8 | 272/5000 [11:23<3:18:00, 2.51s/it, lr: 2.0e-04 loss: 2.974e-02] lana_lora_wan2.2: 5%|##8 | 273/5000 [11:27<3:18:22, 2.52s/it, lr: 2.0e-04 loss: 1.255e-01] lana_lora_wan2.2: 5%|##8 | 273/5000 [11:27<3:18:22, 2.52s/it, lr: 2.0e-04 loss: 1.255e-01] lana_lora_wan2.2: 5%|##8 | 274/5000 [11:28<3:17:59, 2.51s/it, lr: 2.0e-04 loss: 8.425e-02] lana_lora_wan2.2: 5%|##8 | 274/5000 [11:28<3:17:59, 2.51s/it, lr: 2.0e-04 loss: 8.425e-02] lana_lora_wan2.2: 6%|##8 | 275/5000 [11:32<3:18:17, 2.52s/it, lr: 2.0e-04 loss: 2.872e-02] lana_lora_wan2.2: 6%|##8 | 275/5000 [11:32<3:18:17, 2.52s/it, lr: 2.0e-04 loss: 2.872e-02] lana_lora_wan2.2: 6%|##8 | 276/5000 [11:34<3:18:09, 2.52s/it, lr: 2.0e-04 loss: 4.792e-02] lana_lora_wan2.2: 6%|##8 | 276/5000 [11:34<3:18:09, 2.52s/it, lr: 2.0e-04 loss: 4.792e-02] lana_lora_wan2.2: 6%|##8 | 277/5000 [11:36<3:18:01, 2.52s/it, lr: 2.0e-04 loss: 4.686e-02] lana_lora_wan2.2: 6%|##8 | 277/5000 [11:36<3:18:01, 2.52s/it, lr: 2.0e-04 loss: 4.686e-02] lana_lora_wan2.2: 6%|##8 | 278/5000 [11:38<3:17:39, 2.51s/it, lr: 2.0e-04 loss: 8.441e-02] lana_lora_wan2.2: 6%|##8 | 278/5000 [11:38<3:17:39, 2.51s/it, lr: 2.0e-04 loss: 8.441e-02] lana_lora_wan2.2: 6%|##9 | 279/5000 [11:39<3:17:23, 2.51s/it, lr: 2.0e-04 loss: 4.588e-02] lana_lora_wan2.2: 6%|##9 | 279/5000 [11:39<3:17:23, 2.51s/it, lr: 2.0e-04 loss: 4.588e-02] lana_lora_wan2.2: 6%|##9 | 280/5000 [11:41<3:17:01, 2.50s/it, lr: 2.0e-04 loss: 5.317e-02] lana_lora_wan2.2: 6%|##9 | 280/5000 [11:41<3:17:01, 2.50s/it, lr: 2.0e-04 loss: 5.317e-02] lana_lora_wan2.2: 6%|##9 | 281/5000 [11:44<3:17:18, 2.51s/it, lr: 2.0e-04 loss: 5.112e-02] lana_lora_wan2.2: 6%|##9 | 281/5000 [11:44<3:17:18, 2.51s/it, lr: 2.0e-04 loss: 5.112e-02] lana_lora_wan2.2: 6%|##9 | 282/5000 [11:46<3:16:56, 2.50s/it, lr: 2.0e-04 loss: 2.452e-01] lana_lora_wan2.2: 6%|##9 | 282/5000 [11:46<3:16:56, 2.50s/it, lr: 2.0e-04 loss: 2.452e-01] lana_lora_wan2.2: 6%|##9 | 283/5000 [11:49<3:17:14, 2.51s/it, lr: 2.0e-04 loss: 6.514e-02] lana_lora_wan2.2: 6%|##9 | 283/5000 [11:49<3:17:14, 2.51s/it, lr: 2.0e-04 loss: 6.514e-02] lana_lora_wan2.2: 6%|##9 | 284/5000 [11:52<3:17:06, 2.51s/it, lr: 2.0e-04 loss: 2.918e-02] lana_lora_wan2.2: 6%|##9 | 284/5000 [11:52<3:17:06, 2.51s/it, lr: 2.0e-04 loss: 2.918e-02] lana_lora_wan2.2: 6%|##9 | 285/5000 [11:56<3:17:27, 2.51s/it, lr: 2.0e-04 loss: 3.411e-02] lana_lora_wan2.2: 6%|##9 | 285/5000 [11:56<3:17:27, 2.51s/it, lr: 2.0e-04 loss: 3.411e-02] lana_lora_wan2.2: 6%|##9 | 286/5000 [11:59<3:17:44, 2.52s/it, lr: 2.0e-04 loss: 9.499e-02] lana_lora_wan2.2: 6%|##9 | 286/5000 [11:59<3:17:44, 2.52s/it, lr: 2.0e-04 loss: 9.499e-02] lana_lora_wan2.2: 6%|##9 | 287/5000 [12:02<3:17:37, 2.52s/it, lr: 2.0e-04 loss: 4.363e-02] lana_lora_wan2.2: 6%|##9 | 287/5000 [12:02<3:17:37, 2.52s/it, lr: 2.0e-04 loss: 4.363e-02] lana_lora_wan2.2: 6%|##9 | 288/5000 [12:05<3:17:53, 2.52s/it, lr: 2.0e-04 loss: 4.273e-02] lana_lora_wan2.2: 6%|##9 | 288/5000 [12:05<3:17:53, 2.52s/it, lr: 2.0e-04 loss: 4.273e-02] lana_lora_wan2.2: 6%|### | 289/5000 [12:07<3:17:32, 2.52s/it, lr: 2.0e-04 loss: 5.929e-02] lana_lora_wan2.2: 6%|### | 289/5000 [12:07<3:17:32, 2.52s/it, lr: 2.0e-04 loss: 5.929e-02] lana_lora_wan2.2: 6%|### | 290/5000 [12:10<3:17:48, 2.52s/it, lr: 2.0e-04 loss: 4.119e-02] lana_lora_wan2.2: 6%|### | 290/5000 [12:10<3:17:48, 2.52s/it, lr: 2.0e-04 loss: 4.119e-02] lana_lora_wan2.2: 6%|### | 291/5000 [12:12<3:17:32, 2.52s/it, lr: 2.0e-04 loss: 1.167e-01] lana_lora_wan2.2: 6%|### | 291/5000 [12:12<3:17:32, 2.52s/it, lr: 2.0e-04 loss: 1.167e-01] lana_lora_wan2.2: 6%|### | 292/5000 [12:16<3:17:48, 2.52s/it, lr: 2.0e-04 loss: 3.936e-02] lana_lora_wan2.2: 6%|### | 292/5000 [12:16<3:17:48, 2.52s/it, lr: 2.0e-04 loss: 3.936e-02] lana_lora_wan2.2: 6%|### | 293/5000 [12:18<3:17:41, 2.52s/it, lr: 2.0e-04 loss: 6.175e-02] lana_lora_wan2.2: 6%|### | 293/5000 [12:18<3:17:41, 2.52s/it, lr: 2.0e-04 loss: 6.175e-02] lana_lora_wan2.2: 6%|### | 294/5000 [12:22<3:17:57, 2.52s/it, lr: 2.0e-04 loss: 2.519e-01] lana_lora_wan2.2: 6%|### | 294/5000 [12:22<3:17:57, 2.52s/it, lr: 2.0e-04 loss: 2.519e-01] lana_lora_wan2.2: 6%|### | 295/5000 [12:25<3:18:13, 2.53s/it, lr: 2.0e-04 loss: 4.817e-02] lana_lora_wan2.2: 6%|### | 295/5000 [12:25<3:18:13, 2.53s/it, lr: 2.0e-04 loss: 4.817e-02] lana_lora_wan2.2: 6%|### | 296/5000 [12:27<3:18:06, 2.53s/it, lr: 2.0e-04 loss: 5.528e-02] lana_lora_wan2.2: 6%|### | 296/5000 [12:27<3:18:06, 2.53s/it, lr: 2.0e-04 loss: 5.528e-02] lana_lora_wan2.2: 6%|### | 297/5000 [12:31<3:18:25, 2.53s/it, lr: 2.0e-04 loss: 4.472e-02] lana_lora_wan2.2: 6%|### | 297/5000 [12:31<3:18:25, 2.53s/it, lr: 2.0e-04 loss: 4.472e-02] lana_lora_wan2.2: 6%|### | 298/5000 [12:33<3:18:05, 2.53s/it, lr: 2.0e-04 loss: 4.905e-02] lana_lora_wan2.2: 6%|### | 298/5000 [12:33<3:18:05, 2.53s/it, lr: 2.0e-04 loss: 4.905e-02] lana_lora_wan2.2: 6%|###1 | 299/5000 [12:34<3:17:45, 2.52s/it, lr: 2.0e-04 loss: 6.874e-02] lana_lora_wan2.2: 6%|###1 | 299/5000 [12:34<3:17:45, 2.52s/it, lr: 2.0e-04 loss: 6.874e-02] lana_lora_wan2.2: 6%|###1 | 300/5000 [12:36<3:17:37, 2.52s/it, lr: 2.0e-04 loss: 4.731e-02] lana_lora_wan2.2: 6%|###1 | 300/5000 [12:36<3:17:37, 2.52s/it, lr: 2.0e-04 loss: 4.731e-02] lana_lora_wan2.2: 6%|###1 | 301/5000 [12:40<3:17:53, 2.53s/it, lr: 2.0e-04 loss: 3.618e-02] lana_lora_wan2.2: 6%|###1 | 301/5000 [12:40<3:17:53, 2.53s/it, lr: 2.0e-04 loss: 3.618e-02] lana_lora_wan2.2: 6%|###1 | 302/5000 [12:44<3:18:09, 2.53s/it, lr: 2.0e-04 loss: 4.557e-02] lana_lora_wan2.2: 6%|###1 | 302/5000 [12:44<3:18:09, 2.53s/it, lr: 2.0e-04 loss: 4.557e-02] lana_lora_wan2.2: 6%|###1 | 303/5000 [12:46<3:18:06, 2.53s/it, lr: 2.0e-04 loss: 4.148e-02] lana_lora_wan2.2: 6%|###1 | 303/5000 [12:46<3:18:06, 2.53s/it, lr: 2.0e-04 loss: 4.148e-02] lana_lora_wan2.2: 6%|###1 | 304/5000 [12:50<3:18:21, 2.53s/it, lr: 2.0e-04 loss: 4.736e-02] lana_lora_wan2.2: 6%|###1 | 304/5000 [12:50<3:18:21, 2.53s/it, lr: 2.0e-04 loss: 4.736e-02] lana_lora_wan2.2: 6%|###1 | 305/5000 [12:51<3:18:01, 2.53s/it, lr: 2.0e-04 loss: 8.063e-02] lana_lora_wan2.2: 6%|###1 | 305/5000 [12:51<3:18:01, 2.53s/it, lr: 2.0e-04 loss: 8.063e-02] lana_lora_wan2.2: 6%|###1 | 306/5000 [12:55<3:18:16, 2.53s/it, lr: 2.0e-04 loss: 2.606e-02] lana_lora_wan2.2: 6%|###1 | 306/5000 [12:55<3:18:16, 2.53s/it, lr: 2.0e-04 loss: 2.606e-02] lana_lora_wan2.2: 6%|###1 | 307/5000 [12:57<3:18:09, 2.53s/it, lr: 2.0e-04 loss: 6.985e-02] lana_lora_wan2.2: 6%|###1 | 307/5000 [12:57<3:18:09, 2.53s/it, lr: 2.0e-04 loss: 6.985e-02] lana_lora_wan2.2: 6%|###2 | 308/5000 [13:01<3:18:28, 2.54s/it, lr: 2.0e-04 loss: 4.608e-02] lana_lora_wan2.2: 6%|###2 | 308/5000 [13:01<3:18:28, 2.54s/it, lr: 2.0e-04 loss: 4.608e-02] lana_lora_wan2.2: 6%|###2 | 309/5000 [13:03<3:18:21, 2.54s/it, lr: 2.0e-04 loss: 7.057e-02] lana_lora_wan2.2: 6%|###2 | 309/5000 [13:03<3:18:21, 2.54s/it, lr: 2.0e-04 loss: 7.057e-02] lana_lora_wan2.2: 6%|###2 | 310/5000 [13:06<3:18:13, 2.54s/it, lr: 2.0e-04 loss: 4.655e-02] lana_lora_wan2.2: 6%|###2 | 310/5000 [13:06<3:18:13, 2.54s/it, lr: 2.0e-04 loss: 4.655e-02] lana_lora_wan2.2: 6%|###2 | 311/5000 [13:07<3:17:53, 2.53s/it, lr: 2.0e-04 loss: 8.367e-02] lana_lora_wan2.2: 6%|###2 | 311/5000 [13:07<3:17:53, 2.53s/it, lr: 2.0e-04 loss: 8.367e-02] lana_lora_wan2.2: 6%|###2 | 312/5000 [13:09<3:17:46, 2.53s/it, lr: 2.0e-04 loss: 6.861e-02] lana_lora_wan2.2: 6%|###2 | 312/5000 [13:09<3:17:46, 2.53s/it, lr: 2.0e-04 loss: 6.861e-02] lana_lora_wan2.2: 6%|###2 | 313/5000 [13:13<3:18:01, 2.53s/it, lr: 2.0e-04 loss: 8.861e-02] lana_lora_wan2.2: 6%|###2 | 313/5000 [13:13<3:18:01, 2.53s/it, lr: 2.0e-04 loss: 8.861e-02] lana_lora_wan2.2: 6%|###2 | 314/5000 [13:17<3:18:16, 2.54s/it, lr: 2.0e-04 loss: 8.938e-02] lana_lora_wan2.2: 6%|###2 | 314/5000 [13:17<3:18:16, 2.54s/it, lr: 2.0e-04 loss: 8.938e-02] lana_lora_wan2.2: 6%|###2 | 315/5000 [13:21<3:18:35, 2.54s/it, lr: 2.0e-04 loss: 3.202e-02] lana_lora_wan2.2: 6%|###2 | 315/5000 [13:21<3:18:35, 2.54s/it, lr: 2.0e-04 loss: 3.202e-02] lana_lora_wan2.2: 6%|###2 | 316/5000 [13:23<3:18:27, 2.54s/it, lr: 2.0e-04 loss: 3.225e-02] lana_lora_wan2.2: 6%|###2 | 316/5000 [13:23<3:18:27, 2.54s/it, lr: 2.0e-04 loss: 3.225e-02] lana_lora_wan2.2: 6%|###2 | 317/5000 [13:25<3:18:20, 2.54s/it, lr: 2.0e-04 loss: 6.911e-02] lana_lora_wan2.2: 6%|###2 | 317/5000 [13:25<3:18:20, 2.54s/it, lr: 2.0e-04 loss: 6.911e-02] lana_lora_wan2.2: 6%|###3 | 318/5000 [13:26<3:18:00, 2.54s/it, lr: 2.0e-04 loss: 7.620e-02] lana_lora_wan2.2: 6%|###3 | 318/5000 [13:26<3:18:00, 2.54s/it, lr: 2.0e-04 loss: 7.620e-02] lana_lora_wan2.2: 6%|###3 | 319/5000 [13:30<3:18:14, 2.54s/it, lr: 2.0e-04 loss: 4.319e-02] lana_lora_wan2.2: 6%|###3 | 319/5000 [13:30<3:18:14, 2.54s/it, lr: 2.0e-04 loss: 4.319e-02] lana_lora_wan2.2: 6%|###3 | 320/5000 [13:34<3:18:32, 2.55s/it, lr: 2.0e-04 loss: 3.216e-02] lana_lora_wan2.2: 6%|###3 | 320/5000 [13:34<3:18:32, 2.55s/it, lr: 2.0e-04 loss: 3.216e-02] lana_lora_wan2.2: 6%|###3 | 321/5000 [13:35<3:18:13, 2.54s/it, lr: 2.0e-04 loss: 5.804e-02] lana_lora_wan2.2: 6%|###3 | 321/5000 [13:35<3:18:13, 2.54s/it, lr: 2.0e-04 loss: 5.804e-02] lana_lora_wan2.2: 6%|###3 | 322/5000 [13:37<3:17:53, 2.54s/it, lr: 2.0e-04 loss: 1.529e-01] lana_lora_wan2.2: 6%|###3 | 322/5000 [13:37<3:17:53, 2.54s/it, lr: 2.0e-04 loss: 1.529e-01] lana_lora_wan2.2: 6%|###3 | 323/5000 [13:39<3:17:45, 2.54s/it, lr: 2.0e-04 loss: 5.788e-02] lana_lora_wan2.2: 6%|###3 | 323/5000 [13:39<3:17:45, 2.54s/it, lr: 2.0e-04 loss: 5.788e-02] lana_lora_wan2.2: 6%|###3 | 324/5000 [13:43<3:17:59, 2.54s/it, lr: 2.0e-04 loss: 3.917e-02] lana_lora_wan2.2: 6%|###3 | 324/5000 [13:43<3:17:59, 2.54s/it, lr: 2.0e-04 loss: 3.917e-02] lana_lora_wan2.2: 6%|###3 | 325/5000 [13:45<3:17:52, 2.54s/it, lr: 2.0e-04 loss: 9.742e-02] lana_lora_wan2.2: 6%|###3 | 325/5000 [13:45<3:17:52, 2.54s/it, lr: 2.0e-04 loss: 9.742e-02] lana_lora_wan2.2: 7%|###3 | 326/5000 [13:49<3:18:10, 2.54s/it, lr: 2.0e-04 loss: 4.856e-02] lana_lora_wan2.2: 7%|###3 | 326/5000 [13:49<3:18:10, 2.54s/it, lr: 2.0e-04 loss: 4.856e-02] lana_lora_wan2.2: 7%|###4 | 327/5000 [13:51<3:18:02, 2.54s/it, lr: 2.0e-04 loss: 4.533e-02] lana_lora_wan2.2: 7%|###4 | 327/5000 [13:51<3:18:02, 2.54s/it, lr: 2.0e-04 loss: 4.533e-02] lana_lora_wan2.2: 7%|###4 | 328/5000 [13:52<3:17:43, 2.54s/it, lr: 2.0e-04 loss: 1.034e-01] lana_lora_wan2.2: 7%|###4 | 328/5000 [13:52<3:17:43, 2.54s/it, lr: 2.0e-04 loss: 1.034e-01] lana_lora_wan2.2: 7%|###4 | 329/5000 [13:54<3:17:23, 2.54s/it, lr: 2.0e-04 loss: 1.387e-01] lana_lora_wan2.2: 7%|###4 | 329/5000 [13:54<3:17:23, 2.54s/it, lr: 2.0e-04 loss: 1.387e-01] lana_lora_wan2.2: 7%|###4 | 330/5000 [13:55<3:17:05, 2.53s/it, lr: 2.0e-04 loss: 7.939e-02] lana_lora_wan2.2: 7%|###4 | 330/5000 [13:55<3:17:05, 2.53s/it, lr: 2.0e-04 loss: 7.939e-02] lana_lora_wan2.2: 7%|###4 | 331/5000 [13:57<3:16:57, 2.53s/it, lr: 2.0e-04 loss: 3.394e-02] lana_lora_wan2.2: 7%|###4 | 331/5000 [13:57<3:16:57, 2.53s/it, lr: 2.0e-04 loss: 3.394e-02] lana_lora_wan2.2: 7%|###4 | 332/5000 [14:00<3:16:54, 2.53s/it, lr: 2.0e-04 loss: 6.555e-02] lana_lora_wan2.2: 7%|###4 | 332/5000 [14:00<3:16:54, 2.53s/it, lr: 2.0e-04 loss: 6.555e-02] lana_lora_wan2.2: 7%|###4 | 333/5000 [14:01<3:16:35, 2.53s/it, lr: 2.0e-04 loss: 1.022e-01] lana_lora_wan2.2: 7%|###4 | 333/5000 [14:01<3:16:35, 2.53s/it, lr: 2.0e-04 loss: 1.022e-01] lana_lora_wan2.2: 7%|###4 | 334/5000 [14:05<3:16:49, 2.53s/it, lr: 2.0e-04 loss: 4.334e-02] lana_lora_wan2.2: 7%|###4 | 334/5000 [14:05<3:16:49, 2.53s/it, lr: 2.0e-04 loss: 4.334e-02] lana_lora_wan2.2: 7%|###4 | 335/5000 [14:07<3:16:42, 2.53s/it, lr: 2.0e-04 loss: 6.158e-02] lana_lora_wan2.2: 7%|###4 | 335/5000 [14:07<3:16:42, 2.53s/it, lr: 2.0e-04 loss: 6.158e-02] lana_lora_wan2.2: 7%|###4 | 336/5000 [14:08<3:16:24, 2.53s/it, lr: 2.0e-04 loss: 4.050e-01] lana_lora_wan2.2: 7%|###4 | 336/5000 [14:08<3:16:24, 2.53s/it, lr: 2.0e-04 loss: 4.050e-01] lana_lora_wan2.2: 7%|###5 | 337/5000 [14:11<3:16:17, 2.53s/it, lr: 2.0e-04 loss: 5.825e-02] lana_lora_wan2.2: 7%|###5 | 337/5000 [14:11<3:16:17, 2.53s/it, lr: 2.0e-04 loss: 5.825e-02] lana_lora_wan2.2: 7%|###5 | 338/5000 [14:13<3:16:13, 2.53s/it, lr: 2.0e-04 loss: 6.241e-02] lana_lora_wan2.2: 7%|###5 | 338/5000 [14:13<3:16:13, 2.53s/it, lr: 2.0e-04 loss: 6.241e-02] lana_lora_wan2.2: 7%|###5 | 339/5000 [14:15<3:16:07, 2.52s/it, lr: 2.0e-04 loss: 4.559e-02] lana_lora_wan2.2: 7%|###5 | 339/5000 [14:15<3:16:07, 2.52s/it, lr: 2.0e-04 loss: 4.559e-02] lana_lora_wan2.2: 7%|###5 | 340/5000 [14:17<3:15:51, 2.52s/it, lr: 2.0e-04 loss: 3.617e-02] lana_lora_wan2.2: 7%|###5 | 340/5000 [14:17<3:15:51, 2.52s/it, lr: 2.0e-04 loss: 3.617e-02] lana_lora_wan2.2: 7%|###5 | 341/5000 [14:19<3:15:45, 2.52s/it, lr: 2.0e-04 loss: 5.867e-02] lana_lora_wan2.2: 7%|###5 | 341/5000 [14:19<3:15:45, 2.52s/it, lr: 2.0e-04 loss: 5.867e-02] lana_lora_wan2.2: 7%|###5 | 342/5000 [14:23<3:15:58, 2.52s/it, lr: 2.0e-04 loss: 2.270e-01] lana_lora_wan2.2: 7%|###5 | 342/5000 [14:23<3:15:58, 2.52s/it, lr: 2.0e-04 loss: 2.270e-01] lana_lora_wan2.2: 7%|###5 | 343/5000 [14:24<3:15:40, 2.52s/it, lr: 2.0e-04 loss: 5.228e-02] lana_lora_wan2.2: 7%|###5 | 343/5000 [14:24<3:15:40, 2.52s/it, lr: 2.0e-04 loss: 5.228e-02] lana_lora_wan2.2: 7%|###5 | 344/5000 [14:28<3:15:56, 2.53s/it, lr: 2.0e-04 loss: 6.158e-02] lana_lora_wan2.2: 7%|###5 | 344/5000 [14:28<3:15:56, 2.53s/it, lr: 2.0e-04 loss: 6.158e-02] lana_lora_wan2.2: 7%|###5 | 345/5000 [14:30<3:15:38, 2.52s/it, lr: 2.0e-04 loss: 5.822e-02] lana_lora_wan2.2: 7%|###5 | 345/5000 [14:30<3:15:38, 2.52s/it, lr: 2.0e-04 loss: 5.822e-02] \ No newline at end of file