rag-gesture-weights / diffusion /base_beatx_len150fps15_finalweights /basegesture_len150_beat_backup.py
| data_keys = [ | |
| 'motion', 'motion_mask', 'motion_length', 'melspec', 'audio', 'text', | |
| 'gest_types', 'discourse', 'speaker_id' | |
| ] | |
| train_pipeline = [] | |
| motion_length = 150 | |
| motion_fps = 15 | |
| audio_sr = 16000 | |
| base_data_cfg = dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| './cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, | |
| 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo') | |
| train_cfg = dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| './cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, | |
| 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo', | |
| split='train') | |
| val_cfg = dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| './cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, | |
| 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo', | |
| split='val') | |
| test_cfg = dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| './cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, | |
| 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo', | |
| split='test') | |
| data = dict( | |
| samples_per_gpu=128, | |
| workers_per_gpu=8, | |
| train=dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| './cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, | |
| 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo', | |
| split='train'), | |
| val=dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| '/CT/GestureSynth1/work/DiscourseAwareGesture/RAGGesture_BEATX/cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, | |
| 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo', | |
| split='val'), | |
| test=dict( | |
| type='BEATXDataset', | |
| pose_rep='smplxflame_30', | |
| facial_rep='smplxflame_30', | |
| data_path= | |
| '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', | |
| cache_path= | |
| './cache/beatx_cache_backup/', | |
| debug=False, | |
| tiny=False, | |
| face_joint_idx=[18, 13, 9, 5], | |
| sample_rate=16000, | |
| num_mels=80, | |
| hop_length=512, | |
| fps=15, | |
| stride=5, | |
| pose_length=150, | |
| ori_joints='beat_smplx_joints', | |
| deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', | |
| training_speakers=[ | |
| 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, | |
| 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 | |
| ], | |
| additional_data=True, | |
| pose_fps=15, | |
| audio_sr=16000, | |
| audio_fps=16000, | |
| new_cache=False, | |
| clean_first_seconds=0, | |
| clean_final_seconds=0, | |
| audio_rep='wav2vec', | |
| word_rep='bert_framealigned', | |
| id_rep='idx', | |
| sem_rep='info', | |
| prom_rep='prom', | |
| emo_rep='emo', | |
| split='test')) | |
| checkpoint_config = dict(interval=2) | |
| dist_params = dict(backend='nccl') | |
| log_level = 'INFO' | |
| load_from = None | |
| resume_from = './experiments/base_beatx_len150fps15_lowertrans4lats_remodiffuseclfguide_8l16h/latest.pth' | |
| workflow = [('train', 1)] | |
| optimizer = dict(type='Adam', lr=0.0001) | |
| optimizer_config = dict(grad_clip=None) | |
| lr_config = dict(policy='CosineAnnealing', min_lr_ratio=1e-06, by_epoch=False) | |
| runner = dict(type='EpochBasedRunner', max_epochs=500) | |
| log_config = dict( | |
| interval=10, | |
| hooks=[dict(type='TextLoggerHook'), | |
| dict(type='TensorboardLoggerHook')]) | |
| custom_hooks = [ | |
| dict(type='VAE_FreezeHook'), | |
| dict( | |
| type='DatabaseSaveHook', | |
| save_dir= | |
| './experiments/retrieval_dicts' | |
| ) | |
| ] | |
| input_feats = 189 | |
| max_seq_len = 150 | |
| frame_chunk_size = 15 | |
| latent_dim = 512 | |
| time_embed_dim = 2048 | |
| inp_text_latent_dim = 768 | |
| ff_size = 1024 | |
| num_heads = 16 | |
| num_layers = 8 | |
| dropout = 0 | |
| model = dict( | |
| type='MotionDiffusion', | |
| model=dict( | |
| type='ReGestureTransformer', | |
| input_feats=189, | |
| max_seq_len=150, | |
| frame_chunk_size=15, | |
| latent_dim=512, | |
| time_embed_dim=2048, | |
| num_layers=8, | |
| body_part_cat_axis='time', | |
| sa_block_cfg=dict( | |
| type='EfficientSelfAttention', | |
| latent_dim=512, | |
| num_heads=16, | |
| dropout=0, | |
| time_embed_dim=2048), | |
| ca_block_cfg=dict( | |
| type='EfficientCrossAttention', | |
| latent_dim=512, | |
| text_latent_dim=512, | |
| num_heads=16, | |
| dropout=0, | |
| time_embed_dim=2048), | |
| ffn_cfg=dict( | |
| latent_dim=512, ffn_dim=1024, dropout=0, time_embed_dim=2048), | |
| vae_cfg=dict( | |
| upper_cfg= | |
| '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/0903_020101_gesture_lexicon_transformer_vae_upper_allspk_len256_l8h4_fchunksize15/0903_020101_gesture_lexicon_transformer_vae_upper_allspk_len256_l8h4_fchunksize15.yaml', | |
| lowertrans_cfg= | |
| '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/1031_142417_gesture_lexicon_transformer_vae_lowerplustrans_allspk_len10s_l8h8_fchunksize15_run2/1031_142417_gesture_lexicon_transformer_vae_lowerplustrans_allspk_len10s_l8h8_fchunksize15_run2.yaml', | |
| face_cfg= | |
| '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/0909_130750_gesture_lexicon_transformer_vae_face_allspk_len10s_l8h4_fchunksize15/0909_130750_gesture_lexicon_transformer_vae_face_allspk_len10s_l8h4_fchunksize15.yaml', | |
| hands_cfg= | |
| '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/0909_132647_gesture_lexicon_transformer_vae_hands_allspk_len10s_l8h4_fchunksize15/0909_132647_gesture_lexicon_transformer_vae_hands_allspk_len10s_l8h4_fchunksize15.yaml', | |
| latent_dim=512, | |
| frame_chunk_size=15), | |
| text_encoder=dict( | |
| pretrained_model=None, | |
| latent_dim=768, | |
| num_layers=0, | |
| ff_size=2048, | |
| dropout=0, | |
| use_text_proj=False), | |
| audio_encoder=dict( | |
| pretrained_model=None, latent_dim=768, num_layers=0, dropout=0.1), | |
| speaker_embedding=dict(num_speakers=25), | |
| retrieval_train=False, | |
| retrieval_cfg=dict( | |
| motion_feat_dim=189, | |
| num_retrieval=1, | |
| stride=4, | |
| num_layers=2, | |
| num_motion_layers=2, | |
| kinematic_coef=0.1, | |
| topk=2, | |
| latent_dim=512, | |
| text_latent_dim=768, | |
| output_dim=512, | |
| max_seq_len=150, | |
| motion_fps=15, | |
| motion_framechunksize=15, | |
| num_heads=16, | |
| ff_size=1024, | |
| dropout=0, | |
| lmdb_paths= | |
| '/CT/GestureSynth1/work/DiscourseAwareGesture/RAGGesture_BEATX/cache/retrieval_cache_backup/retrieval_cache_stratified/', | |
| new_lmdb_cache=False, | |
| stratified_db_creation=True, | |
| stratification_interval=15, | |
| ffn_cfg=dict(latent_dim=512, ffn_dim=1024, dropout=0), | |
| sa_block_cfg=dict( | |
| type='EfficientSelfAttention', | |
| latent_dim=512, | |
| num_heads=16, | |
| dropout=0)), | |
| scale_func_cfg=dict( | |
| coarse_scale=6.5, | |
| both_coef=0.52351, | |
| text_coef=-0.28419, | |
| retr_coef=2.39872), | |
| per_joint_scale=dict(upper=1.0, hands=1.0, face=1.0, lowertransl=1.0), | |
| ), | |
| loss_recon=dict(type='MSELoss', loss_weight=1, reduction='none'), | |
| body_part_lossweights=dict( | |
| upper=1.0, hands=1.0, face=1.0, lowertransl=1.0), | |
| diffusion_train=dict( | |
| beta_scheduler='scaled_linear', | |
| diffusion_steps=1000, | |
| model_mean_type='start_x', | |
| model_var_type='fixed_large'), | |
| diffusion_test=dict( | |
| beta_scheduler='scaled_linear', | |
| diffusion_steps=1000, | |
| model_mean_type='start_x', | |
| model_var_type='fixed_large', | |
| respace='15,15,8,6,6', | |
| num_inference_timesteps=50, | |
| classifier_free_guidance_scale=0), | |
| inference_type='ddim') | |
| work_dir = './experiments/base_beatx_len150fps15_lowertrans4lats_remodiffuseclfguide_8l16h/' | |
| gpu_ids = range(0, 1) | |