data_keys = [ 'motion', 'motion_mask', 'motion_length', 'melspec', 'audio', 'text', 'gest_types', 'discourse', 'speaker_id' ] train_pipeline = [] motion_length = 150 motion_fps = 15 audio_sr = 16000 base_data_cfg = dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= './cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo') train_cfg = dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= './cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo', split='train') val_cfg = dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= './cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo', split='val') test_cfg = dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= './cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo', split='test') data = dict( samples_per_gpu=128, workers_per_gpu=8, train=dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= './cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo', split='train'), val=dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= '/CT/GestureSynth1/work/DiscourseAwareGesture/RAGGesture_BEATX/cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo', split='val'), test=dict( type='BEATXDataset', pose_rep='smplxflame_30', facial_rep='smplxflame_30', data_path= '/CT/GestureSynth1/work/GestureGPT/PantoMatrix/BEAT2/beat_english_v2.0.0/', cache_path= './cache/beatx_cache_backup/', debug=False, tiny=False, face_joint_idx=[18, 13, 9, 5], sample_rate=16000, num_mels=80, hop_length=512, fps=15, stride=5, pose_length=150, ori_joints='beat_smplx_joints', deps_path='/CT/GestureSynth1/work/GestureGPT/GestureRep/deps/', training_speakers=[ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 ], additional_data=True, pose_fps=15, audio_sr=16000, audio_fps=16000, new_cache=False, clean_first_seconds=0, clean_final_seconds=0, audio_rep='wav2vec', word_rep='bert_framealigned', id_rep='idx', sem_rep='info', prom_rep='prom', emo_rep='emo', split='test')) checkpoint_config = dict(interval=2) dist_params = dict(backend='nccl') log_level = 'INFO' load_from = None resume_from = './experiments/base_beatx_len150fps15_lowertrans4lats_remodiffuseclfguide_8l16h/latest.pth' workflow = [('train', 1)] optimizer = dict(type='Adam', lr=0.0001) optimizer_config = dict(grad_clip=None) lr_config = dict(policy='CosineAnnealing', min_lr_ratio=1e-06, by_epoch=False) runner = dict(type='EpochBasedRunner', max_epochs=500) log_config = dict( interval=10, hooks=[dict(type='TextLoggerHook'), dict(type='TensorboardLoggerHook')]) custom_hooks = [ dict(type='VAE_FreezeHook'), dict( type='DatabaseSaveHook', save_dir= './experiments/retrieval_dicts' ) ] input_feats = 189 max_seq_len = 150 frame_chunk_size = 15 latent_dim = 512 time_embed_dim = 2048 inp_text_latent_dim = 768 ff_size = 1024 num_heads = 16 num_layers = 8 dropout = 0 model = dict( type='MotionDiffusion', model=dict( type='ReGestureTransformer', input_feats=189, max_seq_len=150, frame_chunk_size=15, latent_dim=512, time_embed_dim=2048, num_layers=8, body_part_cat_axis='time', sa_block_cfg=dict( type='EfficientSelfAttention', latent_dim=512, num_heads=16, dropout=0, time_embed_dim=2048), ca_block_cfg=dict( type='EfficientCrossAttention', latent_dim=512, text_latent_dim=512, num_heads=16, dropout=0, time_embed_dim=2048), ffn_cfg=dict( latent_dim=512, ffn_dim=1024, dropout=0, time_embed_dim=2048), vae_cfg=dict( upper_cfg= '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/0903_020101_gesture_lexicon_transformer_vae_upper_allspk_len256_l8h4_fchunksize15/0903_020101_gesture_lexicon_transformer_vae_upper_allspk_len256_l8h4_fchunksize15.yaml', lowertrans_cfg= '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/1031_142417_gesture_lexicon_transformer_vae_lowerplustrans_allspk_len10s_l8h8_fchunksize15_run2/1031_142417_gesture_lexicon_transformer_vae_lowerplustrans_allspk_len10s_l8h8_fchunksize15_run2.yaml', face_cfg= '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/0909_130750_gesture_lexicon_transformer_vae_face_allspk_len10s_l8h4_fchunksize15/0909_130750_gesture_lexicon_transformer_vae_face_allspk_len10s_l8h4_fchunksize15.yaml', hands_cfg= '/CT/GestureSynth1/work/GestureGPT/GestureRep/experiments/0909_132647_gesture_lexicon_transformer_vae_hands_allspk_len10s_l8h4_fchunksize15/0909_132647_gesture_lexicon_transformer_vae_hands_allspk_len10s_l8h4_fchunksize15.yaml', latent_dim=512, frame_chunk_size=15), text_encoder=dict( pretrained_model=None, latent_dim=768, num_layers=0, ff_size=2048, dropout=0, use_text_proj=False), audio_encoder=dict( pretrained_model=None, latent_dim=768, num_layers=0, dropout=0.1), speaker_embedding=dict(num_speakers=25), retrieval_train=False, retrieval_cfg=dict( motion_feat_dim=189, num_retrieval=1, stride=4, num_layers=2, num_motion_layers=2, kinematic_coef=0.1, topk=2, latent_dim=512, text_latent_dim=768, output_dim=512, max_seq_len=150, motion_fps=15, motion_framechunksize=15, num_heads=16, ff_size=1024, dropout=0, lmdb_paths= '/CT/GestureSynth1/work/DiscourseAwareGesture/RAGGesture_BEATX/cache/retrieval_cache_backup/retrieval_cache_stratified/', new_lmdb_cache=False, stratified_db_creation=True, stratification_interval=15, ffn_cfg=dict(latent_dim=512, ffn_dim=1024, dropout=0), sa_block_cfg=dict( type='EfficientSelfAttention', latent_dim=512, num_heads=16, dropout=0)), scale_func_cfg=dict( coarse_scale=6.5, both_coef=0.52351, text_coef=-0.28419, retr_coef=2.39872), per_joint_scale=dict(upper=1.0, hands=1.0, face=1.0, lowertransl=1.0), ), loss_recon=dict(type='MSELoss', loss_weight=1, reduction='none'), body_part_lossweights=dict( upper=1.0, hands=1.0, face=1.0, lowertransl=1.0), diffusion_train=dict( beta_scheduler='scaled_linear', diffusion_steps=1000, model_mean_type='start_x', model_var_type='fixed_large'), diffusion_test=dict( beta_scheduler='scaled_linear', diffusion_steps=1000, model_mean_type='start_x', model_var_type='fixed_large', respace='15,15,8,6,6', num_inference_timesteps=50, classifier_free_guidance_scale=0), inference_type='ddim') work_dir = './experiments/base_beatx_len150fps15_lowertrans4lats_remodiffuseclfguide_8l16h/' gpu_ids = range(0, 1)