{ "_model_size_label": "9B", "architectures": [ "ZDTaichu5_0_ForConditionalGeneration" ], "auto_map": { "AutoConfig": "configuration.ZDTaichu5_0_Config", "AutoImageProcessor": "image_processing.ZDTaichu5_0_ImageProcessor", "AutoModel": "modeling.ZDTaichu5_0_ForConditionalGeneration", "AutoModelForCausalLM": "modeling.ZDTaichu5_0_ForConditionalGeneration", "AutoProcessor": "processing.ZDTaichu5_0_Processor" }, "bos_token_id": null, "downsample_ratio": 0.5, "dtype": "bfloat16", "eos_token_id": 248046, "force_image_size": 512, "hidden_size": 4096, "image_tag_type": "internvl", "img_context_token": "<|image_pad|>", "img_context_token_id": 248056, "img_end_token": "<|vision_end|>", "img_start_token": "<|vision_start|>", "keys_to_ignore_at_inference": [ "past_key_values" ], "llm_config": { "architectures": [ "Qwen3_5ForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "bos_token_id": null, "dtype": "bfloat16", "eos_token_id": null, "full_attention_interval": 4, "head_dim": 256, "hidden_act": "silu", "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 12288, "layer_types": [ "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention" ], "linear_conv_kernel_dim": 4, "linear_key_head_dim": 128, "linear_num_key_heads": 16, "linear_num_value_heads": 32, "linear_value_head_dim": 128, "max_position_embeddings": 262144, "model_type": "qwen3_5_text", "mtp_loss_scaling_factor": 0.1, "mtp_num_layers": 1, "num_attention_heads": 16, "num_hidden_layers": 32, "num_key_value_heads": 4, "pad_token_id": 248044, "partial_rotary_factor": 0.25, "rms_norm_eps": 1e-06, "rope_parameters": { "mrope_interleaved": true, "mrope_section": [ 11, 11, 10 ], "partial_rotary_factor": 0.25, "rope_theta": 10000000, "rope_type": "default" }, "tie_word_embeddings": false, "use_cache": false, "mamba_ssm_dtype": "float32", "vocab_size": 248320 }, "max_dynamic_patch": 12, "min_dynamic_patch": 1, "model_type": "zdtaichu5_0", "pad_token_id": 248044, "patch_size": 16, "projector_hidden_size": 20480, "ps_version": "v2", "quantization_config": { "config_groups": { "group_0": { "format": "float-quantized", "input_activations": { "actorder": null, "block_structure": null, "dynamic": true, "group_size": null, "num_bits": 8, "observer": null, "observer_kwargs": {}, "scale_dtype": null, "strategy": "token", "symmetric": true, "type": "float", "zp_dtype": null }, "output_activations": null, "targets": [ "re:.*self_attn\\.(q|k|v|o)_proj$", "re:.*linear_attn\\.(in_proj_qkv|in_proj_z|out_proj)$", "re:.*lm_head$", "re:.*layers\\.(?:28|29|30|31)\\.mlp\\.(?:gate|up|down)_proj$" ], "weights": { "actorder": null, "block_structure": null, "dynamic": false, "group_size": null, "num_bits": 8, "observer": "memoryless_minmax", "observer_kwargs": {}, "scale_dtype": null, "strategy": "channel", "symmetric": true, "type": "float", "zp_dtype": null } }, "group_1": { "format": "nvfp4-pack-quantized", "input_activations": { "actorder": null, "block_structure": null, "dynamic": "local", "group_size": 16, "num_bits": 4, "observer": "static_minmax", "observer_kwargs": {}, "scale_dtype": "torch.float8_e4m3fn", "strategy": "tensor_group", "symmetric": true, "type": "float", "zp_dtype": null }, "output_activations": null, "targets": [ "re:.*layers\\.(?:0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27)\\.mlp\\.(?:gate|up|down)_proj$" ], "weights": { "actorder": "static", "block_structure": null, "dynamic": false, "group_size": 16, "num_bits": 4, "observer": "imatrix_mse", "observer_kwargs": {}, "scale_dtype": "torch.float8_e4m3fn", "strategy": "tensor_group", "symmetric": true, "type": "float", "zp_dtype": null } } }, "format": "mixed-precision", "global_compression_ratio": null, "ignore": [ "language_model.model.layers.0.linear_attn.in_proj_b", "language_model.model.layers.0.linear_attn.in_proj_a", "language_model.model.layers.1.linear_attn.in_proj_b", "language_model.model.layers.1.linear_attn.in_proj_a", "language_model.model.layers.2.linear_attn.in_proj_b", "language_model.model.layers.2.linear_attn.in_proj_a", "language_model.model.layers.4.linear_attn.in_proj_b", "language_model.model.layers.4.linear_attn.in_proj_a", "language_model.model.layers.5.linear_attn.in_proj_b", "language_model.model.layers.5.linear_attn.in_proj_a", "language_model.model.layers.6.linear_attn.in_proj_b", "language_model.model.layers.6.linear_attn.in_proj_a", "language_model.model.layers.8.linear_attn.in_proj_b", "language_model.model.layers.8.linear_attn.in_proj_a", "language_model.model.layers.9.linear_attn.in_proj_b", "language_model.model.layers.9.linear_attn.in_proj_a", "language_model.model.layers.10.linear_attn.in_proj_b", "language_model.model.layers.10.linear_attn.in_proj_a", "language_model.model.layers.12.linear_attn.in_proj_b", "language_model.model.layers.12.linear_attn.in_proj_a", "language_model.model.layers.13.linear_attn.in_proj_b", "language_model.model.layers.13.linear_attn.in_proj_a", "language_model.model.layers.14.linear_attn.in_proj_b", "language_model.model.layers.14.linear_attn.in_proj_a", "language_model.model.layers.16.linear_attn.in_proj_b", "language_model.model.layers.16.linear_attn.in_proj_a", "language_model.model.layers.17.linear_attn.in_proj_b", "language_model.model.layers.17.linear_attn.in_proj_a", "language_model.model.layers.18.linear_attn.in_proj_b", "language_model.model.layers.18.linear_attn.in_proj_a", "language_model.model.layers.20.linear_attn.in_proj_b", "language_model.model.layers.20.linear_attn.in_proj_a", "language_model.model.layers.21.linear_attn.in_proj_b", "language_model.model.layers.21.linear_attn.in_proj_a", "language_model.model.layers.22.linear_attn.in_proj_b", "language_model.model.layers.22.linear_attn.in_proj_a", "language_model.model.layers.24.linear_attn.in_proj_b", "language_model.model.layers.24.linear_attn.in_proj_a", "language_model.model.layers.25.linear_attn.in_proj_b", "language_model.model.layers.25.linear_attn.in_proj_a", "language_model.model.layers.26.linear_attn.in_proj_b", "language_model.model.layers.26.linear_attn.in_proj_a", "language_model.model.layers.28.linear_attn.in_proj_b", "language_model.model.layers.28.linear_attn.in_proj_a", "language_model.model.layers.29.linear_attn.in_proj_b", "language_model.model.layers.29.linear_attn.in_proj_a", "language_model.model.layers.30.linear_attn.in_proj_b", "language_model.model.layers.30.linear_attn.in_proj_a", "vision_model.radio_model.model.blocks.0.attn.qkv", "vision_model.radio_model.model.blocks.0.attn.proj", "vision_model.radio_model.model.blocks.0.mlp.fc1", "vision_model.radio_model.model.blocks.0.mlp.fc2", "vision_model.radio_model.model.blocks.1.attn.qkv", "vision_model.radio_model.model.blocks.1.attn.proj", "vision_model.radio_model.model.blocks.1.mlp.fc1", "vision_model.radio_model.model.blocks.1.mlp.fc2", "vision_model.radio_model.model.blocks.2.attn.qkv", "vision_model.radio_model.model.blocks.2.attn.proj", "vision_model.radio_model.model.blocks.2.mlp.fc1", "vision_model.radio_model.model.blocks.2.mlp.fc2", "vision_model.radio_model.model.blocks.3.attn.qkv", "vision_model.radio_model.model.blocks.3.attn.proj", "vision_model.radio_model.model.blocks.3.mlp.fc1", "vision_model.radio_model.model.blocks.3.mlp.fc2", "vision_model.radio_model.model.blocks.4.attn.qkv", "vision_model.radio_model.model.blocks.4.attn.proj", "vision_model.radio_model.model.blocks.4.mlp.fc1", "vision_model.radio_model.model.blocks.4.mlp.fc2", "vision_model.radio_model.model.blocks.5.attn.qkv", "vision_model.radio_model.model.blocks.5.attn.proj", "vision_model.radio_model.model.blocks.5.mlp.fc1", "vision_model.radio_model.model.blocks.5.mlp.fc2", "vision_model.radio_model.model.blocks.6.attn.qkv", "vision_model.radio_model.model.blocks.6.attn.proj", "vision_model.radio_model.model.blocks.6.mlp.fc1", "vision_model.radio_model.model.blocks.6.mlp.fc2", "vision_model.radio_model.model.blocks.7.attn.qkv", "vision_model.radio_model.model.blocks.7.attn.proj", "vision_model.radio_model.model.blocks.7.mlp.fc1", "vision_model.radio_model.model.blocks.7.mlp.fc2", "vision_model.radio_model.model.blocks.8.attn.qkv", "vision_model.radio_model.model.blocks.8.attn.proj", "vision_model.radio_model.model.blocks.8.mlp.fc1", "vision_model.radio_model.model.blocks.8.mlp.fc2", "vision_model.radio_model.model.blocks.9.attn.qkv", "vision_model.radio_model.model.blocks.9.attn.proj", "vision_model.radio_model.model.blocks.9.mlp.fc1", "vision_model.radio_model.model.blocks.9.mlp.fc2", "vision_model.radio_model.model.blocks.10.attn.qkv", "vision_model.radio_model.model.blocks.10.attn.proj", "vision_model.radio_model.model.blocks.10.mlp.fc1", "vision_model.radio_model.model.blocks.10.mlp.fc2", "vision_model.radio_model.model.blocks.11.attn.qkv", "vision_model.radio_model.model.blocks.11.attn.proj", "vision_model.radio_model.model.blocks.11.mlp.fc1", "vision_model.radio_model.model.blocks.11.mlp.fc2", "vision_model.radio_model.model.blocks.12.attn.qkv", "vision_model.radio_model.model.blocks.12.attn.proj", "vision_model.radio_model.model.blocks.12.mlp.fc1", "vision_model.radio_model.model.blocks.12.mlp.fc2", "vision_model.radio_model.model.blocks.13.attn.qkv", "vision_model.radio_model.model.blocks.13.attn.proj", "vision_model.radio_model.model.blocks.13.mlp.fc1", "vision_model.radio_model.model.blocks.13.mlp.fc2", "vision_model.radio_model.model.blocks.14.attn.qkv", "vision_model.radio_model.model.blocks.14.attn.proj", "vision_model.radio_model.model.blocks.14.mlp.fc1", "vision_model.radio_model.model.blocks.14.mlp.fc2", "vision_model.radio_model.model.blocks.15.attn.qkv", "vision_model.radio_model.model.blocks.15.attn.proj", "vision_model.radio_model.model.blocks.15.mlp.fc1", "vision_model.radio_model.model.blocks.15.mlp.fc2", "vision_model.radio_model.model.blocks.16.attn.qkv", "vision_model.radio_model.model.blocks.16.attn.proj", "vision_model.radio_model.model.blocks.16.mlp.fc1", "vision_model.radio_model.model.blocks.16.mlp.fc2", "vision_model.radio_model.model.blocks.17.attn.qkv", "vision_model.radio_model.model.blocks.17.attn.proj", "vision_model.radio_model.model.blocks.17.mlp.fc1", "vision_model.radio_model.model.blocks.17.mlp.fc2", "vision_model.radio_model.model.blocks.18.attn.qkv", "vision_model.radio_model.model.blocks.18.attn.proj", "vision_model.radio_model.model.blocks.18.mlp.fc1", "vision_model.radio_model.model.blocks.18.mlp.fc2", "vision_model.radio_model.model.blocks.19.attn.qkv", "vision_model.radio_model.model.blocks.19.attn.proj", "vision_model.radio_model.model.blocks.19.mlp.fc1", "vision_model.radio_model.model.blocks.19.mlp.fc2", "vision_model.radio_model.model.blocks.20.attn.qkv", "vision_model.radio_model.model.blocks.20.attn.proj", "vision_model.radio_model.model.blocks.20.mlp.fc1", "vision_model.radio_model.model.blocks.20.mlp.fc2", "vision_model.radio_model.model.blocks.21.attn.qkv", "vision_model.radio_model.model.blocks.21.attn.proj", "vision_model.radio_model.model.blocks.21.mlp.fc1", "vision_model.radio_model.model.blocks.21.mlp.fc2", "vision_model.radio_model.model.blocks.22.attn.qkv", "vision_model.radio_model.model.blocks.22.attn.proj", "vision_model.radio_model.model.blocks.22.mlp.fc1", "vision_model.radio_model.model.blocks.22.mlp.fc2", "vision_model.radio_model.model.blocks.23.attn.qkv", "vision_model.radio_model.model.blocks.23.attn.proj", "vision_model.radio_model.model.blocks.23.mlp.fc1", "vision_model.radio_model.model.blocks.23.mlp.fc2", "vision_model.radio_model.model.blocks.24.attn.qkv", "vision_model.radio_model.model.blocks.24.attn.proj", "vision_model.radio_model.model.blocks.24.mlp.fc1", "vision_model.radio_model.model.blocks.24.mlp.fc2", "vision_model.radio_model.model.blocks.25.attn.qkv", "vision_model.radio_model.model.blocks.25.attn.proj", "vision_model.radio_model.model.blocks.25.mlp.fc1", "vision_model.radio_model.model.blocks.25.mlp.fc2", "vision_model.radio_model.model.blocks.26.attn.qkv", "vision_model.radio_model.model.blocks.26.attn.proj", "vision_model.radio_model.model.blocks.26.mlp.fc1", "vision_model.radio_model.model.blocks.26.mlp.fc2", "vision_model.radio_model.model.blocks.27.attn.qkv", "vision_model.radio_model.model.blocks.27.attn.proj", "vision_model.radio_model.model.blocks.27.mlp.fc1", "vision_model.radio_model.model.blocks.27.mlp.fc2", "vision_model.radio_model.model.blocks.28.attn.qkv", "vision_model.radio_model.model.blocks.28.attn.proj", "vision_model.radio_model.model.blocks.28.mlp.fc1", "vision_model.radio_model.model.blocks.28.mlp.fc2", "vision_model.radio_model.model.blocks.29.attn.qkv", "vision_model.radio_model.model.blocks.29.attn.proj", "vision_model.radio_model.model.blocks.29.mlp.fc1", "vision_model.radio_model.model.blocks.29.mlp.fc2", "vision_model.radio_model.model.blocks.30.attn.qkv", "vision_model.radio_model.model.blocks.30.attn.proj", "vision_model.radio_model.model.blocks.30.mlp.fc1", "vision_model.radio_model.model.blocks.30.mlp.fc2", "vision_model.radio_model.model.blocks.31.attn.qkv", "vision_model.radio_model.model.blocks.31.attn.proj", "vision_model.radio_model.model.blocks.31.mlp.fc1", "vision_model.radio_model.model.blocks.31.mlp.fc2", "mlp1.1", "mlp1.3" ], "kv_cache_scheme": { "actorder": null, "block_structure": null, "dynamic": false, "group_size": null, "num_bits": 8, "observer": "static_minmax", "observer_kwargs": {}, "scale_dtype": null, "strategy": "tensor", "symmetric": true, "type": "float", "zp_dtype": null }, "quant_method": "compressed-tensors", "quantization_status": "compressed", "sparsity_config": {}, "transform_config": {}, "version": "0.16.0" }, "template": "qwen3_5", "tie_word_embeddings": false, "transformers_version": "5.3.0", "use_thumbnail": true, "video_context_token": "<|video_pad|>", "video_context_token_id": 248057, "vision_config": { "adaptor_configs": {}, "adaptor_names": null, "architectures": [ "RADIOModel" ], "args": { "aa": null, "amp": true, "amp_dtype": "bfloat16", "amp_impl": "native", "aug_repeats": 0, "aug_splits": 0, "auto_workload_inspector": false, "bn_eps": null, "bn_momentum": null, "cache_dir": null, "channels_last": false, "checkpoint_folder": null, "checkpoint_hist": 10, "chk_keep_forever": 100, "class_map": "", "clip_grad": null, "clip_mode": "norm", "cls_token_per_teacher": true, "coco_annotations_file": null, "coco_image_dir": null, "color_jitter": 0.4, "cooldown_epochs": 0, "cpe_max_size": 2048, "cpe_num_registers": null, "crd_loss": false, "crd_loss_weight": 0.8, "crop_pct": null, "cutmix": 0.0, "cutmix_minmax": null, "dataset_download": false, "debug_full_knn": false, "decay_epochs": 90, "decay_milestones": [ 90, 180, 270 ], "decay_rate": 0.1, "depchain": true, "detect_anomaly": false, "dist_bn": "reduce", "dist_norm_weight": 0.0, "distributed": true, "drop": 0.0, "drop_block": null, "drop_connect": null, "drop_path": null, "dtype": "float32", "epoch": 299, "epoch_repeats": 0.0, "eval": false, "eval_metric": "knn_top1", "eval_teacher": false, "eval_teacher_only": false, "eval_throughput": false, "fast_norm": false, "fd_loss_fn": "MSE", "feature_normalization": "PHI_STANDARDIZE", "feature_summarizer": "cls_token", "feature_upscale_factor": null, "force_disable_damp": false, "force_disable_spectral_reparam": false, "force_new_wandb_id": false, "force_spectral_reparam": false, "freeze_bn": false, "fsdp": true, "full_equivariance": false, "fuser": "", "gp": null, "grad_accum_steps": 1, "grad_checkpointing": false, "head_init_bias": null, "head_init_scale": null, "head_lr": null, "head_warmup": 3, "head_weight_decay": 0.0005, "hflip": 0.5, "img_size": null, "in_chans": null, "initial_checkpoint": null, "input_size": null, "interpolation": "", "layer_decay": null, "local_rank": 0, "log_interval": 50, "log_mlflow": false, "log_teacher_timings": true, "log_train_metrics_per_epoch": true, "log_train_metrics_per_log_interval": true, "log_wandb": true, "loss_auto_balance": false, "lr_base": 0.1, "lr_base_scale": "", "lr_base_size": 256, "lr_cycle_decay": 0.5, "lr_cycle_limit": 1, "lr_cycle_mul": 1.0, "lr_k_decay": 1.0, "lr_noise": null, "lr_noise_pct": 0.67, "lr_noise_std": 1.0, "mean": null, "mesa": false, "min_lr": 1e-05, "mixup": 0.0, "mixup_mode": "batch", "mixup_off_epoch": 0, "mixup_prob": 1.0, "mixup_switch_prob": 0.5, "mlp_hidden_size": 1520, "mlp_num_inner": 2, "mlp_version": "v2", "model": "vit_huge_patch16_224", "model_kwargs": {}, "model_norm": false, "momentum": 0.9, "no_custom_validation": false, "no_ddp_bb": true, "no_knn": false, "no_prefetcher": false, "no_resume_opt": false, "no_save_checkpoint": false, "no_val": false, "num_classes": null, "on_demand_workload_inspector": false, "one_logger_app_tag": "", "one_logger_is_baseline": false, "one_logger_run_name": "", "onelogger": null, "opt_betas": null, "opt_eps": null, "overfit": false, "patience_epochs": 10, "perf_test_no_aug": false, "perf_test_no_decode": false, "perf_test_no_io": false, "perf_test_only_dataloader": false, "perf_test_simple_aug": false, "pin_mem": false, "prefetcher": true, "pretrained": false, "processed_neck_outputs": null, "profile_train_exit_after_profiling": false, "profile_train_export_chrome_trace": true, "profile_train_export_csv": false, "profile_train_iterations": 0, "qradio": false, "qradio_max_tokens": 512, "qradio_min_tokens": 32, "qradio_patch_token_mask_initial_ratio": 0.95, "qradio_progressive_2d": false, "qradio_quantizer": null, "qradio_ramp_alpha": 1.5, "rank": 0, "ratio": [ 0.75, 1.3333333333333333 ], "recount": 1, "recovery_interval": 0, "register_multiple": 10, "remode": "pixel", "reprob": 0.0, "reset_loss_state": true, "resplit": false, "sample_tracking": false, "save_images": false, "scale": [ 0.5, 1.0 ], "sched": "cosine", "seed": 42, "shift_equivariance": false, "smoothing": 0.1, "source_tracking": false, "spectral_heads": false, "spectral_reparam": false, "spectral_weight_decay": null, "split_bn": false, "start_epoch": null, "std": null, "stream_teachers": false, "student_intermediate_indices": null, "student_load_skip_state_dict_keys_regex": null, "student_reinit_model_layers_regex": null, "student_strict_load_ignore_mismatched_shape_keys_regex": null, "student_strict_load_ignore_missing_keys_regex": null, "student_strict_load_ignore_unexpected_keys_regex": null, "student_strict_load_state_dict": false, "sync_bn": false, "sync_resolutions_across_ranks": true, "synchronize_step": false, "teachers": [ { "model": "siglip2-g-384", "name": "siglip2-g", "spatial_mlp_version": "attn", "type": "siglip2", "use_summary": true }, { "model": "dinov3_vit7b16", "name": "dino_v3_7b", "type": "dino_v3", "use_summary": true }, { "model": "default", "name": "sam3", "type": "sam3", "use_summary": false } ], "timing_warmup_iters": 20, "tokenizer_kwargs": {}, "tokenizer_type": null, "tome": null, "torchcompile": null, "torchscript": false, "train_interpolation": "random", "train_split": "train", "tta": 0, "untie_neck_weights": false, "use_coco": false, "use_multi_epochs_loader": false, "val_ema_only": false, "val_split": "val", "vflip": 0.0, "vitdet_version": 1, "wandb_entity": "", "wandb_id": "", "wandb_job_type": "", "wandb_name": "", "wandb_project": "", "wandb_tags": null, "warmup_lr": 1e-05, "warmup_prefix": false, "worker_seeding": "all", "workers": 8, "workload_inspector_analyze_nsys_traces": false, "workload_inspector_baseline_start_iter": 1500, "workload_inspector_major_slowdown_p95_factor": 10.0, "workload_inspector_minor_slowdown_p95_factor": 3.0, "workload_inspector_no_slowdown_check": false, "workload_inspector_simulate_slowdown_num_times": 1, "workload_inspector_simulate_slowdown_start_iter": null, "world_size": 256 }, "auto_map": { "AutoConfig": "cradio_config.RADIOConfig", "AutoModel": "cradio_model.RADIOModel" }, "dtype": "bfloat16", "feature_normalizer_config": null, "inter_feature_normalizer_config": null, "max_resolution": 2048, "model_type": "radio", "patch_size": 16, "preferred_resolution": [ 512, 512 ], "use_flash_attn": false, "version": "c-radio_v4-h", "vitdet_window_size": null }, "vit_hidden_size": 1280 }