Hugging Face's logo

Arsh9210
/
Nemotron-3-Embed-1B-NVFP4

Sentence Similarity
Safetensors
vllm
ministral3
text
text-embeddings
retrieval
semantic-search
nvfp4
rag
8-bit precision
modelopt
Model card Files
xet
 Community
Arsh9210 commited on
Commit
d5c36b7
·
verified ·
1 Parent(s): 964e15a

Added quantization_metadata.json

Browse files
Files changed (1) hide show
  1. quantization_metadata.json +165 -0
quantization_metadata.json ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "model": {
4
+ "checkpoint": "/tmp/nemotron-3-embed-1b-bf16",
5
+ "trust_remote_code": true,
6
+ "dtype": "bfloat16",
7
+ "device": "cuda"
8
+ },
9
+ "calibration": {
10
+ "sources": [
11
+ {
12
+ "dataset": "abisee/cnn_dailymail",
13
+ "dataset_config": "3.0.0",
14
+ "split": "train",
15
+ "text_field": "article",
16
+ "num_samples": 256,
17
+ "prompt_prefix": "passage: ",
18
+ "seed": 42
19
+ },
20
+ {
21
+ "dataset": "abisee/cnn_dailymail",
22
+ "dataset_config": "3.0.0",
23
+ "split": "train",
24
+ "text_field": "highlights",
25
+ "num_samples": 256,
26
+ "prompt_prefix": "query: ",
27
+ "seed": 43
28
+ }
29
+ ],
30
+ "batch_size": 1,
31
+ "max_length": 16384,
32
+ "seed": 42
33
+ },
34
+ "quantization": {
35
+ "recipe": "mtq.NVFP4_DEFAULT_CFG",
36
+ "precision": "nvfp4",
37
+ "output_dir": "/tmp/nemotron-3-embed-1b-nvfp4",
38
+ "save_modelopt_state": false,
39
+ "export_hf_checkpoint": true
40
+ },
41
+ "quant_cfg_overrides": [
42
+ {
43
+ "quantizer_name": "*embed_tokens*",
44
+ "enable": false
45
+ },
46
+ {
47
+ "quantizer_name": "lm_head*",
48
+ "enable": false
49
+ }
50
+ ]
51
+ },
52
+ "quant_cfg": {
53
+ "quant_cfg": [
54
+ {
55
+ "quantizer_name": "*",
56
+ "enable": false
57
+ },
58
+ {
59
+ "quantizer_name": "*weight_quantizer",
60
+ "cfg": {
61
+ "num_bits": [
62
+ 2,
63
+ 1
64
+ ],
65
+ "block_sizes": {
66
+ "-1": 16,
67
+ "type": "dynamic",
68
+ "scale_bits": [
69
+ 4,
70
+ 3
71
+ ]
72
+ }
73
+ }
74
+ },
75
+ {
76
+ "quantizer_name": "*input_quantizer",
77
+ "cfg": {
78
+ "num_bits": [
79
+ 2,
80
+ 1
81
+ ],
82
+ "block_sizes": {
83
+ "-1": 16,
84
+ "type": "dynamic",
85
+ "scale_bits": [
86
+ 4,
87
+ 3
88
+ ]
89
+ }
90
+ }
91
+ },
92
+ {
93
+ "parent_class": "nn.BatchNorm1d",
94
+ "quantizer_name": "*",
95
+ "enable": false
96
+ },
97
+ {
98
+ "parent_class": "nn.BatchNorm2d",
99
+ "quantizer_name": "*",
100
+ "enable": false
101
+ },
102
+ {
103
+ "parent_class": "nn.BatchNorm3d",
104
+ "quantizer_name": "*",
105
+ "enable": false
106
+ },
107
+ {
108
+ "parent_class": "nn.LeakyReLU",
109
+ "quantizer_name": "*",
110
+ "enable": false
111
+ },
112
+ {
113
+ "quantizer_name": "*lm_head*",
114
+ "enable": false
115
+ },
116
+ {
117
+ "quantizer_name": "*proj_out.*",
118
+ "enable": false
119
+ },
120
+ {
121
+ "quantizer_name": "*block_sparse_moe.gate*",
122
+ "enable": false
123
+ },
124
+ {
125
+ "quantizer_name": "*router*",
126
+ "enable": false
127
+ },
128
+ {
129
+ "quantizer_name": "*mlp.gate.*",
130
+ "enable": false
131
+ },
132
+ {
133
+ "quantizer_name": "*mlp.shared_expert_gate.*",
134
+ "enable": false
135
+ },
136
+ {
137
+ "quantizer_name": "*linear_attn.conv1d*",
138
+ "enable": false
139
+ },
140
+ {
141
+ "quantizer_name": "*mixer.conv1d*",
142
+ "enable": false
143
+ },
144
+ {
145
+ "quantizer_name": "*output_layer*",
146
+ "enable": false
147
+ },
148
+ {
149
+ "quantizer_name": "output.*",
150
+ "enable": false
151
+ },
152
+ {
153
+ "quantizer_name": "*embed_tokens*",
154
+ "enable": false
155
+ },
156
+ {
157
+ "quantizer_name": "lm_head*",
158
+ "enable": false
159
+ }
160
+ ],
161
+ "algorithm": "max"
162
+ },
163
+ "quant_type": "nvfp4",
164
+ "modelopt_quantized": true
165
+ }