Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-70B-Instruct-quantized.w8a8

Text Generation
Transformers
Safetensors
llama
int8
vllm
conversational
text-generation-inference
Inference Endpoints
8-bit precision
compressed-tensors
Model card Files Community
3
alexmarques commited on
Commit
c1e6631
·
verified ·
1 Parent(s): 1677fbe

Upload folder using huggingface_hub

Browse files
Files changed (18) hide show
  1. config.json +2 -2
  2. generation_config.json +1 -1
  3. model-00001-of-00015.safetensors +1 -1
  4. model-00002-of-00015.safetensors +1 -1
  5. model-00003-of-00015.safetensors +1 -1
  6. model-00004-of-00015.safetensors +1 -1
  7. model-00005-of-00015.safetensors +1 -1
  8. model-00006-of-00015.safetensors +1 -1
  9. model-00007-of-00015.safetensors +1 -1
  10. model-00008-of-00015.safetensors +1 -1
  11. model-00009-of-00015.safetensors +1 -1
  12. model-00010-of-00015.safetensors +1 -1
  13. model-00011-of-00015.safetensors +1 -1
  14. model-00012-of-00015.safetensors +1 -1
  15. model-00013-of-00015.safetensors +1 -1
  16. model-00014-of-00015.safetensors +1 -1
  17. model-00015-of-00015.safetensors +1 -1
  18. recipe.yaml +12 -2
config.json CHANGED
@@ -47,7 +47,7 @@
47
  "quantization_status": "frozen",
48
  "sparsity_config": {
49
  "format": "dense",
50
- "global_sparsity": 2.690222063822556,
51
  "registry_requires_subclass": false,
52
  "sparsity_structure": "unstructured"
53
  }
@@ -79,7 +79,7 @@
79
  "rope_theta": 500000.0,
80
  "tie_word_embeddings": false,
81
  "torch_dtype": "bfloat16",
82
- "transformers_version": "4.43.2",
83
  "use_cache": true,
84
  "vocab_size": 128256
85
  }
 
47
  "quantization_status": "frozen",
48
  "sparsity_config": {
49
  "format": "dense",
50
+ "global_sparsity": 2.0350219291192033,
51
  "registry_requires_subclass": false,
52
  "sparsity_structure": "unstructured"
53
  }
 
79
  "rope_theta": 500000.0,
80
  "tie_word_embeddings": false,
81
  "torch_dtype": "bfloat16",
82
+ "transformers_version": "4.43.3",
83
  "use_cache": true,
84
  "vocab_size": 128256
85
  }
generation_config.json CHANGED
@@ -6,5 +6,5 @@
6
  128008,
7
  128009
8
  ],
9
- "transformers_version": "4.43.2"
10
  }
 
6
  128008,
7
  128009
8
  ],
9
+ "transformers_version": "4.43.3"
10
  }
model-00001-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a2d1d7a98a7c848890714e23b8e53a5a42f0673068942aa0aa959ef07bee3ef6
3
  size 4819900928
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:11414693dd88807a2cc34159e92c2f98b0060ad12bbcb853341f740780dd2b5d
3
  size 4819900928
model-00002-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:eb3d319b631094c753d933766e7100513c6550710e654b51fe8b7d09ee3b24bd
3
  size 4984010360
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ffa38ea00dd7b9337441231c30a840201057dffee3ffb3df7031ecbb2a11c16
3
  size 4984010360
model-00003-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ebc24b65ef6599067eb3d6c19c7402b80245740f4e9e564c67b0522ac56a8016
3
  size 4900112536
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5bf94685a5adf09a38035e54c2bfbaba872ebec426bb2aa02e76bf5a00f2b9a3
3
  size 4900112536
model-00004-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:37fe8142a2f5156d971c096b5c2fb24d37161ad15e1054cfa4a03c3c74a7fa4f
3
  size 4900104584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1abde96cfe492fc47303c5d54f7af9d32dc0b80c0814b1934e58f5c280606938
3
  size 4900104584
model-00005-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:cad322ef15ff22b046a6231483fea6eeb2558700cc0eff167d2ec43c23330a3a
3
  size 4900104584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09db1268e643ac958c5d340e7629ea8d2664c22bc5b63f919130642214269d8a
3
  size 4900104584
model-00006-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d03f12b4570556399e3e660f21a3393a9b22d3c50ddc970446be0f48a3cac018
3
  size 4984010448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8e285e7ab13a54a0d3723cd3e39c7a0ad21f066c425e2123b6e23bfe3c29eca4
3
  size 4984010448
model-00007-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:cab2f22a216417dddd346b11b336ff03d0d3c4fad544c30f0d286b0c5c653d03
3
  size 4900112552
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:45836d673a45a7b336ab9eecf17d6b50264e62d61d15d1161d6edf4152fc35db
3
  size 4900112552
model-00008-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4ce5dac4f45fd0870aed75d1486577e870d30479c9fdfa4aa036c09e0f21c8cb
3
  size 4900104584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb7cb74df9582c752075c28e9de6147c4cff1967585cb2d90e392399af53aa3c
3
  size 4900104584
model-00009-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f34025c5f810b05b49afc3a87d200665a17ce1d167a651e66fecbefb705b139b
3
  size 4900104584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c8d6bc4fe62197d4b933910140420b9b457444d91daa1b609ca0c2dfca7910a
3
  size 4900104584
model-00010-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6e8b44d155a0f596698f845371e608618f5f92eab2950c82cc25e3ec371f60d2
3
  size 4984010448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29ca07e2a2084e98a3e7d05b361721ed4d4b71ebf07a5a565d37de69c3f14348
3
  size 4984010448
model-00011-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:a5980457cd5544259f2de1b2985147d4d930a83cac4f0db6f93ed82b8ce95d70
3
  size 4900112552
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:996d4539fcd59d7bca288c2d64221ba6359b8275ba2d595511a10c105a5b812d
3
  size 4900112552
model-00012-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5a52b4fadfc94877c65f6a936c48d019ff5ec5cc1abf7ab841eab02be47d4e10
3
  size 4900104584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ee5cb9888eb4cfb7685184267e04c971766f79fc4cb5f974bfe70777f10f9fe
3
  size 4900104584
model-00013-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:01e54f41f1e41101b9b304ff7886ca31703fa0cc7418869ad0cb93fe7e5dd310
3
  size 4900104584
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e495362d49634554dee2b498f7eca1dcd0c4bb7bf48d6145098f260e203042a
3
  size 4900104584
model-00014-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:37d183c555d1a40d43785429718f22faac7e499628eebd3dd5e4b207da845344
3
  size 4984010448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e202bcf3246adf2d3223fa964ff489774a4a36a5fb5df01e32393f0c4e89c94
3
  size 4984010448
model-00015-of-00015.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1ebc27a71f7105330f942e55dba26189dabe5b38bcba5134de70b77c2bb46005
3
  size 3813044096
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6774db5aae488ddcd66cff453787a1481198d4a4b85cce3fc12d9e5e8bf5f621
3
  size 3813044096
recipe.yaml CHANGED
@@ -1,8 +1,18 @@
1
  quant_stage:
2
  quant_modifiers:
 
 
 
 
 
 
 
 
 
3
  GPTQModifier:
4
  sequential_update: false
5
- dampening_frac: 0.1
6
  ignore: [lm_head]
7
  scheme: W8A8
8
- targets: Linear
 
 
1
  quant_stage:
2
  quant_modifiers:
3
+ SmoothQuantModifier:
4
+ smoothing_strength: 0.7
5
+ mappings:
6
+ - - ['re:.*q_proj', 're:.*k_proj', 're:.*v_proj']
7
+ - re:.*input_layernorm
8
+ - - ['re:.*gate_proj', 're:.*up_proj']
9
+ - re:.*post_attention_layernorm
10
+ - - ['re:.*down_proj']
11
+ - re:.*up_proj
12
  GPTQModifier:
13
  sequential_update: false
14
+ dampening_frac: 0.2
15
  ignore: [lm_head]
16
  scheme: W8A8
17
+ targets: [Linear]
18
+ observer: mse