Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-405B-Instruct-quantized.w8a8

Text Generation
Safetensors
llama
int8
vllm
conversational
8-bit precision
compressed-tensors
Model card Files Community
alexmarques commited on
Commit
57e82a4
·
verified ·
1 Parent(s): bb7190f

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +79 -0
  2. generation_config.json +12 -0
  3. model-00001-of-00086.safetensors +3 -0
  4. model-00002-of-00086.safetensors +3 -0
  5. model-00003-of-00086.safetensors +3 -0
  6. model-00004-of-00086.safetensors +3 -0
  7. model-00005-of-00086.safetensors +3 -0
  8. model-00006-of-00086.safetensors +3 -0
  9. model-00007-of-00086.safetensors +3 -0
  10. model-00008-of-00086.safetensors +3 -0
  11. model-00009-of-00086.safetensors +3 -0
  12. model-00010-of-00086.safetensors +3 -0
  13. model-00011-of-00086.safetensors +3 -0
  14. model-00012-of-00086.safetensors +3 -0
  15. model-00013-of-00086.safetensors +3 -0
  16. model-00014-of-00086.safetensors +3 -0
  17. model-00015-of-00086.safetensors +3 -0
  18. model-00016-of-00086.safetensors +3 -0
  19. model-00017-of-00086.safetensors +3 -0
  20. model-00018-of-00086.safetensors +3 -0
  21. model-00019-of-00086.safetensors +3 -0
  22. model-00020-of-00086.safetensors +3 -0
  23. model-00021-of-00086.safetensors +3 -0
  24. model-00022-of-00086.safetensors +3 -0
  25. model-00023-of-00086.safetensors +3 -0
  26. model-00024-of-00086.safetensors +3 -0
  27. model-00025-of-00086.safetensors +3 -0
  28. model-00026-of-00086.safetensors +3 -0
  29. model-00027-of-00086.safetensors +3 -0
  30. model-00028-of-00086.safetensors +3 -0
  31. model-00029-of-00086.safetensors +3 -0
  32. model-00030-of-00086.safetensors +3 -0
  33. model-00031-of-00086.safetensors +3 -0
  34. model-00032-of-00086.safetensors +3 -0
  35. model-00033-of-00086.safetensors +3 -0
  36. model-00034-of-00086.safetensors +3 -0
  37. model-00035-of-00086.safetensors +3 -0
  38. model-00036-of-00086.safetensors +3 -0
  39. model-00037-of-00086.safetensors +3 -0
  40. model-00038-of-00086.safetensors +3 -0
  41. model-00039-of-00086.safetensors +3 -0
  42. model-00040-of-00086.safetensors +3 -0
  43. model-00041-of-00086.safetensors +3 -0
  44. model-00042-of-00086.safetensors +3 -0
  45. model-00043-of-00086.safetensors +3 -0
  46. model-00044-of-00086.safetensors +3 -0
  47. model-00045-of-00086.safetensors +3 -0
  48. model-00046-of-00086.safetensors +3 -0
  49. model-00047-of-00086.safetensors +3 -0
  50. model-00048-of-00086.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3.1-405B-Instruct/snapshots/069992c75aed59df00ec06c17177e76c63296a26",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "compression_config": {
10
+ "config_groups": {
11
+ "group_0": {
12
+ "input_activations": {
13
+ "block_structure": null,
14
+ "dynamic": true,
15
+ "group_size": null,
16
+ "num_bits": 8,
17
+ "observer": "memoryless",
18
+ "observer_kwargs": {},
19
+ "strategy": "token",
20
+ "symmetric": true,
21
+ "type": "int"
22
+ },
23
+ "output_activations": null,
24
+ "targets": [
25
+ "Linear"
26
+ ],
27
+ "weights": {
28
+ "block_structure": null,
29
+ "dynamic": false,
30
+ "group_size": null,
31
+ "num_bits": 8,
32
+ "observer": "minmax",
33
+ "observer_kwargs": {},
34
+ "strategy": "channel",
35
+ "symmetric": true,
36
+ "type": "int"
37
+ }
38
+ }
39
+ },
40
+ "format": "int-quantized",
41
+ "global_compression_ratio": 1.240844678218891,
42
+ "ignore": [
43
+ "lm_head"
44
+ ],
45
+ "kv_cache_scheme": null,
46
+ "quant_method": "compressed-tensors",
47
+ "quantization_status": "frozen"
48
+ },
49
+ "eos_token_id": [
50
+ 128001,
51
+ 128008,
52
+ 128009
53
+ ],
54
+ "hidden_act": "silu",
55
+ "hidden_size": 16384,
56
+ "initializer_range": 0.02,
57
+ "intermediate_size": 53248,
58
+ "max_position_embeddings": 131072,
59
+ "mlp_bias": false,
60
+ "model_type": "llama",
61
+ "num_attention_heads": 128,
62
+ "num_hidden_layers": 126,
63
+ "num_key_value_heads": 8,
64
+ "pretraining_tp": 1,
65
+ "rms_norm_eps": 1e-05,
66
+ "rope_scaling": {
67
+ "factor": 8.0,
68
+ "high_freq_factor": 4.0,
69
+ "low_freq_factor": 1.0,
70
+ "original_max_position_embeddings": 8192,
71
+ "rope_type": "llama3"
72
+ },
73
+ "rope_theta": 500000.0,
74
+ "tie_word_embeddings": false,
75
+ "torch_dtype": "bfloat16",
76
+ "transformers_version": "4.44.0",
77
+ "use_cache": true,
78
+ "vocab_size": 128256
79
+ }
generation_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 128000,
3
+ "do_sample": true,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "temperature": 0.6,
10
+ "top_p": 0.9,
11
+ "transformers_version": "4.44.0"
12
+ }
model-00001-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea010c5783472f5054bd29a8b610a89262f2f08a3b5ff4747a473712d2ff72cd
3
+ size 4773188672
model-00002-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8294bf71dba6a64b31b5302bd20702bdba44c306346e2e95516f3c4662f0f96
3
+ size 4933097696
model-00003-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a01d047ed16621fefb7f0ce3d9ce4c265e08fc30715b0ad4abe2389755292b6e
3
+ size 4631063664
model-00004-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39424f7f13c564e66867edb9afa36793806b5ebf3c1de594752eaab34d5e68e1
3
+ size 4933097696
model-00005-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56e37bc0706333511061f87a256b85f987300eac56ceaa095ef20580357eb9eb
3
+ size 4631063664
model-00006-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4e02212752e4c5e8687535f7c7df097accd650d7fd1a35bc63c61acce06ec8e
3
+ size 4933097696
model-00007-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13b61ac6531dbf77c8f57d886139b6117d13704f565b22efdf4ead7ea6e09674
3
+ size 4631063664
model-00008-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b5b8d20229616dee53612868363f46696bac1368e1ff027ee4eff48f1af4c66
3
+ size 4933097712
model-00009-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c975f8767573ab377d79b106659e65dc3e2f8c7ca55ad0d861105288aafd460
3
+ size 4631063696
model-00010-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf243cb35eec84746802775bd962dfce5e08d7aaaa26a5d9783102f974370ea0
3
+ size 4933097720
model-00011-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6022c330ca40b4b241ec3d221fbee3d834b05f5dc9501539c7658281d87d245c
3
+ size 4631063696
model-00012-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ff9e16eeed7ceb7aff827ef3ba2550456d38e62344318bfb3250cedf75b605b
3
+ size 4933097720
model-00013-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c14cf9374168796da552383fce326fb00049a4a71f3dd6ee0ed698f1b70c40bc
3
+ size 4631063696
model-00014-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:036ac51f512088df13805723bc0e247218e9fc5050610c305086b3b300feaea7
3
+ size 4933097720
model-00015-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:443a8a72dd4ab66f5611337d423e3ac272bf0cb464b253f1ecd2717285122837
3
+ size 4631063696
model-00016-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9cc470c71bd7e580b6e72b60172569281b2baff94c0bc48604f122940978b3bb
3
+ size 4933097720
model-00017-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dfd8e7dc49a9e5d58887262748ed824d9b1b36db00b4a0813b2a4c8a14f719d7
3
+ size 4631063696
model-00018-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7487876798e9e78b957a4527809abd63fd7e48943164821b57c8ac436da0cf53
3
+ size 4933097720
model-00019-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:66cfa9ef32962d7e85efceb062da357f09adc3dd8d15a6a3ade9c51aa8f81a61
3
+ size 4631063696
model-00020-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:744b7c40738cde5ff4fa387b155611389a147503044602446e8c52155801b3d8
3
+ size 4933097720
model-00021-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:479541751ceea5b2d1f4a32216f705d162075c5f419be6d13394aa90b3e635ea
3
+ size 4631063696
model-00022-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:226b1c85ca6629c3483c6774ea617632998f6dcea44c3f6540698c2338399ed6
3
+ size 4933097720
model-00023-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5258191e504857ffab41ec71d5291239a6fa77a9caf6cde11311852d0d81848
3
+ size 4631063696
model-00024-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7cf0266e7cd605b95fa01a278e8d08ab277df83dd129cf041c55773c2035c50c
3
+ size 4933097720
model-00025-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b000533266f2f1f529a270306606249b99ca4fdb3548db64beab1ebc80d16cd1
3
+ size 4631063696
model-00026-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec687a89fa6c17c52e34a213f3b5c54e5f3a2be45721c74e34b6f894a45cd4fc
3
+ size 4933097720
model-00027-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03175661bc6452a8c1b8c9ae0dc15a480f43c3f74c48f0126bca22215b956874
3
+ size 4631063696
model-00028-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff81a402a33464b5b26689e5fd2de31c60b58220d5f6bbe1c806caf6369b1e4d
3
+ size 4933097720
model-00029-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24bccde6a37a1e8d80942500fd41c1cef67c50148b3c6e020f3aad9fe211c793
3
+ size 4631063696
model-00030-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:705135282b416b3d44ccb850539f7702cb3c7b5fe6c4a7e21d0bd94f275fc222
3
+ size 4933097720
model-00031-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:752b7ecc260ef733ea9caac9f824dcdc0a8754408638880ebf05a61c87ec865a
3
+ size 4631063696
model-00032-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0eb0742f38b9bd7518076a4742bea6b59fd14f3a997e69fafa749387568d852
3
+ size 4933097720
model-00033-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:58949d9d98bd71dde35355b9740473d942f1edfddf38239c03e1d5de6a4d08b8
3
+ size 4631063696
model-00034-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:548d49da1c22661d106e16ec3b2a89a52c19f4d5abaf593598055127fb3b8c83
3
+ size 4933097720
model-00035-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a446707175b74a3c83629704f69ff082634bf0f52a7e5740d8da69e9311dbac3
3
+ size 4631063696
model-00036-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf4db9a97c1b62ce04ca1d3a0a9a47e167b3e6404ccef28797407409d78c9d88
3
+ size 4933097720
model-00037-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de247bc8317aaac3b3215706f27fc93ecebd5674f9e24774d9f2896edece3811
3
+ size 4631063696
model-00038-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12f7db2e549e011450b1e22b95f5f27e24390f18d97949b7c48a20f22da7544d
3
+ size 4933097720
model-00039-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d8840eae1199b825b32b58222c34cabbeb1bcfe6f72d2118cef2cfbc9f8d005f
3
+ size 4631063696
model-00040-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:deb04f7a1e4d2198cc5870869a9abf633120e0fbf3c4b657f1d675f6dfb6d5bd
3
+ size 4933097720
model-00041-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:260643f8191597c66af71a9183e1313cefb2e2591fb481c927a2eca47ff27c7a
3
+ size 4631063696
model-00042-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6522e27b342810bba342e03266b097d2df3c5ffd2faa1c2bada09bb568a1bc4c
3
+ size 4933097720
model-00043-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c886451abb77758b45239ff49172c197176cb14b5cc2408507a01d222e097de
3
+ size 4631063696
model-00044-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5bf8b7d789a390590ce60626f98f3e127c708b5a7d60c1a2b987d5b7d663423
3
+ size 4933097720
model-00045-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c48b512bac361c641bad6d774d6c0f50890cc416bb20881b2d347bb3d7688344
3
+ size 4631063696
model-00046-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ae2f78ff87a246c28a80665a0e551dbea6ccb437866ff8d23e5d182d1708e15
3
+ size 4933097720
model-00047-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddf16ee3adeab6124a939c14211c16476ce114c34438a03ae64ea03f3113faa8
3
+ size 4631063696
model-00048-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8c490fb4bdaf05a732d7d6b4a90ad44e7730d4450bc8a0235acb4c3b51ce0b0
3
+ size 4933097720