Upload 9 files

README.md

@@ -1,3 +1,100 @@
----
-license: apache-2.0
----
+# Hibernates-2B-R1-V1
+
+A highly efficient 2B parameter language model optimized for reasoning and dialogue tasks.
+
+## Model Overview
+
+Hibernates-2B is a custom transformer architecture designed for advanced language understanding and generation. Built with performance and efficiency in mind, it leverages state-of-the-art techniques for natural language processing.
+
+### Key Features
+- 2B Parameters
+- 4096 Token Context Window
+- Custom Transformer Architecture
+- Optimized for CPU and GPU Inference
+- Multi-Turn Dialogue Support
+
+## Technical Specifications
+
+- **Architecture**: Custom Transformer
+- **Parameters**: 2 Billion
+- **Context Length**: 4096 tokens
+- **Model Type**: Decoder-only
+- **Tokenizer**: Custom WordPiece
+- **Format**: SafeTensors
+
+## Usage Guide
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+# Load model and tokenizer
+model_id = "Hibernates-2B-R1-V1"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.float16,
+    device_map="auto"
+)
+
+# Example conversation
+messages = [
+    {"role": "system", "content": "You are a helpful AI assistant."},
+    {"role": "user", "content": "How can you help me today?"}
+]
+
+# Generate response
+input_text = tokenizer.apply_chat_template(messages, tokenize=False)
+inputs = tokenizer(input_text, return_tensors="pt").to(model.device)
+outputs = model.generate(
+    inputs["input_ids"],
+    max_new_tokens=512,
+    temperature=0.7,
+    top_p=0.95
+)
+response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+```
+
+## Performance Characteristics
+
+### Strengths
+- Efficient Resource Usage
+- Strong Reasoning Capabilities
+- Multi-Turn Dialogue
+- Context Awareness
+- Instruction Following
+
+### Considerations
+- Resource Requirements: 8GB+ GPU RAM recommended
+- Task Specificity: Best suited for dialogue and reasoning tasks
+- Language Support: Primary focus on English
+- Model Size: Optimized for balance of performance and efficiency
+
+## License and Usage
+- Research and commercial use permitted
+- Attribution appreciated but not required
+- No warranty provided
+
+## Citation
+
+If you use this model in your research, please cite:
+
+```bibtex
+@software{hibernates2b_2024,
+  title={Hibernates-2B: Efficient Language Model for Reasoning},
+  year={2024},
+  version={R1-V1}
+}
+```
+
+## Acknowledgments
+Built using PyTorch and Hugging Face Transformers. Special thanks to the open-source AI community.
+
+## Download Instructions
+
+Due to file size limitations, the model files are hosted externally. Download them from:
+
+1. [model-00001-of-00002.safetensors](https://huggingface.co/HibernatesAI/Hibernates-2B-R1-V1/blob/main/model-00001-of-00002.safetensors)
+2. [model-00002-of-00002.safetensors](https://huggingface.co/HibernatesAI/Hibernates-2B-R1-V1/blob/main/model-00002-of-00002.safetensors)
+
+Place these files in the root directory of the project before running.

config.json

@@ -0,0 +1,38 @@
+{
+    "_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 128000,
+    "eos_token_id": 128009,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 3072,
+    "initializer_range": 0.02,
+    "intermediate_size": 8192,
+    "max_position_embeddings": 131072,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "num_attention_heads": 24,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "pad_token_id": 128004,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "factor": 32.0,
+      "high_freq_factor": 4.0,
+      "low_freq_factor": 1.0,
+      "original_max_position_embeddings": 8192,
+      "rope_type": "llama3"
+    },
+    "rope_theta": 500000.0,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.47.1",
+    "use_cache": true,
+    "vocab_size": 128256
+  }
+  

generation_config.json

@@ -0,0 +1,18 @@
+{
+    "bos_token_id": 1,
+    "do_sample": true,
+    "eos_token_id": [2, 3, 4],
+    "max_length": 4096,
+    "pad_token_id": 0,
+    "temperature": 0.8,
+    "top_p": 0.95,
+    "top_k": 40,
+    "repetition_penalty": 1.15,
+    "length_penalty": 1.0,
+    "no_repeat_ngram_size": 3,
+    "num_beam_groups": 1,
+    "diversity_penalty": 0.0,
+    "early_stopping": true,
+    "transformers_version": "4.47.1"
+  }
+  

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c95bf0efd733da5847e1762a7d7317a7c2f84b1411ba5f008656597eb6c8b200
+size 135

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22b8ef63d734c9dbe7d8ac14e299623741ef696474808d8ca69a0adc7eb9617b
+size 135

model.safetensors.index.json

@@ -0,0 +1,17 @@
+{
+    "metadata": {
+      "model_type": "custom",
+      "total_size": 2000000000,
+      "framework": "pytorch",
+      "format": "safetensors",
+      "model_version": "1.0.0",
+      "creation_date": "2024",
+      "architecture": "transformer",
+      "quantization": null
+    },
+    "weight_map": {
+      "model.layers.0": "model-00001-of-00002.safetensors",
+      "model.layers.1": "model-00002-of-00002.safetensors"
+    }
+  }
+  

special_tokens_map.json

@@ -0,0 +1,14 @@
+{
+    "bos_token": "<|start|>",
+    "eos_token": "<|end|>",
+    "pad_token": "<|pad|>",
+    "unk_token": "<|unk|>",
+    "mask_token": "<|mask|>",
+    "sep_token": "<|sep|>",
+    "additional_special_tokens": [
+      "<|system|>",
+      "<|user|>",
+      "<|assistant|>"
+    ]
+  }
+  

tokenizer.json

@@ -0,0 +1,36 @@
+{
+    "version": 1,
+    "truncation": {
+      "max_length": 4096,
+      "strategy": "longest_first",
+      "direction": "right"
+    },
+    "padding": {
+      "strategy": "max_length",
+      "side": "left",
+      "length": null
+    },
+    "added_tokens": [],
+    "normalizer": {
+      "type": "BertNormalizer",
+      "clean_text": true,
+      "handle_chinese_chars": true,
+      "strip_accents": true,
+      "lowercase": true
+    },
+    "pre_tokenizer": {
+      "type": "Whitespace"
+    },
+    "post_processor": null,
+    "decoder": {
+      "type": "WordPiece",
+      "cleanup": true
+    },
+    "model": {
+      "type": "WordPiece",
+      "unk_token": "[UNK]",
+      "continuing_subword_prefix": "##",
+      "max_input_chars_per_word": 100
+    }
+  }
+  

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+    "name_or_path": "Hibernates-2B-R1-V1",
+    "padding_side": "left",
+    "truncation_side": "right",
+    "model_max_length": 4096,
+    "use_fast": true,
+    "clean_up_tokenization_spaces": true,
+    "model_type": "custom",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "do_lower_case": false,
+    "unicode_normalizer": "nfkc",
+    "special_tokens_map_file": "special_tokens_map.json",
+    "tokenizer_class": "PreTrainedTokenizerFast"
+  }
+