asas

config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "OBILanguageModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "modelConfig.OBIConfig",
+    "AutoModel": "modelLM.OBILanguageModel",
+    "AutoModelForCausalLM": "modelLM.OBILanguageModel",
+    "AutoModelForQuestionAnswering": "modelLM.OBILanguageModel"
+  },
+  "batch_size": 5,
+  "block_size": 20,
+  "device": "cpu",
+  "eval_interval": 100,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 4,
+  "learning_rate": 0.001,
+  "max_iters": 900,
+  "num_attention_heads": 2,
+  "num_hidden_layers": 2,
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "vocab_size": 92656
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b20b5dbce3759e8c4e5bd443cff00d5487468d7a5a519f63aa5c05ab4eea6e22
+size 3362505

tokenizeConfig.py

@@ -0,0 +1,49 @@
+from tokenizers import Tokenizer, models, pre_tokenizers, trainers, decoders
+import json
+
+class OBITokenizer:
+    def __init__(self):
+        # Initialize a BPE model for tokenization
+        bpe_model = models.BPE()
+        # Initialize the tokenizer
+        self.tokenizer = Tokenizer(bpe_model)
+        # Add pre-tokenization and decoding steps if needed
+        self.tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
+        self.tokenizer.decoder = decoders.ByteLevel()
+
+    def train(self, files,save_path):
+        # Training: Fit the tokenizer on your text data
+        trainer = trainers.BpeTrainer(special_tokens=["[PAD]", "[CLS]", "[SEP]", "[MASK]", "[UNK]"])
+        self.tokenizer.train(trainer=trainer, files=files)
+        # Save the trained tokenizer to a file
+        self.tokenizer.save(save_path)
+
+
+
+    def save_config(self, config_file):
+        # Serialize the tokenizer's config to a JSON file
+        config_dict = {
+            "tokenizer_type": "custom",
+            "vocab_size": self.tokenizer.get_vocab_size(),
+            "tokenizer_class": "OBITokenizer",
+            "auto_map": {
+                "AutoTokenizer": ["tokenizeConfig.OBITokenizer"]
+            },
+            "bos_token": "[CLS]",
+            "eos_token": "[SEP]",
+            "unk_token": "[UNK]",
+            "pad_token": "[PAD]",
+            "mask_token": "[MASK]"
+            # Add other custom settings if needed
+        }
+        with open(config_file, "w") as f:
+            json.dump(config_dict, f)
+
+    def encode(self, text):
+        # Encode text using the custom tokenizer
+        encoding = self.tokenizer.encode(text)
+        return encoding.ids
+
+    def decode(self, ids):
+        # Decode IDs to text using the custom tokenizer
+        return self.tokenizer.decode(ids)

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "tokenizer_type": "custom",
+  "vocab_size": 10187,
+  "tokenizer_class": "OBITokenizer",
+  "auto_map": { "AutoTokenizer": ["tokenizeConfig.OBITokenizer"] },
+  "bos_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "unk_token": "[UNK]",
+  "pad_token": "[PAD]",
+  "mask_token": "[MASK]"
+}