Rafii
/

f1llama

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/anaconda3/envs/f1llama/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "Fetching 7 files: 100%|██████████| 7/7 [00:00<00:00, 92038.02it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from mlx_lm import load, generate\n",
+    "model, tokenizer = load(\"mlx-community/Meta-Llama-3-8B-Instruct-8bit\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Surprising Facts About F1, title:Surprising Facts About F1, type:article, url:/surprising-facts-about-f1/,}\\nSurprising Facts About F1\\nFormula 1 (F1) is one of the most popular and thrilling forms of motorsport, with millions of fans around the world. However, there are many surprising facts about F1 that even the most dedicated fans may not know. Here are some of the most interesting and surprising facts about F1:\\n\\n'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "generate(model=model, tokenizer=tokenizer, prompt=\"role:Tell me something surprising about f1, content:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "train_model() missing 1 required positional argument: 'args'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[7], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmlx_lm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m lora\n\u001b[0;32m----> 3\u001b[0m \u001b[43mlora\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtrain_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfine_tune_train.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43mvalid_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfine_tune_test.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mTypeError\u001b[0m: train_model() missing 1 required positional argument: 'args'"
+     ]
+    }
+   ],
+   "source": [
+    "from mlx_lm import lora\n",
+    "\n",
+    "lora.train_model(\n",
+    "    model=model, \n",
+    "    tokenizer=tokenizer,\n",
+    "    train_set=\"fine_tune_train.jsonl\",\n",
+    "    valid_set=\"fine_tune_test.jsonl\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Help on function train_model in module mlx_lm.lora:\n",
+      "\n",
+      "train_model(args, model: mlx.nn.layers.base.Module, tokenizer: mlx_lm.tokenizer_utils.TokenizerWrapper, train_set, valid_set, training_callback: mlx_lm.tuner.trainer.TrainingCallback = None)\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "help(lora.train_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "train_model() got an unexpected keyword argument 'lora_params'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[18], line 6\u001b[0m\n\u001b[1;32m      1\u001b[0m lora_params \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m      2\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlora_rank\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;241m8\u001b[39m,  \u001b[38;5;66;03m# Rank of the LoRA adapters\u001b[39;00m\n\u001b[1;32m      3\u001b[0m     \u001b[38;5;66;03m# Add other parameters as needed\u001b[39;00m\n\u001b[1;32m      4\u001b[0m }\n\u001b[0;32m----> 6\u001b[0m \u001b[43mlora\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtrain_set\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfine_tune_train.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlora_params\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlora_params\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m3\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Number of training epochs\u001b[39;49;00m\n\u001b[1;32m     11\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m32\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Batch size\u001b[39;49;00m\n\u001b[1;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlearning_rate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1e-5\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Learning rate\u001b[39;49;00m\n\u001b[1;32m     13\u001b[0m \u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mTypeError\u001b[0m: train_model() got an unexpected keyword argument 'lora_params'"
+     ]
+    }
+   ],
+   "source": [
+    "lora_params = {\n",
+    "    'lora_rank': 8,  # Rank of the LoRA adapters\n",
+    "    # Add other parameters as needed\n",
+    "}\n",
+    "\n",
+    "lora.train_model(\n",
+    "    model=model,\n",
+    "    train_set=\"fine_tune_train.jsonl\",\n",
+    "    lora_params=lora_params,\n",
+    "    epochs=3,  # Number of training epochs\n",
+    "    batch_size=32,  # Batch size\n",
+    "    learning_rate=1e-5,  # Learning rate\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Starting training..., iters: 100\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[15], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmlx\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01moptimizers\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01moptim\u001b[39;00m\n\u001b[1;32m      2\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m optim\u001b[38;5;241m.\u001b[39mAdam(learning_rate\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1e-3\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mlora\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtrain_dataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfine_tune_train.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m    \u001b[49m\u001b[43mval_dataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfine_tune_test.jsonl\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moptimizer\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/f1llama/lib/python3.10/site-packages/mlx_lm/tuner/trainer.py:242\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(model, tokenizer, optimizer, train_dataset, val_dataset, args, loss, iterate_batches, training_callback)\u001b[0m\n\u001b[1;32m    240\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m it \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m it \u001b[38;5;241m%\u001b[39m args\u001b[38;5;241m.\u001b[39msteps_per_eval \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m it \u001b[38;5;241m==\u001b[39m args\u001b[38;5;241m.\u001b[39miters:\n\u001b[1;32m    241\u001b[0m     stop \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mperf_counter()\n\u001b[0;32m--> 242\u001b[0m     val_loss \u001b[38;5;241m=\u001b[39m \u001b[43mevaluate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    243\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    244\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mval_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    245\u001b[0m \u001b[43m        \u001b[49m\u001b[43mloss\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mloss\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    246\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtokenizer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    247\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    248\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnum_batches\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mval_batches\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    249\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmax_seq_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_seq_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    250\u001b[0m \u001b[43m        \u001b[49m\u001b[43miterate_batches\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miterate_batches\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    251\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    252\u001b[0m     val_time \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mperf_counter() \u001b[38;5;241m-\u001b[39m stop\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m rank \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/f1llama/lib/python3.10/site-packages/mlx_lm/tuner/trainer.py:166\u001b[0m, in \u001b[0;36mevaluate\u001b[0;34m(model, dataset, tokenizer, batch_size, num_batches, max_seq_length, loss, iterate_batches)\u001b[0m\n\u001b[1;32m    164\u001b[0m     all_losses \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m losses \u001b[38;5;241m*\u001b[39m toks\n\u001b[1;32m    165\u001b[0m     ntokens \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m toks\n\u001b[0;32m--> 166\u001b[0m     \u001b[43mmx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43meval\u001b[49m\u001b[43m(\u001b[49m\u001b[43mall_losses\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mntokens\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    168\u001b[0m all_losses \u001b[38;5;241m=\u001b[39m mx\u001b[38;5;241m.\u001b[39mdistributed\u001b[38;5;241m.\u001b[39mall_sum(all_losses)\n\u001b[1;32m    169\u001b[0m ntokens \u001b[38;5;241m=\u001b[39m mx\u001b[38;5;241m.\u001b[39mdistributed\u001b[38;5;241m.\u001b[39mall_sum(ntokens)\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "import mlx.optimizers as optim\n",
+    "optimizer = optim.Adam(learning_rate=1e-3)\n",
+    "lora.train(model=model, \n",
+    "    tokenizer=tokenizer,\n",
+    "    train_dataset=\"fine_tune_train.jsonl\",\n",
+    "    val_dataset=\"fine_tune_test.jsonl\",\n",
+    "    optimizer=optimizer)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Trainable parameters: 0.085% (6.816M/8030.261M)\n",
+      "Starting training..., iters: 10\n",
+      "Iter 1: Val loss 14.203, Val took 3.526s\n",
+      "Iter 10: Val loss 7.847, Val took 1.549s\n",
+      "Iter 10: Train loss 10.478, Learning Rate 1.000e-05, It/sec 14.209, Tokens/sec 113.670, Trained Tokens 80, Peak mem 36.209 GB\n",
+      "Saved final weights to adapters.safetensors.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "import mlx.optimizers as optim\n",
+    "from mlx_lm import lora\n",
+    "from mlx_lm import load, generate,\n",
+    "\n",
+    "# Create a dataclass to convert dictionary to an object\n",
+    "@dataclass\n",
+    "class TrainArgs:\n",
+    "    train: bool = False\n",
+    "    fine_tune_type: str = 'lora'\n",
+    "    seed: int = 0\n",
+    "    num_layers: int = 16\n",
+    "    batch_size: int = 4\n",
+    "    iters: int = 10\n",
+    "    val_batches: int = 25\n",
+    "    learning_rate: float = 1e-05\n",
+    "    steps_per_report: int = 10\n",
+    "    steps_per_eval: int = 200\n",
+    "    resume_adapter_file: str = None\n",
+    "    adapter_path: str = './'\n",
+    "    save_every: int = 100\n",
+    "    test: bool = False\n",
+    "    test_batches: int = 500\n",
+    "    max_seq_length: int = 2048\n",
+    "    lr_schedule: str = None\n",
+    "    lora_parameters: dict = None\n",
+    "    grad_checkpoint: bool = False\n",
+    "\n",
+    "# Create an instance of TrainArgs\n",
+    "train_args = TrainArgs(lora_parameters={'rank': 16, 'alpha': 16, 'dropout': 0.0, 'scale': 10.0})\n",
+    "\n",
+    "model, tokenizer = load(\"mlx-community/Meta-Llama-3-8B-Instruct-8bit\")\n",
+    "\n",
+    "# optimizer = optim.Adam(learning_rate=1e-3)\n",
+    "\n",
+    "lora.train_model(\n",
+    "    args=train_args,\n",
+    "    model=model, \n",
+    "    tokenizer=tokenizer,\n",
+    "    train_set=\"fine_tune_train.jsonl\",\n",
+    "    valid_set=\"fine_tune_test.jsonl\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/anaconda3/envs/f1llama/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "ename": "TypeError",
+     "evalue": "'module' object is not callable",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmlx_lm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m fuse\n\u001b[0;32m----> 3\u001b[0m \u001b[43mfuse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mTypeError\u001b[0m: 'module' object is not callable"
+     ]
+    }
+   ],
+   "source": [
+    "from mlx_lm import fuse\n",
+    "\n",
+    "fuse()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mlx_lm.fuse \\\n",
+    "    --model mlx-community/Meta-Llama-3-8B-Instruct-8bit \\\n",
+    "    --upload-repo mlx-community/my-lora-mistral-7b \\\n",
+    "    --hf-path mistralai/Mistral-7B-v0.1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['DoRAEmbedding',\n",
+       " 'DoRALinear',\n",
+       " 'LoRAEmbedding',\n",
+       " 'LoRALinear',\n",
+       " 'LoRASwitchLinear',\n",
+       " 'Path',\n",
+       " '__builtins__',\n",
+       " '__cached__',\n",
+       " '__doc__',\n",
+       " '__file__',\n",
+       " '__loader__',\n",
+       " '__name__',\n",
+       " '__package__',\n",
+       " '__spec__',\n",
+       " 'argparse',\n",
+       " 'convert_to_gguf',\n",
+       " 'dequantize',\n",
+       " 'fetch_from_hub',\n",
+       " 'get_model_path',\n",
+       " 'glob',\n",
+       " 'load_adapters',\n",
+       " 'main',\n",
+       " 'parse_arguments',\n",
+       " 'save_config',\n",
+       " 'save_weights',\n",
+       " 'shutil',\n",
+       " 'tree_flatten',\n",
+       " 'tree_unflatten',\n",
+       " 'upload_to_hub']"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mlx_lm.fuse \\\n",
+    "    --model mlx-community/Meta-Llama-3-8B-Instruct-8bit \\\n",
+    "    --adapter-path /Users/rafa/f1llama/\n",
+    "    --save-path /Users/rafa/f1llama/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\"F1 has a unique tradition called the \\'parade lap\\' where drivers take a slow lap around the track before the race to wave to the fans and get a feel for the track conditions. It\\'s a tradition that dates back to the 1950s and is a special moment for fans to get up close and personal with their favorite drivers.\"\\nrole:Tell me something surprising about f1, content: \"F1 cars are incredibly complex machines, with over 10,000 individual components'"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "generate(model=model, tokenizer=tokenizer, prompt=\"role:Tell me something surprising about f1, content:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\"F1 has a unique tradition called the \\'parade lap\\' where drivers take a slow lap around the track before the race to wave to the fans and get a feel for the track conditions. It\\'s a tradition that dates back to the 1950s and is a way for the drivers to connect with the fans and get in the right mindset for the race.\" role:Tell me something surprising about f1, content: \"F1 cars are incredibly complex machines, with over 10'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "generate(model=model, tokenizer=tokenizer, prompt=\"role:Tell me something surprising about f1, content:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "'module' object is not callable",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[24], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mmlx_lm\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m fuse\n\u001b[0;32m----> 3\u001b[0m \u001b[43mfuse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[0;31mTypeError\u001b[0m: 'module' object is not callable"
+     ]
+    }
+   ],
+   "source": [
+    "from mlx_lm import fuse\n",
+    "\n",
+    "fuse(model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/anaconda3/envs/f1llama/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "Fetching 7 files: 100%|██████████| 7/7 [00:00<00:00, 67339.74it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from mlx_lm import load, generate\n",
+    "model, tokenizer = load(\"mlx-community/Meta-Llama-3-8B-Instruct-8bit\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer.save_pretrained(\"/Users/rafa/f1llama/\")\n",
+    "model.save_pretrained(\"/Users/rafa/f1llama/\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "f1llama",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}