Upload folder using huggingface_hub

config.json

@@ -0,0 +1,43 @@
+{
+  "architectures": [
+    "UltravoxModel"
+  ],
+  "audio_latency_block_size": null,
+  "audio_model_id": "openai/whisper-large-v3-turbo",
+  "audio_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "auto_map": {
+    "AutoConfig": "ultravox_config.UltravoxConfig",
+    "AutoModel": "ultravox_model.UltravoxModel"
+  },
+  "hidden_size": 4096,
+  "ignore_index": -100,
+  "initializer_range": 0.02,
+  "model_type": "ultravox",
+  "norm_init": 0.4,
+  "pad_token_id": 2,
+  "projector_act": "swiglu",
+  "stack_factor": 8,
+  "text_model_id": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+  "text_model_lora_config": {
+    "lora_alpha": 8,
+    "r": 0,
+    "target_modules": [
+      "k_proj",
+      "q_proj",
+      "linear_k",
+      "linear_q"
+    ]
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "vocab_size": 49152
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.47.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e93b03d2e83320dcc9d1bcabea1d7ce48834a4ac10c87c7e20b90c7f629c98
+size 92299736

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca2a84237b754b634b8f915b57b07e69cdd191cb37d1823838e225178202efab
+size 184602962

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:171c449d4b396f9127e9ede85b442294e3914ded97fe428a0e1968cffff78f1b
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14976c1b3dd2b157698f792725a91270a736615d07a5dd5eae32d2ba4a41d313
+size 1064

special_tokens_map.json

@@ -0,0 +1,28 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,155 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

trainer_state.json

@@ -0,0 +1,712 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.49996528018887576,
+  "eval_steps": 1000,
+  "global_step": 7200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 6.943962224845496e-05,
+      "grad_norm": 9.8125,
+      "learning_rate": 2e-06,
+      "loss": 0.7188,
+      "step": 1
+    },
+    {
+      "epoch": 0.006943962224845497,
+      "grad_norm": 0.1376953125,
+      "learning_rate": 0.0002,
+      "loss": 0.3907,
+      "step": 100
+    },
+    {
+      "epoch": 0.013887924449690994,
+      "grad_norm": 0.0849609375,
+      "learning_rate": 0.0004,
+      "loss": 0.2341,
+      "step": 200
+    },
+    {
+      "epoch": 0.02083188667453649,
+      "grad_norm": 0.146484375,
+      "learning_rate": 0.0006,
+      "loss": 0.2181,
+      "step": 300
+    },
+    {
+      "epoch": 0.027775848899381988,
+      "grad_norm": 0.10546875,
+      "learning_rate": 0.0008,
+      "loss": 0.197,
+      "step": 400
+    },
+    {
+      "epoch": 0.034719811124227486,
+      "grad_norm": 0.11376953125,
+      "learning_rate": 0.001,
+      "loss": 0.1695,
+      "step": 500
+    },
+    {
+      "epoch": 0.04166377334907298,
+      "grad_norm": 0.0849609375,
+      "learning_rate": 0.0012,
+      "loss": 0.14,
+      "step": 600
+    },
+    {
+      "epoch": 0.04860773557391848,
+      "grad_norm": 0.078125,
+      "learning_rate": 0.0014,
+      "loss": 0.1231,
+      "step": 700
+    },
+    {
+      "epoch": 0.055551697798763976,
+      "grad_norm": 0.0712890625,
+      "learning_rate": 0.0016,
+      "loss": 0.1141,
+      "step": 800
+    },
+    {
+      "epoch": 0.06249566002360947,
+      "grad_norm": 0.06298828125,
+      "learning_rate": 0.0018000000000000002,
+      "loss": 0.1077,
+      "step": 900
+    },
+    {
+      "epoch": 0.06943962224845497,
+      "grad_norm": 0.0615234375,
+      "learning_rate": 0.002,
+      "loss": 0.1045,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06943962224845497,
+      "eval_covost2-en-de_loss": 1.4858413934707642,
+      "eval_covost2-en-de_runtime": 32.1589,
+      "eval_covost2-en-de_samples_per_second": 1.99,
+      "eval_covost2-en-de_steps_per_second": 0.062,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06943962224845497,
+      "eval_covost2-zh-en_loss": 2.7152516841888428,
+      "eval_covost2-zh-en_runtime": 31.3842,
+      "eval_covost2-zh-en_samples_per_second": 2.039,
+      "eval_covost2-zh-en_steps_per_second": 0.064,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06943962224845497,
+      "eval_peoplespeech-clean-transcription_loss": 2.0398874282836914,
+      "eval_peoplespeech-clean-transcription_runtime": 32.088,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 1.995,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.062,
+      "step": 1000
+    },
+    {
+      "epoch": 0.07638358447330046,
+      "grad_norm": 0.053955078125,
+      "learning_rate": 0.001999725185109816,
+      "loss": 0.101,
+      "step": 1100
+    },
+    {
+      "epoch": 0.08332754669814596,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 0.0019989008914857113,
+      "loss": 0.0956,
+      "step": 1200
+    },
+    {
+      "epoch": 0.09027150892299146,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 0.00199752757218401,
+      "loss": 0.0936,
+      "step": 1300
+    },
+    {
+      "epoch": 0.09721547114783696,
+      "grad_norm": 0.0390625,
+      "learning_rate": 0.001995605982021898,
+      "loss": 0.0917,
+      "step": 1400
+    },
+    {
+      "epoch": 0.10415943337268245,
+      "grad_norm": 0.0517578125,
+      "learning_rate": 0.0019931371771625545,
+      "loss": 0.0894,
+      "step": 1500
+    },
+    {
+      "epoch": 0.11110339559752795,
+      "grad_norm": 0.0419921875,
+      "learning_rate": 0.001990122514534651,
+      "loss": 0.0868,
+      "step": 1600
+    },
+    {
+      "epoch": 0.11804735782237345,
+      "grad_norm": 0.039306640625,
+      "learning_rate": 0.0019865636510865464,
+      "loss": 0.0861,
+      "step": 1700
+    },
+    {
+      "epoch": 0.12499132004721894,
+      "grad_norm": 0.047119140625,
+      "learning_rate": 0.001982462542875576,
+      "loss": 0.0854,
+      "step": 1800
+    },
+    {
+      "epoch": 0.13193528227206444,
+      "grad_norm": 0.0390625,
+      "learning_rate": 0.001977821443992945,
+      "loss": 0.0837,
+      "step": 1900
+    },
+    {
+      "epoch": 0.13887924449690994,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 0.001972642905324813,
+      "loss": 0.0818,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13887924449690994,
+      "eval_covost2-en-de_loss": 1.4137890338897705,
+      "eval_covost2-en-de_runtime": 32.5714,
+      "eval_covost2-en-de_samples_per_second": 1.965,
+      "eval_covost2-en-de_steps_per_second": 0.061,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13887924449690994,
+      "eval_covost2-zh-en_loss": 2.667837381362915,
+      "eval_covost2-zh-en_runtime": 31.1685,
+      "eval_covost2-zh-en_samples_per_second": 2.053,
+      "eval_covost2-zh-en_steps_per_second": 0.064,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13887924449690994,
+      "eval_peoplespeech-clean-transcription_loss": 1.835880160331726,
+      "eval_peoplespeech-clean-transcription_runtime": 32.0265,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 1.998,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.062,
+      "step": 2000
+    },
+    {
+      "epoch": 0.14582320672175544,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.0019669297731502505,
+      "loss": 0.0813,
+      "step": 2100
+    },
+    {
+      "epoch": 0.15276716894660092,
+      "grad_norm": 0.03515625,
+      "learning_rate": 0.00196068518757684,
+      "loss": 0.0811,
+      "step": 2200
+    },
+    {
+      "epoch": 0.15971113117144642,
+      "grad_norm": 0.04443359375,
+      "learning_rate": 0.001953912580814779,
+      "loss": 0.0793,
+      "step": 2300
+    },
+    {
+      "epoch": 0.16665509339629192,
+      "grad_norm": 0.037841796875,
+      "learning_rate": 0.0019466156752904343,
+      "loss": 0.0788,
+      "step": 2400
+    },
+    {
+      "epoch": 0.17359905562113742,
+      "grad_norm": 0.04052734375,
+      "learning_rate": 0.0019387984816003866,
+      "loss": 0.0783,
+      "step": 2500
+    },
+    {
+      "epoch": 0.18054301784598292,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 0.0019304652963070869,
+      "loss": 0.0772,
+      "step": 2600
+    },
+    {
+      "epoch": 0.18748698007082842,
+      "grad_norm": 0.036376953125,
+      "learning_rate": 0.0019216206995773372,
+      "loss": 0.0771,
+      "step": 2700
+    },
+    {
+      "epoch": 0.19443094229567393,
+      "grad_norm": 0.0400390625,
+      "learning_rate": 0.0019122695526648968,
+      "loss": 0.0766,
+      "step": 2800
+    },
+    {
+      "epoch": 0.2013749045205194,
+      "grad_norm": 0.0361328125,
+      "learning_rate": 0.0019024169952385887,
+      "loss": 0.0753,
+      "step": 2900
+    },
+    {
+      "epoch": 0.2083188667453649,
+      "grad_norm": 0.03125,
+      "learning_rate": 0.0018920684425573864,
+      "loss": 0.075,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2083188667453649,
+      "eval_covost2-en-de_loss": 1.3828500509262085,
+      "eval_covost2-en-de_runtime": 32.2972,
+      "eval_covost2-en-de_samples_per_second": 1.982,
+      "eval_covost2-en-de_steps_per_second": 0.062,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2083188667453649,
+      "eval_covost2-zh-en_loss": 2.646721839904785,
+      "eval_covost2-zh-en_runtime": 31.0128,
+      "eval_covost2-zh-en_samples_per_second": 2.064,
+      "eval_covost2-zh-en_steps_per_second": 0.064,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2083188667453649,
+      "eval_peoplespeech-clean-transcription_loss": 1.773127555847168,
+      "eval_peoplespeech-clean-transcription_runtime": 32.5192,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 1.968,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.062,
+      "step": 3000
+    },
+    {
+      "epoch": 0.2152628289702104,
+      "grad_norm": 0.03369140625,
+      "learning_rate": 0.0018812295824940284,
+      "loss": 0.0743,
+      "step": 3100
+    },
+    {
+      "epoch": 0.2222067911950559,
+      "grad_norm": 0.034912109375,
+      "learning_rate": 0.0018699063724087904,
+      "loss": 0.074,
+      "step": 3200
+    },
+    {
+      "epoch": 0.2291507534199014,
+      "grad_norm": 0.033203125,
+      "learning_rate": 0.0018581050358751443,
+      "loss": 0.0742,
+      "step": 3300
+    },
+    {
+      "epoch": 0.2360947156447469,
+      "grad_norm": 0.03857421875,
+      "learning_rate": 0.0018458320592590974,
+      "loss": 0.0742,
+      "step": 3400
+    },
+    {
+      "epoch": 0.24303867786959238,
+      "grad_norm": 0.033935546875,
+      "learning_rate": 0.0018330941881540914,
+      "loss": 0.0728,
+      "step": 3500
+    },
+    {
+      "epoch": 0.24998264009443788,
+      "grad_norm": 0.031005859375,
+      "learning_rate": 0.0018198984236734246,
+      "loss": 0.0728,
+      "step": 3600
+    },
+    {
+      "epoch": 0.2569266023192834,
+      "grad_norm": 0.030029296875,
+      "learning_rate": 0.0018062520186022297,
+      "loss": 0.0714,
+      "step": 3700
+    },
+    {
+      "epoch": 0.2638705645441289,
+      "grad_norm": 0.02734375,
+      "learning_rate": 0.0017921624734111292,
+      "loss": 0.071,
+      "step": 3800
+    },
+    {
+      "epoch": 0.2708145267689744,
+      "grad_norm": 0.03271484375,
+      "learning_rate": 0.001777637532133752,
+      "loss": 0.0705,
+      "step": 3900
+    },
+    {
+      "epoch": 0.2777584889938199,
+      "grad_norm": 0.035400390625,
+      "learning_rate": 0.0017626851781103819,
+      "loss": 0.0714,
+      "step": 4000
+    },
+    {
+      "epoch": 0.2777584889938199,
+      "eval_covost2-en-de_loss": 1.3778624534606934,
+      "eval_covost2-en-de_runtime": 32.6181,
+      "eval_covost2-en-de_samples_per_second": 1.962,
+      "eval_covost2-en-de_steps_per_second": 0.061,
+      "step": 4000
+    },
+    {
+      "epoch": 0.2777584889938199,
+      "eval_covost2-zh-en_loss": 2.6438870429992676,
+      "eval_covost2-zh-en_runtime": 31.4603,
+      "eval_covost2-zh-en_samples_per_second": 2.034,
+      "eval_covost2-zh-en_steps_per_second": 0.064,
+      "step": 4000
+    },
+    {
+      "epoch": 0.2777584889938199,
+      "eval_peoplespeech-clean-transcription_loss": 1.7361584901809692,
+      "eval_peoplespeech-clean-transcription_runtime": 32.442,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 1.973,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.062,
+      "step": 4000
+    },
+    {
+      "epoch": 0.2847024512186654,
+      "grad_norm": 0.0281982421875,
+      "learning_rate": 0.001747313629600077,
+      "loss": 0.0713,
+      "step": 4100
+    },
+    {
+      "epoch": 0.2916464134435109,
+      "grad_norm": 0.028076171875,
+      "learning_rate": 0.001731531335263669,
+      "loss": 0.0699,
+      "step": 4200
+    },
+    {
+      "epoch": 0.2985903756683564,
+      "grad_norm": 0.0277099609375,
+      "learning_rate": 0.0017153469695201276,
+      "loss": 0.0702,
+      "step": 4300
+    },
+    {
+      "epoch": 0.30553433789320184,
+      "grad_norm": 0.031982421875,
+      "learning_rate": 0.0016987694277788418,
+      "loss": 0.0692,
+      "step": 4400
+    },
+    {
+      "epoch": 0.31247830011804734,
+      "grad_norm": 0.02880859375,
+      "learning_rate": 0.001681807821550438,
+      "loss": 0.0686,
+      "step": 4500
+    },
+    {
+      "epoch": 0.31942226234289284,
+      "grad_norm": 0.0289306640625,
+      "learning_rate": 0.0016644714734388218,
+      "loss": 0.0698,
+      "step": 4600
+    },
+    {
+      "epoch": 0.32636622456773834,
+      "grad_norm": 0.029541015625,
+      "learning_rate": 0.0016467699120171987,
+      "loss": 0.0683,
+      "step": 4700
+    },
+    {
+      "epoch": 0.33331018679258384,
+      "grad_norm": 0.034423828125,
+      "learning_rate": 0.001628712866590885,
+      "loss": 0.0687,
+      "step": 4800
+    },
+    {
+      "epoch": 0.34025414901742934,
+      "grad_norm": 0.0289306640625,
+      "learning_rate": 0.0016103102618497923,
+      "loss": 0.0684,
+      "step": 4900
+    },
+    {
+      "epoch": 0.34719811124227484,
+      "grad_norm": 0.0263671875,
+      "learning_rate": 0.0015915722124135226,
+      "loss": 0.0681,
+      "step": 5000
+    },
+    {
+      "epoch": 0.34719811124227484,
+      "eval_covost2-en-de_loss": 1.3711879253387451,
+      "eval_covost2-en-de_runtime": 32.6293,
+      "eval_covost2-en-de_samples_per_second": 1.961,
+      "eval_covost2-en-de_steps_per_second": 0.061,
+      "step": 5000
+    },
+    {
+      "epoch": 0.34719811124227484,
+      "eval_covost2-zh-en_loss": 2.6346511840820312,
+      "eval_covost2-zh-en_runtime": 32.1513,
+      "eval_covost2-zh-en_samples_per_second": 1.991,
+      "eval_covost2-zh-en_steps_per_second": 0.062,
+      "step": 5000
+    },
+    {
+      "epoch": 0.34719811124227484,
+      "eval_peoplespeech-clean-transcription_loss": 1.7350472211837769,
+      "eval_peoplespeech-clean-transcription_runtime": 32.5813,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 1.964,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.061,
+      "step": 5000
+    },
+    {
+      "epoch": 0.35414207346712034,
+      "grad_norm": 0.028076171875,
+      "learning_rate": 0.001572509017272072,
+      "loss": 0.0693,
+      "step": 5100
+    },
+    {
+      "epoch": 0.36108603569196585,
+      "grad_norm": 0.030517578125,
+      "learning_rate": 0.0015531311541251993,
+      "loss": 0.0683,
+      "step": 5200
+    },
+    {
+      "epoch": 0.36802999791681135,
+      "grad_norm": 0.031494140625,
+      "learning_rate": 0.0015334492736235703,
+      "loss": 0.0677,
+      "step": 5300
+    },
+    {
+      "epoch": 0.37497396014165685,
+      "grad_norm": 0.0284423828125,
+      "learning_rate": 0.0015134741935148419,
+      "loss": 0.0669,
+      "step": 5400
+    },
+    {
+      "epoch": 0.38191792236650235,
+      "grad_norm": 0.0302734375,
+      "learning_rate": 0.0014932168926979072,
+      "loss": 0.0669,
+      "step": 5500
+    },
+    {
+      "epoch": 0.38886188459134785,
+      "grad_norm": 0.0260009765625,
+      "learning_rate": 0.0014726885051885652,
+      "loss": 0.0666,
+      "step": 5600
+    },
+    {
+      "epoch": 0.3958058468161933,
+      "grad_norm": 0.033447265625,
+      "learning_rate": 0.0014519003139999338,
+      "loss": 0.0659,
+      "step": 5700
+    },
+    {
+      "epoch": 0.4027498090410388,
+      "grad_norm": 0.0283203125,
+      "learning_rate": 0.0014308637449409706,
+      "loss": 0.0653,
+      "step": 5800
+    },
+    {
+      "epoch": 0.4096937712658843,
+      "grad_norm": 0.0283203125,
+      "learning_rate": 0.0014095903603365066,
+      "loss": 0.0662,
+      "step": 5900
+    },
+    {
+      "epoch": 0.4166377334907298,
+      "grad_norm": 0.0267333984375,
+      "learning_rate": 0.0013880918526722496,
+      "loss": 0.0665,
+      "step": 6000
+    },
+    {
+      "epoch": 0.4166377334907298,
+      "eval_covost2-en-de_loss": 1.3651559352874756,
+      "eval_covost2-en-de_runtime": 32.5621,
+      "eval_covost2-en-de_samples_per_second": 1.965,
+      "eval_covost2-en-de_steps_per_second": 0.061,
+      "step": 6000
+    },
+    {
+      "epoch": 0.4166377334907298,
+      "eval_covost2-zh-en_loss": 2.6372551918029785,
+      "eval_covost2-zh-en_runtime": 31.028,
+      "eval_covost2-zh-en_samples_per_second": 2.063,
+      "eval_covost2-zh-en_steps_per_second": 0.064,
+      "step": 6000
+    },
+    {
+      "epoch": 0.4166377334907298,
+      "eval_peoplespeech-clean-transcription_loss": 1.7209596633911133,
+      "eval_peoplespeech-clean-transcription_runtime": 32.6773,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 1.959,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.061,
+      "step": 6000
+    },
+    {
+      "epoch": 0.4235816957155753,
+      "grad_norm": 0.028564453125,
+      "learning_rate": 0.0013663800381682463,
+      "loss": 0.0658,
+      "step": 6100
+    },
+    {
+      "epoch": 0.4305256579404208,
+      "grad_norm": 0.0299072265625,
+      "learning_rate": 0.0013444668502843329,
+      "loss": 0.0657,
+      "step": 6200
+    },
+    {
+      "epoch": 0.4374696201652663,
+      "grad_norm": 0.0296630859375,
+      "learning_rate": 0.0013223643331611537,
+      "loss": 0.0655,
+      "step": 6300
+    },
+    {
+      "epoch": 0.4444135823901118,
+      "grad_norm": 0.0286865234375,
+      "learning_rate": 0.001300084635000341,
+      "loss": 0.0654,
+      "step": 6400
+    },
+    {
+      "epoch": 0.4513575446149573,
+      "grad_norm": 0.028564453125,
+      "learning_rate": 0.0012776400013875004,
+      "loss": 0.0655,
+      "step": 6500
+    },
+    {
+      "epoch": 0.4583015068398028,
+      "grad_norm": 0.030029296875,
+      "learning_rate": 0.0012550427685616766,
+      "loss": 0.0648,
+      "step": 6600
+    },
+    {
+      "epoch": 0.4652454690646483,
+      "grad_norm": 0.037109375,
+      "learning_rate": 0.0012323053566349834,
+      "loss": 0.0654,
+      "step": 6700
+    },
+    {
+      "epoch": 0.4721894312894938,
+      "grad_norm": 0.029296875,
+      "learning_rate": 0.0012094402627661448,
+      "loss": 0.0643,
+      "step": 6800
+    },
+    {
+      "epoch": 0.47913339351433926,
+      "grad_norm": 0.030517578125,
+      "learning_rate": 0.0011864600542916813,
+      "loss": 0.0646,
+      "step": 6900
+    },
+    {
+      "epoch": 0.48607735573918476,
+      "grad_norm": 0.037353515625,
+      "learning_rate": 0.0011633773618185302,
+      "loss": 0.0642,
+      "step": 7000
+    },
+    {
+      "epoch": 0.48607735573918476,
+      "eval_covost2-en-de_loss": 1.3594859838485718,
+      "eval_covost2-en-de_runtime": 32.6807,
+      "eval_covost2-en-de_samples_per_second": 1.958,
+      "eval_covost2-en-de_steps_per_second": 0.061,
+      "step": 7000
+    },
+    {
+      "epoch": 0.48607735573918476,
+      "eval_covost2-zh-en_loss": 2.626713514328003,
+      "eval_covost2-zh-en_runtime": 31.0228,
+      "eval_covost2-zh-en_samples_per_second": 2.063,
+      "eval_covost2-zh-en_steps_per_second": 0.064,
+      "step": 7000
+    },
+    {
+      "epoch": 0.48607735573918476,
+      "eval_peoplespeech-clean-transcription_loss": 1.693739652633667,
+      "eval_peoplespeech-clean-transcription_runtime": 31.9776,
+      "eval_peoplespeech-clean-transcription_samples_per_second": 2.001,
+      "eval_peoplespeech-clean-transcription_steps_per_second": 0.063,
+      "step": 7000
+    },
+    {
+      "epoch": 0.49302131796403026,
+      "grad_norm": 0.02978515625,
+      "learning_rate": 0.0011402048722818862,
+      "loss": 0.0656,
+      "step": 7100
+    },
+    {
+      "epoch": 0.49996528018887576,
+      "grad_norm": 0.0281982421875,
+      "learning_rate": 0.0011169553219720827,
+      "loss": 0.064,
+      "step": 7200
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 14400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3600,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.8942929465072026e+18,
+  "train_batch_size": 96,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:526c930de4f2a14f4d9943636fb014003cb001081aed1fbeba40b7395ec5812b
+size 5688

ultravox_config.py

@@ -0,0 +1,170 @@
+import dataclasses
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+import transformers
+
+
+@dataclasses.dataclass
+class LoraConfigSimplified:
+    """
+    Low Rank Approximation (LoRA) configuration.
+
+    Used for language and audio models separately.
+    """
+
+    # The rank of the approximation
+    r: int = 0
+    lora_alpha: float = 8
+    target_modules: Optional[List[str]] = dataclasses.field(
+        default_factory=lambda: ["k_proj", "q_proj", "linear_k", "linear_q"]
+    )
+
+
+class LossFunction(str, Enum):
+    CrossEntropy = "ce"
+    KL_Divergence = "kl"
+
+
+@dataclasses.dataclass
+class LossConfig:
+    loss_function: LossFunction = LossFunction.KL_Divergence
+    kl_temperature: float = 2.0
+
+    @property
+    def requires_alt_fields(self):
+        return self.loss_function == LossFunction.KL_Divergence
+
+
+class UltravoxConfig(transformers.PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an
+    Ultravox model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        audio_config (`Wav2Vec2Config`,  *optional*):
+            Custom audio config or dict
+        text_config (`Union[AutoConfig, dict]`, *optional*):
+            The config object of the text backbone. Can be any of `LlamaConfig` or `MistralConfig`.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The ignore index for the loss function.
+        audio_token_index (`int`, *optional*, defaults to 32000):
+            The audio token index to encode the audio prompt.
+        stack_factor (`int`, *optional*, defaults to 8):
+            Audio downsampling factor for the multimodal projector.
+        norm_init (`float`, *optional*, defaults to 0.4):
+            The initialization value for the layer normalization.
+        projector_act (`str`, *optional*, defaults to `"swiglu"`):
+            The activation function used by the multimodal projector.
+        text_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the text model.
+        audio_model_lora_config (`LoraConfigSimplified`, *optional*):
+            The LoRA configuration for finetuning the audio model.
+        audio_latency_block_size (`int`, *optional*, defaults to `None`):
+            The latency block size for simulating audio streaming.
+
+
+    Example:
+
+    ```python
+    >>> from transformers import UltravoxForConditionalGeneration, Wav2Vec2Config, UltravoxConfig, LlamaConfig
+
+    >>> # Initializing an audio encoder config
+    >>> audio_config = Wav2Vec2Config()
+
+    >>> # Initializing a Llama config
+    >>> text_config = LlamaConfig()
+
+    >>> # Initializing a default configuration
+    >>> configuration = UltravoxConfig(audio_config, text_config)
+
+    >>> # Initializing a completely untrained model from the configuration
+    >>> model = UltravoxForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # Initialize a model from pretrained checkpoints and random projector weights
+    >>> config = UltravoxConfig(audio_model_id="facebook/wav2vec2-base-960h", text_model_id="meta-llama/Llama-2-7b-chat-hf")
+    ```"""
+
+    model_type = "ultravox"
+    is_composition = False
+
+    def __init__(
+        self,
+        audio_config: Optional[Dict[str, Any]] = None,
+        text_config: Optional[Dict[str, Any]] = None,
+        audio_model_id: Optional[str] = None,
+        text_model_id: Optional[str] = None,
+        ignore_index: int = -100,
+        hidden_size: int = 4096,
+        stack_factor: int = 8,
+        norm_init: float = 0.4,
+        projector_act: str = "swiglu",
+        text_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_model_lora_config: Optional[LoraConfigSimplified] = None,
+        audio_latency_block_size: Optional[int] = None,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+
+        self.audio_model_id = audio_model_id
+        self.text_model_id = text_model_id
+
+        self.hidden_size = hidden_size
+        self.stack_factor = stack_factor
+        self.norm_init = norm_init
+        self.projector_act = projector_act
+
+        if text_model_id is not None:
+            self.text_config: transformers.LlamaConfig = (
+                transformers.AutoConfig.from_pretrained(text_model_id)
+            )
+        else:
+            text_config = text_config or {}
+            self.text_config = transformers.CONFIG_MAPPING[
+                text_config.get("model_type", "llama")
+            ](**text_config)
+
+        if audio_model_id is not None:
+            self.audio_config: transformers.PretrainedConfig = (
+                transformers.AutoConfig.from_pretrained(audio_model_id)
+            )
+        else:
+            audio_config = audio_config or {}
+            self.audio_config = transformers.CONFIG_MAPPING[
+                audio_config.get("model_type", "wav2vec2")
+            ](**audio_config)
+
+        self.text_model_lora_config = (
+            text_model_lora_config
+            if isinstance(text_model_lora_config, dict)
+            else dataclasses.asdict(text_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_model_lora_config = (
+            audio_model_lora_config
+            if isinstance(audio_model_lora_config, dict)
+            else dataclasses.asdict(audio_model_lora_config or LoraConfigSimplified())
+        )
+        self.audio_latency_block_size = audio_latency_block_size
+
+        self.vocab_size = self.text_config.vocab_size
+
+        self.initializer_range = self.text_config.initializer_range
+
+        super().__init__(**kwargs)
+
+    def to_diff_dict(self) -> Dict[str, Any]:
+        diff_dict = super().to_diff_dict()
+
+        # remove text_config and audio_config if text_model_id and audio_model_id are present
+        if self.text_model_id is not None:
+            diff_dict.pop("text_config", None)
+        if self.audio_model_id is not None:
+            diff_dict.pop("audio_config", None)
+
+        return diff_dict

ultravox_model.py

@@ -0,0 +1,723 @@
+import logging
+from typing import Any, Dict, Optional, Set, Tuple, Union
+
+import peft
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import transformers
+import transformers.activations
+import transformers.modeling_outputs
+import transformers.models
+from transformers.models.whisper import modeling_whisper as whisper
+
+# We must use relative import in this directory to allow uploading to HF Hub
+# Even "from . import X" pattern doesn't work (undocumented and unclear why)
+from .ultravox_config import LossConfig
+from .ultravox_config import LossFunction
+from .ultravox_config import UltravoxConfig
+
+
+class UltravoxModel(transformers.LlamaPreTrainedModel):
+    """
+    The Ultravox model which consists of an audio encoder and a language model.
+
+    Audio input is processed by the audio encoder, then every `stack_factor` frames are stacked together and
+    projected to the language model's embedding space using a few linear layers.
+    The text is embedded by the language model as usual and then the audio and text embeddings are merged together.
+
+    A special token `<|audio|>` is used to indicate the start of the audio embeddings in the merged embeddings.
+
+    Parameters:
+        config: Model configuration class with all the parameters of the model.
+    """
+
+    config_class = UltravoxConfig
+    config: UltravoxConfig  # for type hinting
+    # Usually we load encoder and LLM weights from a pretrained model separately, so they are allowed to be missing
+    _keys_to_ignore_on_load_missing = ["audio_tower.*", "language_model.*"]
+
+    def __init__(self, config: UltravoxConfig):
+        super().__init__(config)
+        self._register_load_state_dict_pre_hook(self._pre_load_state_dict_hook)
+
+        self.keep_params: Set[str] = set()
+        self.vocab_size = config.vocab_size
+
+        self.audio_tower = self._create_audio_tower(config)
+        self.multi_modal_projector = self._create_multi_modal_projector(config)
+        self.language_model = self._create_language_model(config)
+
+        # Determine no_split_modules dynamically to use with FSDP auto_wrap policy.
+        # FSDP throws an error if some of the layer types are not found in the model.
+        # This would be something like ["LlamaDecoderLayer", "WhisperEncoderLayer"]
+        self._no_split_modules = (self.language_model._no_split_modules or []) + (
+            self.audio_tower._no_split_modules or []
+        )
+
+        self.loss_config = LossConfig()
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+    def set_loss_config(self, loss_config: LossConfig):
+        self.loss_config = loss_config
+
+    def _setup_cache(
+        self, cache_cls, max_batch_size: int, max_cache_len: Optional[int] = None
+    ):
+        self.language_model._setup_cache(cache_cls, max_batch_size, max_cache_len)
+
+    def _reorder_cache(self, past_key_values, beam_idx):
+        return self.language_model._reorder_cache(past_key_values, beam_idx)
+
+    def resize_token_embeddings(
+        self,
+        new_num_tokens: Optional[int] = None,
+        pad_to_multiple_of: Optional[int] = None,
+    ) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(
+            new_num_tokens, pad_to_multiple_of
+        )
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def _compute_kl_loss(
+        self,
+        lm_output: transformers.modeling_outputs.CausalLMOutputWithPast,
+        labels: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        # disable gradient computation for the teacher model
+        with torch.no_grad():
+            # compute the teacher (text-only) model's distribution
+            alt_inputs_embeds = self.get_input_embeddings().forward(alt_input_ids)
+            alt_lm_output = self.language_model.forward(
+                inputs_embeds=alt_inputs_embeds,
+                labels=alt_labels,
+                attention_mask=alt_attention_mask,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+        # compute the KL divergence loss between the two models
+        kl_loss = F.kl_div(
+            F.log_softmax(
+                lm_output.logits[labels != -100] / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            F.softmax(
+                alt_lm_output.logits[alt_labels != -100]
+                / self.loss_config.kl_temperature,
+                dim=-1,
+            ),
+            reduction="batchmean",
+        )
+        return {"loss": kl_loss}
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        # the alt_* fields are needed for KL divergence loss
+        alt_input_ids: Optional[torch.Tensor] = None,
+        alt_attention_mask: Optional[torch.Tensor] = None,
+        alt_labels: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Union[Tuple, transformers.modeling_outputs.CausalLMOutputWithPast]:
+        """
+        Forward pass for the Ultravox model.
+
+        `input_ids` are the tokenized text input. They are embedded by the language model as usual.
+        `audio_values` are processed by the audio encoder and then every `stack_factor` frames are stacked together and
+        projected to the language model's embedding space using a few linear layers.
+        The audio and text embeddings are merged together. A special token `<|audio|>` is used to indicate the start
+        of the audio embeddings in the merged embeddings.
+
+        Args:
+            input_ids: The tokenized text input.
+            audio_values: The processed audio values.
+            inputs_embeds: The embeddings for the input tokens.
+            labels: The tokenized text labels.
+            attention_mask: The attention mask for the input.
+            position_ids: The position ids for the input.
+            past_key_values: The past key value cache for the language model attention layers.
+            **kwargs: Additional keyword arguments. Passed directly to the language model.
+        """
+        if inputs_embeds is None:
+            # B x T  ->  B x T x D
+            inputs_embeds = self.get_input_embeddings().forward(input_ids)
+
+        if audio_values is not None:
+            assert (
+                audio_token_start_idx is not None and audio_token_len is not None
+            ), "audio_token_start_idx and audio_token_len must be provided if audio_values are provided."
+            assert (
+                len(audio_token_start_idx) == len(audio_token_len) == len(audio_values)
+            ), "audio_token_start_idx, audio_token_len, and audio_values must have the same batch size."
+
+            # B x A/3200 x D
+            audio_tower_output = self.audio_tower.forward(
+                audio_values.to(self.audio_tower.dtype),
+                audio_len=audio_len,
+            ).last_hidden_state
+            audio_tower_output = audio_tower_output.to(inputs_embeds.dtype)
+
+            audio_embeds = self.multi_modal_projector.forward(audio_tower_output)
+
+            # combine audio and text embeddings
+            for i, (audio, start, length) in enumerate(
+                zip(audio_embeds, audio_token_start_idx, audio_token_len)
+            ):
+                length = min(length, audio.shape[0])
+                inputs_embeds[i, start : start + length] = audio[:length]
+
+        lm_output = self.language_model.forward(
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            **kwargs,
+        )
+        if self.training:
+            if self.loss_config.loss_function == LossFunction.CrossEntropy:
+                return lm_output
+            elif self.loss_config.loss_function == LossFunction.KL_Divergence:
+                return self._compute_kl_loss(
+                    lm_output=lm_output,
+                    labels=labels,
+                    past_key_values=past_key_values,
+                    alt_input_ids=alt_input_ids,
+                    alt_attention_mask=alt_attention_mask,
+                    alt_labels=alt_labels,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(
+                    f"Unsupported loss function: {self.loss_config.loss_function}"
+                )
+        else:
+            return lm_output
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.Tensor,
+        audio_values: Optional[torch.FloatTensor] = None,
+        audio_token_start_idx: Optional[torch.Tensor] = None,
+        audio_token_len: Optional[torch.Tensor] = None,
+        audio_len: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        model_input = self.language_model.prepare_inputs_for_generation(
+            input_ids=input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        # include audio information in model_input only when it is needed during prefilling
+        # audio_token_start_idx should always be relative to the current cache position
+        prefill_start_idx = 0 if cache_position is None else cache_position[0]
+        if (
+            audio_values is not None
+            and audio_token_start_idx is not None
+            and prefill_start_idx <= torch.max(audio_token_start_idx)
+        ):
+            model_input["audio_values"] = audio_values
+            model_input["audio_token_start_idx"] = (
+                audio_token_start_idx - prefill_start_idx
+            )
+            model_input["audio_token_len"] = audio_token_len
+            model_input["audio_len"] = audio_len
+
+        return model_input
+
+    @classmethod
+    def _create_multi_modal_projector(
+        cls, config: UltravoxConfig
+    ) -> "UltravoxProjector":
+        projector = UltravoxProjector(config)
+        projector.to(config.torch_dtype)
+        return projector
+
+    @classmethod
+    def _create_audio_tower(
+        cls, config: UltravoxConfig
+    ) -> Union[transformers.Wav2Vec2Model, "ModifiedWhisperEncoder"]:
+        if config.audio_model_id is not None:
+            if "whisper" in config.audio_model_id is not None:
+                audio_tower = ModifiedWhisperEncoder.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                audio_tower = transformers.AutoModel.from_pretrained(
+                    config.audio_model_id, torch_dtype=config.torch_dtype
+                )
+        else:
+            if "whisper" in config.audio_config._name_or_path:
+                audio_tower = ModifiedWhisperEncoder(config.audio_config)
+                audio_tower.init_latency_mask(
+                    config.audio_latency_block_size, dtype=config.torch_dtype
+                )
+            else:
+                assert config.audio_latency_block_size in (
+                    None,
+                    0,
+                ), "only whisper audio tower supports audio latency masking, got non-zero value for 'audio_latency_block_size'"
+                with transformers.modeling_utils.no_init_weights():
+                    # we only ever use from_config if the weights are retrained, hence initializing is not
+                    # required. This makes the model quite creation faster since init on CPU is quite slow.
+                    audio_tower = transformers.AutoModel.from_config(
+                        config.audio_config
+                    )
+
+        if isinstance(
+            audio_tower,
+            (transformers.Wav2Vec2BertModel, transformers.WhisperModel),
+        ):
+            # For these models we only need the encoder part
+            # Wav2Vec2BertModel -> Wav2Vec2BertEncoder
+            # WhisperModel -> WhisperEncoder
+            audio_tower = audio_tower.encoder
+
+        audio_tower = apply_lora(audio_tower, config.audio_model_lora_config)
+        return audio_tower
+
+    @classmethod
+    def _create_language_model(
+        cls, config: UltravoxConfig
+    ) -> transformers.LlamaForCausalLM:
+        if config.text_model_id is not None:
+            language_model = transformers.AutoModelForCausalLM.from_pretrained(
+                config.text_model_id,
+                attn_implementation=config._attn_implementation,
+                torch_dtype=config.torch_dtype,
+            )
+        else:
+            with transformers.modeling_utils.no_init_weights():
+                # we only ever use from_config if the weights are retrained, hence initializing is not
+                # required. This makes the model quite creation faster since init on CPU is quite slow.
+                language_model = transformers.AutoModelForCausalLM.from_config(
+                    config.text_config,
+                    attn_implementation=config._attn_implementation,
+                    torch_dtype=config.torch_dtype,
+                )
+
+        language_model = apply_lora(language_model, config.text_model_lora_config)
+        return language_model
+
+    def merge_and_unload(self):
+        if isinstance(self.language_model, peft.PeftModel):
+            self.language_model = self.language_model.merge_and_unload()
+            # no need to download base language model weights anymore, so we can remove the id
+            self.config.text_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"language_model.{name}"
+                        for name, _ in self.language_model.named_parameters()
+                    ]
+                )
+            )
+
+        if isinstance(self.audio_tower, peft.PeftModel):
+            self.audio_tower = self.audio_tower.merge_and_unload()
+            # no need to download base audio model weights anymore, so we can remove the id
+            self.config.audio_model_id = None
+            self.keep_params.update(
+                set(
+                    [
+                        f"audio_tower.{name}"
+                        for name, _ in self.audio_tower.named_parameters()
+                    ]
+                )
+            )
+
+        for param in ["text_model_lora_config", "audio_model_lora_config"]:
+            if hasattr(self.config, param):
+                delattr(self.config, param)
+
+    def push_to_hub(self, *args, **kwargs):
+        self.merge_and_unload()
+        self.to(self.language_model.dtype)
+        return super().push_to_hub(*args, **kwargs)
+
+    def save_pretrained(
+        self, *args, state_dict: Optional[Dict[str, Any]] = None, **kwargs
+    ):
+        if state_dict is None:
+            state_dict = super().state_dict()
+
+        named_params = dict(self.named_parameters())
+
+        state_dict = {
+            k: v
+            for k, v in state_dict.items()
+            if k in self.keep_params
+            or (k in named_params and named_params[k].requires_grad)
+        }
+
+        super().save_pretrained(*args, state_dict=state_dict, **kwargs)
+
+    def _pre_load_state_dict_hook(self, state_dict: Dict[str, Any], *args, **kwargs):
+        self.keep_params.update(set(state_dict.keys()))
+
+    def print_trainable_parameters(self):
+        """
+        Prints the number of trainable parameters in the model (reuses Peft model's method)
+        """
+        count_params = peft.peft_model.PeftModel.get_nb_trainable_parameters
+
+        trainable_params, all_param = count_params(self)
+
+        logging.info(
+            f"trainable params: {trainable_params:,d} || all params: {all_param:,d}"
+            f" || trainable%: {100 * trainable_params / all_param:.1f}%"
+        )
+
+        lm_trainable_params, lm_all_params = count_params(self.language_model)
+        audio_trainable_params, audio_all_params = count_params(self.audio_tower)
+
+        projector_trainable_params = (
+            trainable_params - lm_trainable_params - audio_trainable_params
+        )
+        projector_all_params = all_param - lm_all_params - audio_all_params
+
+        logging.info(
+            f"Trainable%:   "
+            f" LLM: {100 * lm_trainable_params / lm_all_params:.1f}%"
+            f" || Audio Encoder: {100 * audio_trainable_params / audio_all_params:.1f}%"
+            f" || Projector: {100 * projector_trainable_params / projector_all_params:.1f}%"
+        )
+
+
+def is_cache_empty(
+    past_key_values: Optional[Union[Tuple, transformers.cache_utils.Cache]]
+) -> bool:
+    """
+    Check if the cache is empty.
+    """
+    if past_key_values is None:
+        return True
+    if isinstance(past_key_values, tuple):
+        return all(len(c) == 0 for c in past_key_values)
+    return past_key_values.get_seq_length() == 0
+
+
+def apply_lora(model: torch.nn.Module, lora_config: dict) -> torch.nn.Module:
+    """
+    Applies LoRA finetuning to the model. If the `r` parameter is set to 0, the model is frozen instead.
+    """
+    lora_config = peft.LoraConfig(**lora_config or {})
+
+    if lora_config.r == 0:
+        # freeze the model entirely
+        for param in model.parameters():
+            param.requires_grad = False
+    else:
+        model = peft.get_peft_model(model, lora_config)
+
+    return model
+
+
+class StackAudioFrames(nn.Module):
+    """
+    Stack the audio embedding frames to reduce the sequence length by a factor of `stack_factor`.
+
+    The number of output frames will be `ceil(T / stack_factor) + 1` where `T` is the number of input frames.
+    NOTE: the extra +1 is intentional: in case the number of audio tokens are over-estimated by the processor,
+    we want to make sure `processor.audio_token_replacement` (i.e. EOS) doesn't get leaked into the middle of embeddings.
+    In most cases this extra padding will get removed in the model's forward function so it has no effect.
+    """
+
+    def __init__(self, stack_factor: int = 8):
+        super().__init__()
+        self.stack_factor = stack_factor
+
+    def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
+        B, T, C = audio_embeds.shape
+        T_pad = (T + self.stack_factor - 1) // self.stack_factor * self.stack_factor
+        audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T + self.stack_factor))
+        B, T, C = audio_embeds.shape
+        audio_embeds = audio_embeds.view(
+            B, T // self.stack_factor, C * self.stack_factor
+        )
+        return audio_embeds
+
+
+class RMSNorm(transformers.models.llama.modeling_llama.LlamaRMSNorm):
+    def __init__(self, hidden_size: int, init: float = 1, eps: float = 1e-6):
+        super().__init__(hidden_size=hidden_size, eps=eps)
+        self.weight.data.fill_(init)
+
+
+class SwiGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+
+class UltravoxProjector(nn.Sequential):
+    def __init__(self, config: UltravoxConfig):
+        super().__init__()
+        self.hidden_dim = config.hidden_size
+        self._pad_and_stack = StackAudioFrames(config.stack_factor)
+        dim = config.audio_config.hidden_size * config.stack_factor
+        self.ln_pre = RMSNorm(dim, init=config.norm_init)
+        self.linear_1 = nn.Linear(dim, self.hidden_dim, bias=False)
+        dim = self.hidden_dim
+        self.act = transformers.activations.get_activation(config.projector_act)
+        dim = dim // 2 if config.projector_act == "swiglu" else dim
+        self.linear_2 = nn.Linear(dim, config.text_config.hidden_size, bias=False)
+        self.ln_post = RMSNorm(config.text_config.hidden_size, init=config.norm_init)
+
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        audio_features = self._pad_and_stack(audio_features)
+        audio_features = self.ln_pre(audio_features)
+        hidden_states = self.linear_1(audio_features)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        hidden_states = self.ln_post(hidden_states)
+        return hidden_states
+
+
+class ModifiedWhisperEncoder(
+    whisper.WhisperEncoder, transformers.modeling_utils.ModuleUtilsMixin
+):
+    """
+    Encoder portion of OpenAI's Whisper model.
+
+    This implementation is a slightly modified version of HF Transformers' Whisper Encoder, with only a few fixes:
+    1. base_model_prefix updated to allow for doing `.from_pretrained` directly on the encoder
+    2. allow less than 30 second of audio padding to be passed in:
+        - relaxed ValueError check for `input_features` length to be less than or equal to `expected_seq_length` instead of strictly equal
+        - embed_pos is now sliced to match the length of `inputs_embeds`
+
+    Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
+    """
+
+    base_model_prefix = "model.encoder"
+    _no_split_modules = ["WhisperEncoderLayer"]
+
+    def init_latency_mask(self, audio_latency_block_size: int, dtype: torch.dtype):
+        if audio_latency_block_size is None:
+            self.audio_streaming_mask = None
+            return
+
+        # maximum sequence length
+        max_seqlen = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        assert (
+            max_seqlen > 0
+        ), f"maximum sequence length must be positive, got {max_seqlen}"
+        assert (
+            max_seqlen % audio_latency_block_size == 0
+        ), f"audio_latency_block_size {audio_latency_block_size} must divide {max_seqlen} evenly."
+        # Given the block size, we calculate number of blocks.
+        audio_latency_nblocks = max_seqlen // audio_latency_block_size
+        audio_streaming_mask = (
+            torch.tril(
+                torch.ones(audio_latency_nblocks, audio_latency_nblocks),
+                diagonal=0,
+            )
+            .repeat_interleave(audio_latency_block_size, dim=0)
+            .repeat_interleave(audio_latency_block_size, dim=1)
+        )
+        audio_streaming_mask = (1.0 - audio_streaming_mask) * torch.finfo(dtype).min
+        audio_streaming_mask = audio_streaming_mask[None, None, :, :]
+        self.register_buffer(
+            "audio_streaming_mask", audio_streaming_mask, persistent=False
+        )
+
+    def forward(
+        self,
+        input_features,
+        audio_len=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        expected_seq_length = (
+            self.config.max_source_positions
+            * self.conv1.stride[0]
+            * self.conv2.stride[0]
+        )
+        if input_features.shape[-1] > expected_seq_length:
+            raise ValueError(
+                f"Whisper expects the mel input features to be of length {expected_seq_length} or less, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}."
+            )
+
+        output_attentions = (
+            output_attentions
+            if output_attentions is not None
+            else self.config.output_attentions
+        )
+        output_hidden_states = (
+            output_hidden_states
+            if output_hidden_states is not None
+            else self.config.output_hidden_states
+        )
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(
+            hidden_states, p=self.dropout, training=self.training
+        )
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # Create attention mask based on audio lengths to mask out padding tokens
+        # For each sample in batch:
+        # - Convert raw audio length to feature length after convolutions
+        # - Create boolean mask that is True for valid positions and False for padding
+        # - Convert to extended attention mask format expected by transformer layers
+        #   (1.0 for positions to attend to, large negative for positions to ignore)
+        # This masking ensures consistent behavior between training and inference
+        # by preventing the model from attending to padding tokens in both cases
+        attention_mask = None
+        if audio_len != None:
+            audio_feature_len = self._get_feat_extract_output_lengths(audio_len)
+            max_seq_len = hidden_states.shape[1]
+            attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
+                None, :
+            ].lt(audio_feature_len.view(-1, 1))
+            attention_mask = self.get_extended_attention_mask(
+                attention_mask,
+                None,
+                device=hidden_states.device,
+                dtype=hidden_states.dtype,
+            )
+
+        if self.audio_streaming_mask is not None:
+            seqlen = hidden_states.size(-2)
+            if attention_mask is not None:
+                attention_mask = torch.minimum(
+                    self.audio_streaming_mask[:, :, :seqlen, :seqlen], attention_mask
+                )  # merge
+            else:
+                attention_mask = self.audio_streaming_mask[:, :, :seqlen, :seqlen]
+            attention_mask = attention_mask.to(hidden_states.dtype)
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (
+                len(self.layers)
+            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        attention_mask,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        attention_mask,
+                        layer_head_mask=(
+                            head_mask[idx] if head_mask is not None else None
+                        ),
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [hidden_states, encoder_states, all_attentions]
+                if v is not None
+            )
+        return transformers.modeling_outputs.BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=encoder_states,
+            attentions=all_attentions,
+        )
+
+
+UltravoxConfig.register_for_auto_class()
+UltravoxModel.register_for_auto_class()
+
+transformers.AutoConfig.register("ultravox", UltravoxConfig)
+transformers.AutoModel.register(UltravoxConfig, UltravoxModel)
+
+transformers.activations.ACT2FN["swiglu"] = SwiGLU