diff --git "a/audio_diffusion_attacks_forhf/notebooks/data_exploration/00_fma_exploration.ipynb" "b/audio_diffusion_attacks_forhf/notebooks/data_exploration/00_fma_exploration.ipynb" new file mode 100644--- /dev/null +++ "b/audio_diffusion_attacks_forhf/notebooks/data_exploration/00_fma_exploration.ipynb" @@ -0,0 +1,674 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 00 - Free Music Archive (FMA) Exploration \n", + "\n", + "A notebook for exploring high-level details of the FMA, to try and understand best ways to split and encode it " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np \n", + "import pandas as pd \n", + "import matplotlib.pyplot as plt \n", + "import seaborn as sns \n", + "import IPython\n", + "\n", + "import torch\n", + "import torchaudio\n", + "import os\n", + "import ast\n", + "\n", + "# Old Code for Importing AudioLDM\n", + "# from audioldm.pipeline import build_model\n", + "\n", + "# HF Code for AudioLDM2\n", + "from diffusers import AudioLDM2Pipeline\n", + "\n", + "# Change the below to desired GPU if using GPU\n", + "os.environ['CUDA_VISIBLE_DEVICES'] = '4'" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "## Helper Function for laoding track\n", + "def load(filepath):\n", + "\n", + " filename = os.path.basename(filepath)\n", + "\n", + " if 'features' in filename:\n", + " return pd.read_csv(filepath, index_col=0, header=[0, 1, 2])\n", + "\n", + " if 'echonest' in filename:\n", + " return pd.read_csv(filepath, index_col=0, header=[0, 1, 2])\n", + "\n", + " if 'genres' in filename:\n", + " return pd.read_csv(filepath, index_col=0)\n", + "\n", + " if 'tracks' in filename:\n", + " tracks = pd.read_csv(filepath, index_col=0, header=[0, 1])\n", + "\n", + " COLUMNS = [('track', 'tags'), ('album', 'tags'), ('artist', 'tags'),\n", + " ('track', 'genres'), ('track', 'genres_all')]\n", + " for column in COLUMNS:\n", + " tracks[column] = tracks[column].map(ast.literal_eval)\n", + "\n", + " COLUMNS = [('track', 'date_created'), ('track', 'date_recorded'),\n", + " ('album', 'date_created'), ('album', 'date_released'),\n", + " ('artist', 'date_created'), ('artist', 'active_year_begin'),\n", + " ('artist', 'active_year_end')]\n", + " for column in COLUMNS:\n", + " tracks[column] = pd.to_datetime(tracks[column])\n", + "\n", + " SUBSETS = ('small', 'medium', 'large')\n", + " try:\n", + " tracks['set', 'subset'] = tracks['set', 'subset'].astype(\n", + " 'category', categories=SUBSETS, ordered=True)\n", + " except (ValueError, TypeError):\n", + " # the categories and ordered arguments were removed in pandas 0.25\n", + " tracks['set', 'subset'] = tracks['set', 'subset'].astype(\n", + " pd.CategoricalDtype(categories=SUBSETS, ordered=True))\n", + "\n", + " COLUMNS = [('track', 'genre_top'), ('track', 'license'),\n", + " ('album', 'type'), ('album', 'information'),\n", + " ('artist', 'bio')]\n", + " for column in COLUMNS:\n", + " tracks[column] = tracks[column].astype('category')\n", + "\n", + " return tracks" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "## Set the Data Location. For this notebook, using Robin's data loc\n", + "data_loc = '/data/robbizorg/music_datasets/fma/'" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "## Let's Explore the Metadata first\n", + "# Load metadata and features.\n", + "tracks = load(os.path.join(data_loc, 'data/fma_metadata/tracks.csv'))\n", + "genres = load(os.path.join(data_loc, 'data/fma_metadata/genres.csv'))\n", + "features = load(os.path.join(data_loc, 'data/fma_metadata/features.csv'))\n", + "echonest = load(os.path.join(data_loc, 'data/fma_metadata/echonest.csv'))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
#tracksparenttitletop_level
genre_id
1869338Avant-Garde38
252710International2
317520Blues3
441260Jazz4
541060Classical5
\n", + "
" + ], + "text/plain": [ + " #tracks parent title top_level\n", + "genre_id \n", + "1 8693 38 Avant-Garde 38\n", + "2 5271 0 International 2\n", + "3 1752 0 Blues 3\n", + "4 4126 0 Jazz 4\n", + "5 4106 0 Classical 5" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "genres.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
album...track
commentsdate_createddate_releasedengineerfavoritesidinformationlistensproducertags...informationinterestlanguage_codelicenselistenslyricistnumberpublishertagstitle
track_id
202008-11-26 01:44:452009-01-05NaN41<p></p>6073NaN[]...NaN4656enAttribution-NonCommercial-ShareAlike 3.0 Inter...1293NaN3NaN[]Food
302008-11-26 01:44:452009-01-05NaN41<p></p>6073NaN[]...NaN1470enAttribution-NonCommercial-ShareAlike 3.0 Inter...514NaN4NaN[]Electric Ave
502008-11-26 01:44:452009-01-05NaN41<p></p>6073NaN[]...NaN1933enAttribution-NonCommercial-ShareAlike 3.0 Inter...1151NaN6NaN[]This World
1002008-11-26 01:45:082008-02-06NaN46NaN47632NaN[]...NaN54881enAttribution-NonCommercial-NoDerivatives (aka M...50135NaN1NaN[]Freeway
2002008-11-26 01:45:052009-01-06NaN24<p> \"spiritual songs\" from Nicky Cook</p>2710NaN[]...NaN978enAttribution-NonCommercial-NoDerivatives (aka M...361NaN3NaN[]Spiritual Level
\n", + "

5 rows × 52 columns

\n", + "
" + ], + "text/plain": [ + " album \\\n", + " comments date_created date_released engineer favorites id \n", + "track_id \n", + "2 0 2008-11-26 01:44:45 2009-01-05 NaN 4 1 \n", + "3 0 2008-11-26 01:44:45 2009-01-05 NaN 4 1 \n", + "5 0 2008-11-26 01:44:45 2009-01-05 NaN 4 1 \n", + "10 0 2008-11-26 01:45:08 2008-02-06 NaN 4 6 \n", + "20 0 2008-11-26 01:45:05 2009-01-06 NaN 2 4 \n", + "\n", + " \\\n", + " information listens producer tags \n", + "track_id \n", + "2

6073 NaN [] \n", + "3

6073 NaN [] \n", + "5

6073 NaN [] \n", + "10 NaN 47632 NaN [] \n", + "20

 \"spiritual songs\" from Nicky Cook

2710 NaN [] \n", + "\n", + " ... track \\\n", + " ... information interest language_code \n", + "track_id ... \n", + "2 ... NaN 4656 en \n", + "3 ... NaN 1470 en \n", + "5 ... NaN 1933 en \n", + "10 ... NaN 54881 en \n", + "20 ... NaN 978 en \n", + "\n", + " \\\n", + " license listens lyricist \n", + "track_id \n", + "2 Attribution-NonCommercial-ShareAlike 3.0 Inter... 1293 NaN \n", + "3 Attribution-NonCommercial-ShareAlike 3.0 Inter... 514 NaN \n", + "5 Attribution-NonCommercial-ShareAlike 3.0 Inter... 1151 NaN \n", + "10 Attribution-NonCommercial-NoDerivatives (aka M... 50135 NaN \n", + "20 Attribution-NonCommercial-NoDerivatives (aka M... 361 NaN \n", + "\n", + " \n", + " number publisher tags title \n", + "track_id \n", + "2 3 NaN [] Food \n", + "3 4 NaN [] Electric Ave \n", + "5 6 NaN [] This World \n", + "10 1 NaN [] Freeway \n", + "20 3 NaN [] Spiritual Level \n", + "\n", + "[5 rows x 52 columns]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tracks.head()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Explore Audio and Encoding\n", + "\n", + "We're going to test out encoding and decoding with AudioLDM's trained encoder" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Audio Size and Sampling Rate\n", + "torch.Size([2, 1321967]) 44100\n", + "Audio is 0.5 minutes long\n" + ] + } + ], + "source": [ + "example_audio_loc = os.path.join(data_loc, 'data/fma_large/000/000418.mp3')\n", + "audio, sr = torchaudio.load(example_audio_loc)\n", + "mono_audio = torch.mean(audio, axis = 0) # Convert to Mono\n", + "\n", + "print(\"Audio Size and Sampling Rate\")\n", + "print(audio.shape, sr)\n", + "\n", + "print(f\"Audio is {np.round(audio.shape[-1]/sr/60, 2)} minutes long\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Play sample of audio \n", + "IPython.display.Audio(audio, rate = sr)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "device = 'cuda' if torch.cuda.is_available() else 'cpu'" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "94fbd16338584380b82375cc5b7d51a2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading pipeline components...: 0%| | 0/11 [00:00= 0 && device < num_gpus INTERNAL ASSERT FAILED at \"../aten/src/ATen/cuda/CUDAContext.cpp\":50, please report a bug to PyTorch. device=1, num_gpus=\u0001\n\nCUDA call was originally invoked at:\n\n File \"/home/robbizorg/miniconda3/lib/python3.10/runpy.py\", line 196, in _run_module_as_main\n return _run_code(code, main_globals, None,\n File \"/home/robbizorg/miniconda3/lib/python3.10/runpy.py\", line 86, in _run_code\n exec(code, run_globals)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel_launcher.py\", line 17, in \n app.launch_new_instance()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/traitlets/config/application.py\", line 992, in launch_instance\n app.start()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelapp.py\", line 711, in start\n self.io_loop.start()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/tornado/platform/asyncio.py\", line 215, in start\n self.asyncio_loop.run_forever()\n File \"/home/robbizorg/miniconda3/lib/python3.10/asyncio/base_events.py\", line 603, in run_forever\n self._run_once()\n File \"/home/robbizorg/miniconda3/lib/python3.10/asyncio/base_events.py\", line 1906, in _run_once\n handle._run()\n File \"/home/robbizorg/miniconda3/lib/python3.10/asyncio/events.py\", line 80, in _run\n self._context.run(self._callback, *self._args)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n await self.process_one()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n await dispatch(*args)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n await result\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 729, in execute_request\n reply_content = await reply_content\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/ipkernel.py\", line 411, in do_execute\n res = shell.run_cell(\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/zmqshell.py\", line 531, in run_cell\n return super().run_cell(*args, **kwargs)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3006, in run_cell\n result = self._run_cell(\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3061, in _run_cell\n result = runner(coro)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n coro.send(None)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3266, in run_cell_async\n has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3445, in run_ast_nodes\n if await self.run_code(code, result, async_=asy):\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3505, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"/tmp/ipykernel_2112872/2098344850.py\", line 7, in \n import torch\n File \"\", line 1027, in _find_and_load\n File \"\", line 1006, in _find_and_load_unlocked\n File \"\", line 688, in _load_unlocked\n File \"\", line 883, in exec_module\n File \"\", line 241, in _call_with_frames_removed\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/torch/__init__.py\", line 1427, in \n _C._initExtension(manager_path())\n File \"\", line 1027, in _find_and_load\n File \"\", line 1006, in _find_and_load_unlocked\n File \"\", line 688, in _load_unlocked\n File \"\", line 883, in exec_module\n File \"\", line 241, in _call_with_frames_removed\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py\", line 247, in \n _lazy_call(_check_capability)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py\", line 244, in _lazy_call\n _queued_calls.append((callable, traceback.format_stack()))\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py:315\u001b[0m, in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 314\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 315\u001b[0m queued_call()\n\u001b[1;32m 316\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py:183\u001b[0m, in \u001b[0;36m_check_capability\u001b[0;34m()\u001b[0m\n\u001b[1;32m 182\u001b[0m \u001b[39mfor\u001b[39;00m d \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(device_count()):\n\u001b[0;32m--> 183\u001b[0m capability \u001b[39m=\u001b[39m get_device_capability(d)\n\u001b[1;32m 184\u001b[0m major \u001b[39m=\u001b[39m capability[\u001b[39m0\u001b[39m]\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py:439\u001b[0m, in \u001b[0;36mget_device_capability\u001b[0;34m(device)\u001b[0m\n\u001b[1;32m 427\u001b[0m \u001b[39m\u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Get the cuda capability of a device.\u001b[39;00m\n\u001b[1;32m 428\u001b[0m \n\u001b[1;32m 429\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 437\u001b[0m \u001b[39m tuple(int, int): the major and minor cuda capability of the device\u001b[39;00m\n\u001b[1;32m 438\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 439\u001b[0m prop \u001b[39m=\u001b[39m get_device_properties(device)\n\u001b[1;32m 440\u001b[0m \u001b[39mreturn\u001b[39;00m prop\u001b[39m.\u001b[39mmajor, prop\u001b[39m.\u001b[39mminor\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py:457\u001b[0m, in \u001b[0;36mget_device_properties\u001b[0;34m(device)\u001b[0m\n\u001b[1;32m 456\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mAssertionError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mInvalid device id\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 457\u001b[0m \u001b[39mreturn\u001b[39;00m _get_device_properties(device)\n", + "\u001b[0;31mRuntimeError\u001b[0m: device >= 0 && device < num_gpus INTERNAL ASSERT FAILED at \"../aten/src/ATen/cuda/CUDAContext.cpp\":50, please report a bug to PyTorch. device=1, num_gpus=\u0001", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[0;31mDeferredCudaCallError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m repo_id \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mcvssp/audioldm2\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 3\u001b[0m pipe \u001b[39m=\u001b[39m AudioLDM2Pipeline\u001b[39m.\u001b[39mfrom_pretrained(repo_id, torch_dtype \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mfloat16)\n\u001b[0;32m----> 4\u001b[0m pipe \u001b[39m=\u001b[39m pipe\u001b[39m.\u001b[39;49mto(\u001b[39m'\u001b[39;49m\u001b[39mcuda\u001b[39;49m\u001b[39m'\u001b[39;49m)\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/diffusers/pipelines/pipeline_utils.py:418\u001b[0m, in \u001b[0;36mDiffusionPipeline.to\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 414\u001b[0m logger\u001b[39m.\u001b[39mwarning(\n\u001b[1;32m 415\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mThe module \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mmodule\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m has been loaded in 8bit and moving it to \u001b[39m\u001b[39m{\u001b[39;00mdtype\u001b[39m}\u001b[39;00m\u001b[39m via `.to()` is not yet supported. Module is still on \u001b[39m\u001b[39m{\u001b[39;00mmodule\u001b[39m.\u001b[39mdevice\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 416\u001b[0m )\n\u001b[1;32m 417\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 418\u001b[0m module\u001b[39m.\u001b[39;49mto(device, dtype)\n\u001b[1;32m 420\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m 421\u001b[0m module\u001b[39m.\u001b[39mdtype \u001b[39m==\u001b[39m torch\u001b[39m.\u001b[39mfloat16\n\u001b[1;32m 422\u001b[0m \u001b[39mand\u001b[39;00m \u001b[39mstr\u001b[39m(device) \u001b[39min\u001b[39;00m [\u001b[39m\"\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 423\u001b[0m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m silence_dtype_warnings\n\u001b[1;32m 424\u001b[0m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m is_offloaded\n\u001b[1;32m 425\u001b[0m ):\n\u001b[1;32m 426\u001b[0m logger\u001b[39m.\u001b[39mwarning(\n\u001b[1;32m 427\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mPipelines loaded with `dtype=torch.float16` cannot run with `cpu` device. It\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 428\u001b[0m \u001b[39m\"\u001b[39m\u001b[39m is not recommended to move them to `cpu` as running them will fail. Please make\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 431\u001b[0m \u001b[39m\"\u001b[39m\u001b[39m `torch_dtype=torch.float16` argument, or use another device for inference.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 432\u001b[0m )\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1152\u001b[0m, in \u001b[0;36mModule.to\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1148\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39mto(device, dtype \u001b[39mif\u001b[39;00m t\u001b[39m.\u001b[39mis_floating_point() \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mis_complex() \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m 1149\u001b[0m non_blocking, memory_format\u001b[39m=\u001b[39mconvert_to_format)\n\u001b[1;32m 1150\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39mto(device, dtype \u001b[39mif\u001b[39;00m t\u001b[39m.\u001b[39mis_floating_point() \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mis_complex() \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m, non_blocking)\n\u001b[0;32m-> 1152\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_apply(convert)\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:802\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn, recurse)\u001b[0m\n\u001b[1;32m 800\u001b[0m \u001b[39mif\u001b[39;00m recurse:\n\u001b[1;32m 801\u001b[0m \u001b[39mfor\u001b[39;00m module \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mchildren():\n\u001b[0;32m--> 802\u001b[0m module\u001b[39m.\u001b[39;49m_apply(fn)\n\u001b[1;32m 804\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied):\n\u001b[1;32m 805\u001b[0m \u001b[39mif\u001b[39;00m torch\u001b[39m.\u001b[39m_has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[1;32m 806\u001b[0m \u001b[39m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[1;32m 807\u001b[0m \u001b[39m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 812\u001b[0m \u001b[39m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[1;32m 813\u001b[0m \u001b[39m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:825\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn, recurse)\u001b[0m\n\u001b[1;32m 821\u001b[0m \u001b[39m# Tensors stored in modules are graph leaves, and we don't want to\u001b[39;00m\n\u001b[1;32m 822\u001b[0m \u001b[39m# track autograd history of `param_applied`, so we have to use\u001b[39;00m\n\u001b[1;32m 823\u001b[0m \u001b[39m# `with torch.no_grad():`\u001b[39;00m\n\u001b[1;32m 824\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[0;32m--> 825\u001b[0m param_applied \u001b[39m=\u001b[39m fn(param)\n\u001b[1;32m 826\u001b[0m should_use_set_data \u001b[39m=\u001b[39m compute_should_use_set_data(param, param_applied)\n\u001b[1;32m 827\u001b[0m \u001b[39mif\u001b[39;00m should_use_set_data:\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1150\u001b[0m, in \u001b[0;36mModule.to..convert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m 1147\u001b[0m \u001b[39mif\u001b[39;00m convert_to_format \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m t\u001b[39m.\u001b[39mdim() \u001b[39min\u001b[39;00m (\u001b[39m4\u001b[39m, \u001b[39m5\u001b[39m):\n\u001b[1;32m 1148\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39mto(device, dtype \u001b[39mif\u001b[39;00m t\u001b[39m.\u001b[39mis_floating_point() \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mis_complex() \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m 1149\u001b[0m non_blocking, memory_format\u001b[39m=\u001b[39mconvert_to_format)\n\u001b[0;32m-> 1150\u001b[0m \u001b[39mreturn\u001b[39;00m t\u001b[39m.\u001b[39;49mto(device, dtype \u001b[39mif\u001b[39;49;00m t\u001b[39m.\u001b[39;49mis_floating_point() \u001b[39mor\u001b[39;49;00m t\u001b[39m.\u001b[39;49mis_complex() \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m, non_blocking)\n", + "File \u001b[0;32m~/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py:321\u001b[0m, in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m 316\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 317\u001b[0m msg \u001b[39m=\u001b[39m (\n\u001b[1;32m 318\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCUDA call failed lazily at initialization with error: \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mstr\u001b[39m(e)\u001b[39m}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 319\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCUDA call was originally invoked at:\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m{\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mjoin(orig_traceback)\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 320\u001b[0m )\n\u001b[0;32m--> 321\u001b[0m \u001b[39mraise\u001b[39;00m DeferredCudaCallError(msg) \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n\u001b[1;32m 322\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 323\u001b[0m \u001b[39mdelattr\u001b[39m(_tls, \u001b[39m\"\u001b[39m\u001b[39mis_initializing\u001b[39m\u001b[39m\"\u001b[39m)\n", + "\u001b[0;31mDeferredCudaCallError\u001b[0m: CUDA call failed lazily at initialization with error: device >= 0 && device < num_gpus INTERNAL ASSERT FAILED at \"../aten/src/ATen/cuda/CUDAContext.cpp\":50, please report a bug to PyTorch. device=1, num_gpus=\u0001\n\nCUDA call was originally invoked at:\n\n File \"/home/robbizorg/miniconda3/lib/python3.10/runpy.py\", line 196, in _run_module_as_main\n return _run_code(code, main_globals, None,\n File \"/home/robbizorg/miniconda3/lib/python3.10/runpy.py\", line 86, in _run_code\n exec(code, run_globals)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel_launcher.py\", line 17, in \n app.launch_new_instance()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/traitlets/config/application.py\", line 992, in launch_instance\n app.start()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelapp.py\", line 711, in start\n self.io_loop.start()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/tornado/platform/asyncio.py\", line 215, in start\n self.asyncio_loop.run_forever()\n File \"/home/robbizorg/miniconda3/lib/python3.10/asyncio/base_events.py\", line 603, in run_forever\n self._run_once()\n File \"/home/robbizorg/miniconda3/lib/python3.10/asyncio/base_events.py\", line 1906, in _run_once\n handle._run()\n File \"/home/robbizorg/miniconda3/lib/python3.10/asyncio/events.py\", line 80, in _run\n self._context.run(self._callback, *self._args)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n await self.process_one()\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n await dispatch(*args)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n await result\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/kernelbase.py\", line 729, in execute_request\n reply_content = await reply_content\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/ipkernel.py\", line 411, in do_execute\n res = shell.run_cell(\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/ipykernel/zmqshell.py\", line 531, in run_cell\n return super().run_cell(*args, **kwargs)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3006, in run_cell\n result = self._run_cell(\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3061, in _run_cell\n result = runner(coro)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n coro.send(None)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3266, in run_cell_async\n has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3445, in run_ast_nodes\n if await self.run_code(code, result, async_=asy):\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/IPython/core/interactiveshell.py\", line 3505, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"/tmp/ipykernel_2112872/2098344850.py\", line 7, in \n import torch\n File \"\", line 1027, in _find_and_load\n File \"\", line 1006, in _find_and_load_unlocked\n File \"\", line 688, in _load_unlocked\n File \"\", line 883, in exec_module\n File \"\", line 241, in _call_with_frames_removed\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/torch/__init__.py\", line 1427, in \n _C._initExtension(manager_path())\n File \"\", line 1027, in _find_and_load\n File \"\", line 1006, in _find_and_load_unlocked\n File \"\", line 688, in _load_unlocked\n File \"\", line 883, in exec_module\n File \"\", line 241, in _call_with_frames_removed\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py\", line 247, in \n _lazy_call(_check_capability)\n File \"/home/robbizorg/miniconda3/lib/python3.10/site-packages/torch/cuda/__init__.py\", line 244, in _lazy_call\n _queued_calls.append((callable, traceback.format_stack()))\n" + ] + } + ], + "source": [ + "# Instantiate AudioLDM\n", + "repo_id = \"cvssp/audioldm2\"\n", + "pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype = torch.float16)\n", + "pipe = pipe.to('cuda')\n", + "\n", + "# audio_codec = build_model().to(device)\n", + "# audio_codec.latent_t_size = 256" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click here for more info. View Jupyter log for further details." + ] + }, + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mFailed to interrupt the Kernel. \n", + "\u001b[1;31mCannot execute code, session has been disposed. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], + "source": [ + "pipe" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}