+
+
+
+
diff --git a/frontend/src/components/TheWelcome.vue b/frontend/src/components/TheWelcome.vue
new file mode 100644
index 0000000000000000000000000000000000000000..a70765c231fe6066338c8d3aee77f3091d46d8cc
--- /dev/null
+++ b/frontend/src/components/TheWelcome.vue
@@ -0,0 +1,86 @@
+
+
+
+ {{ title }}
+ ++ + More instructions are available in
README.md.
+
+
+
+
+
+
diff --git a/frontend/src/components/icons/IconCommunity.vue b/frontend/src/components/icons/IconCommunity.vue
new file mode 100644
index 0000000000000000000000000000000000000000..2dc8b055253af30fb797037e2fe260505f0cf711
--- /dev/null
+++ b/frontend/src/components/icons/IconCommunity.vue
@@ -0,0 +1,7 @@
+
+
+
diff --git a/frontend/src/components/icons/IconDocumentation.vue b/frontend/src/components/icons/IconDocumentation.vue
new file mode 100644
index 0000000000000000000000000000000000000000..6d4791cfbcf2782b3e5ffbabd042d4c47b2fbbed
--- /dev/null
+++ b/frontend/src/components/icons/IconDocumentation.vue
@@ -0,0 +1,7 @@
+
+
+
diff --git a/frontend/src/components/icons/IconEcosystem.vue b/frontend/src/components/icons/IconEcosystem.vue
new file mode 100644
index 0000000000000000000000000000000000000000..c3a4f078c0bd340a33c61ea9ecd8a755d03571ed
--- /dev/null
+++ b/frontend/src/components/icons/IconEcosystem.vue
@@ -0,0 +1,7 @@
+
+
+
diff --git a/frontend/src/components/icons/IconSupport.vue b/frontend/src/components/icons/IconSupport.vue
new file mode 100644
index 0000000000000000000000000000000000000000..7452834d3ef961ce24c3a072ddba2620b6158bae
--- /dev/null
+++ b/frontend/src/components/icons/IconSupport.vue
@@ -0,0 +1,7 @@
+
+
+
diff --git a/frontend/src/components/icons/IconTooling.vue b/frontend/src/components/icons/IconTooling.vue
new file mode 100644
index 0000000000000000000000000000000000000000..660598d7c76644ffe126a1a1feb1606650bfb937
--- /dev/null
+++ b/frontend/src/components/icons/IconTooling.vue
@@ -0,0 +1,19 @@
+
+
+
+
diff --git a/frontend/src/layout/LayoutFrame.vue b/frontend/src/layout/LayoutFrame.vue
new file mode 100644
index 0000000000000000000000000000000000000000..03b8d5d1452b28b126e4571721e860ed457fe6c2
--- /dev/null
+++ b/frontend/src/layout/LayoutFrame.vue
@@ -0,0 +1,151 @@
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+ SVS SYSTEM
+v1.0.0
+
+
+
+
+
+
diff --git a/frontend/src/views/HomeView.vue b/frontend/src/views/HomeView.vue
new file mode 100644
index 0000000000000000000000000000000000000000..9fa1c4d028c2875bc1eae1c8a973429f19fd5af8
--- /dev/null
+++ b/frontend/src/views/HomeView.vue
@@ -0,0 +1,10 @@
+
+
+
+ This is an about page
+
+
+
+
+
+
+
diff --git a/frontend/src/views/SVCView.vue b/frontend/src/views/SVCView.vue
new file mode 100644
index 0000000000000000000000000000000000000000..f992c4fb2329c075efdfd9c6c45a6cd1992c6ecf
--- /dev/null
+++ b/frontend/src/views/SVCView.vue
@@ -0,0 +1,320 @@
+
+
+
+
+
+
+
+
+
+ 选择音频文件并开始转换！
+
+
+
+
+
+
+
+ 歌曲音声分离
+上传歌曲，运行可下载人声（vocals.wav）与背景音乐（accompaniment.wav）
+
+ 需要是wav或者mp3文件哦
+
+
+
+
+
+
+
+
+
diff --git a/frontend/tsconfig.app.json b/frontend/tsconfig.app.json
new file mode 100644
index 0000000000000000000000000000000000000000..f8e4ca8541be2561c8ecc8c821735423182f7532
--- /dev/null
+++ b/frontend/tsconfig.app.json
@@ -0,0 +1,14 @@
+{
+ "extends": "@vue/tsconfig/tsconfig.web.json",
+ "include": [
+ "env.d.ts", "src/**/*", "src/**/*.vue"],
+ "exclude": ["src/**/__tests__/*"],
+ "compilerOptions": {
+ "composite": true,
+ "baseUrl": "./lilyn-svs",
+ "allowJs": true,
+ "paths": {
+ "@/*": ["./src/*"]
+ }
+ }
+}
diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json
new file mode 100644
index 0000000000000000000000000000000000000000..100cf6a8f2b8b43c3fba052e5241951ecdd7a384
--- /dev/null
+++ b/frontend/tsconfig.json
@@ -0,0 +1,14 @@
+{
+ "files": [],
+ "references": [
+ {
+ "path": "./tsconfig.node.json"
+ },
+ {
+ "path": "./tsconfig.app.json"
+ },
+ {
+ "path": "./tsconfig.vitest.json"
+ }
+ ]
+}
diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json
new file mode 100644
index 0000000000000000000000000000000000000000..36b68aada8ab3a288f25f2a99f722a485d26e3e8
--- /dev/null
+++ b/frontend/tsconfig.node.json
@@ -0,0 +1,9 @@
+{
+ "extends": "@vue/tsconfig/tsconfig.node.json",
+ "include": ["vite.config.*", "vitest.config.*", "cypress.config.*", "playwright.config.*"],
+ "compilerOptions": {
+ "composite": true,
+ "types": ["node"],
+ "allowJs": true
+ }
+}
diff --git a/frontend/tsconfig.vitest.json b/frontend/tsconfig.vitest.json
new file mode 100644
index 0000000000000000000000000000000000000000..05b1466e8c6530c83fb9807eba1cfd64008e4d42
--- /dev/null
+++ b/frontend/tsconfig.vitest.json
@@ -0,0 +1,10 @@
+{
+ "extends": "./tsconfig.app.json",
+ "exclude": [],
+ "compilerOptions": {
+ "composite": true,
+ "lib": [],
+ "types": ["node", "jsdom"],
+ "allowJs": true
+ }
+}
diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts
new file mode 100644
index 0000000000000000000000000000000000000000..de5cb31c632332a5867502004ce2ade73d9df29f
--- /dev/null
+++ b/frontend/vite.config.ts
@@ -0,0 +1,14 @@
+import { fileURLToPath, URL } from 'node:url'
+
+import { defineConfig } from 'vite'
+import vue from '@vitejs/plugin-vue'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+ plugins: [vue()],
+ resolve: {
+ alias: {
+ '@': fileURLToPath(new URL('./src', import.meta.url))
+ }
+ }
+})
diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts
new file mode 100644
index 0000000000000000000000000000000000000000..a2f57a5d5e8e226cc1a93515d7942b2c74bef0a7
--- /dev/null
+++ b/frontend/vitest.config.ts
@@ -0,0 +1,15 @@
+import { fileURLToPath } from 'node:url'
+import { mergeConfig } from 'vite'
+import { configDefaults, defineConfig } from 'vitest/config'
+import viteConfig from './vite.config'
+
+export default mergeConfig(
+ viteConfig,
+ defineConfig({
+ test: {
+ environment: 'jsdom',
+ exclude: [...configDefaults.exclude, 'e2e/*'],
+ root: fileURLToPath(new URL('./', import.meta.url))
+ }
+ })
+)
diff --git a/hubert/__init__.py b/hubert/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/hubert/hubert_model.py b/hubert/hubert_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fb642d89b07ca60792debab18e3454f52d8f357
--- /dev/null
+++ b/hubert/hubert_model.py
@@ -0,0 +1,222 @@
+import copy
+import random
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as t_func
+from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
+
+
+class Hubert(nn.Module):
+ def __init__(self, num_label_embeddings: int = 100, mask: bool = True):
+ super().__init__()
+ self._mask = mask
+ self.feature_extractor = FeatureExtractor()
+ self.feature_projection = FeatureProjection()
+ self.positional_embedding = PositionalConvEmbedding()
+ self.norm = nn.LayerNorm(768)
+ self.dropout = nn.Dropout(0.1)
+ self.encoder = TransformerEncoder(
+ nn.TransformerEncoderLayer(
+ 768, 12, 3072, activation="gelu", batch_first=True
+ ),
+ 12,
+ )
+ self.proj = nn.Linear(768, 256)
+
+ self.masked_spec_embed = nn.Parameter(torch.FloatTensor(768).uniform_())
+ self.label_embedding = nn.Embedding(num_label_embeddings, 256)
+
+ def mask(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ mask = None
+ if self.training and self._mask:
+ mask = _compute_mask((x.size(0), x.size(1)), 0.8, 10, x.device, 2)
+ x[mask] = self.masked_spec_embed.to(x.dtype)
+ return x, mask
+
+ def encode(
+ self, x: torch.Tensor, layer: Optional[int] = None
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ x = self.feature_extractor(x)
+ x = self.feature_projection(x.transpose(1, 2))
+ x, mask = self.mask(x)
+ x = x + self.positional_embedding(x)
+ x = self.dropout(self.norm(x))
+ x = self.encoder(x, output_layer=layer)
+ return x, mask
+
+ def logits(self, x: torch.Tensor) -> torch.Tensor:
+ logits = torch.cosine_similarity(
+ x.unsqueeze(2),
+ self.label_embedding.weight.unsqueeze(0).unsqueeze(0),
+ dim=-1,
+ )
+ return logits / 0.1
+
+ def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ x, mask = self.encode(x)
+ x = self.proj(x)
+ logits = self.logits(x)
+ return logits, mask
+
+
+class HubertSoft(Hubert):
+ def __init__(self):
+ super().__init__()
+
+ @torch.inference_mode()
+ def units(self, wav: torch.Tensor) -> torch.Tensor:
+ wav = t_func.pad(wav, ((400 - 320) // 2, (400 - 320) // 2))
+ x, _ = self.encode(wav)
+ return self.proj(x)
+
+
+class FeatureExtractor(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.conv0 = nn.Conv1d(1, 512, 10, 5, bias=False)
+ self.norm0 = nn.GroupNorm(512, 512)
+ self.conv1 = nn.Conv1d(512, 512, 3, 2, bias=False)
+ self.conv2 = nn.Conv1d(512, 512, 3, 2, bias=False)
+ self.conv3 = nn.Conv1d(512, 512, 3, 2, bias=False)
+ self.conv4 = nn.Conv1d(512, 512, 3, 2, bias=False)
+ self.conv5 = nn.Conv1d(512, 512, 2, 2, bias=False)
+ self.conv6 = nn.Conv1d(512, 512, 2, 2, bias=False)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x = t_func.gelu(self.norm0(self.conv0(x)))
+ x = t_func.gelu(self.conv1(x))
+ x = t_func.gelu(self.conv2(x))
+ x = t_func.gelu(self.conv3(x))
+ x = t_func.gelu(self.conv4(x))
+ x = t_func.gelu(self.conv5(x))
+ x = t_func.gelu(self.conv6(x))
+ return x
+
+
+class FeatureProjection(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.norm = nn.LayerNorm(512)
+ self.projection = nn.Linear(512, 768)
+ self.dropout = nn.Dropout(0.1)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x = self.norm(x)
+ x = self.projection(x)
+ x = self.dropout(x)
+ return x
+
+
+class PositionalConvEmbedding(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.conv = nn.Conv1d(
+ 768,
+ 768,
+ kernel_size=128,
+ padding=128 // 2,
+ groups=16,
+ )
+ self.conv = nn.utils.weight_norm(self.conv, name="weight", dim=2)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x = self.conv(x.transpose(1, 2))
+ x = t_func.gelu(x[:, :, :-1])
+ return x.transpose(1, 2)
+
+
+class TransformerEncoder(nn.Module):
+ def __init__(
+ self, encoder_layer: nn.TransformerEncoderLayer, num_layers: int
+ ) -> None:
+ super(TransformerEncoder, self).__init__()
+ self.layers = nn.ModuleList(
+ [copy.deepcopy(encoder_layer) for _ in range(num_layers)]
+ )
+ self.num_layers = num_layers
+
+ def forward(
+ self,
+ src: torch.Tensor,
+ mask: torch.Tensor = None,
+ src_key_padding_mask: torch.Tensor = None,
+ output_layer: Optional[int] = None,
+ ) -> torch.Tensor:
+ output = src
+ for layer in self.layers[:output_layer]:
+ output = layer(
+ output, src_mask=mask, src_key_padding_mask=src_key_padding_mask
+ )
+ return output
+
+
+def _compute_mask(
+ shape: Tuple[int, int],
+ mask_prob: float,
+ mask_length: int,
+ device: torch.device,
+ min_masks: int = 0,
+) -> torch.Tensor:
+ batch_size, sequence_length = shape
+
+ if mask_length < 1:
+ raise ValueError("`mask_length` has to be bigger than 0.")
+
+ if mask_length > sequence_length:
+ raise ValueError(
+ f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length} and `sequence_length`: {sequence_length}`"
+ )
+
+ # compute number of masked spans in batch
+ num_masked_spans = int(mask_prob * sequence_length / mask_length + random.random())
+ num_masked_spans = max(num_masked_spans, min_masks)
+
+ # make sure num masked indices <= sequence_length
+ if num_masked_spans * mask_length > sequence_length:
+ num_masked_spans = sequence_length // mask_length
+
+ # SpecAugment mask to fill
+ mask = torch.zeros((batch_size, sequence_length), device=device, dtype=torch.bool)
+
+ # uniform distribution to sample from, make sure that offset samples are < sequence_length
+ uniform_dist = torch.ones(
+ (batch_size, sequence_length - (mask_length - 1)), device=device
+ )
+
+ # get random indices to mask
+ mask_indices = torch.multinomial(uniform_dist, num_masked_spans)
+
+ # expand masked indices to masked spans
+ mask_indices = (
+ mask_indices.unsqueeze(dim=-1)
+ .expand((batch_size, num_masked_spans, mask_length))
+ .reshape(batch_size, num_masked_spans * mask_length)
+ )
+ offsets = (
+ torch.arange(mask_length, device=device)[None, None, :]
+ .expand((batch_size, num_masked_spans, mask_length))
+ .reshape(batch_size, num_masked_spans * mask_length)
+ )
+ mask_idxs = mask_indices + offsets
+
+ # scatter indices to mask
+ mask = mask.scatter(1, mask_idxs, True)
+
+ return mask
+
+
+def hubert_soft(
+ path: str,
+) -> HubertSoft:
+ r"""HuBERT-Soft from `"A Comparison of Discrete and Soft Speech Units for Improved Voice Conversion"`.
+ Args:
+ path (str): path of a pretrained model
+ """
+ hubert = HubertSoft()
+ checkpoint = torch.load(path)
+ consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+ hubert.load_state_dict(checkpoint)
+ hubert.eval()
+ return hubert
diff --git a/inference/__init__.py b/inference/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/inference/infer_tool.py b/inference/infer_tool.py
new file mode 100644
index 0000000000000000000000000000000000000000..3ecde5c784e1644a5c206557418f243353ee759e
--- /dev/null
+++ b/inference/infer_tool.py
@@ -0,0 +1,267 @@
+import hashlib
+import io
+import json
+import logging
+import os
+import time
+from pathlib import Path
+from inference import slicer
+
+import librosa
+import numpy as np
+# import onnxruntime
+import parselmouth
+import soundfile
+import torch
+import torchaudio
+
+import cluster
+from hubert import hubert_model
+import utils
+from models import SynthesizerTrn
+
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+
+
+def read_temp(file_name):
+ if not os.path.exists(file_name):
+ with open(file_name, "w") as f:
+ f.write(json.dumps({"info": "temp_dict"}))
+ return {}
+ else:
+ try:
+ with open(file_name, "r") as f:
+ data = f.read()
+ data_dict = json.loads(data)
+ if os.path.getsize(file_name) > 50 * 1024 * 1024:
+ f_name = file_name.replace("\\", "/").split("/")[-1]
+ print(f"clean {f_name}")
+ for wav_hash in list(data_dict.keys()):
+ if int(time.time()) - int(data_dict[wav_hash]["time"]) > 14 * 24 * 3600:
+ del data_dict[wav_hash]
+ except Exception as e:
+ print(e)
+ print(f"{file_name} error,auto rebuild file")
+ data_dict = {"info": "temp_dict"}
+ return data_dict
+
+
+def write_temp(file_name, data):
+ with open(file_name, "w") as f:
+ f.write(json.dumps(data))
+
+
+def timeit(func):
+ def run(*args, **kwargs):
+ t = time.time()
+ res = func(*args, **kwargs)
+ print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
+ return res
+
+ return run
+
+
+def format_wav(audio_path):
+ if Path(audio_path).suffix == '.wav':
+ return
+ raw_audio, raw_sample_rate = librosa.load(audio_path, mono=True, sr=None)
+ soundfile.write(Path(audio_path).with_suffix(".wav"), raw_audio, raw_sample_rate)
+
+
+def get_end_file(dir_path, end):
+ file_lists = []
+ for root, dirs, files in os.walk(dir_path):
+ files = [f for f in files if f[0] != '.']
+ dirs[:] = [d for d in dirs if d[0] != '.']
+ for f_file in files:
+ if f_file.endswith(end):
+ file_lists.append(os.path.join(root, f_file).replace("\\", "/"))
+ return file_lists
+
+
+def get_md5(content):
+ return hashlib.new("md5", content).hexdigest()
+
+
+def fill_a_to_b(a, b):
+ if len(a) < len(b):
+ for _ in range(0, len(b) - len(a)):
+ a.append(a[0])
+
+
+def mkdir(paths: list):
+ for path in paths:
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+
+class Svc(object):
+ def __init__(self):
+
+ self.wav_format = "flac"
+ self.auto_predict_f0 = False
+ self.cluster_infer_ratio = 0.5
+ self.noice_scale = 0.6
+ self.pad_seconds = 0.5
+
+ self.net_g_path = None
+
+ self.dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.net_g_ms = None
+ self.hps_ms = None
+ self.target_sample = None
+ self.hop_size = None
+ self.spk2id = None
+
+ def set_device(self, device):
+ self.dev = torch.device(device)
+ # self.hubert_model = utils.get_hubert_model().to(self.dev)
+ # if self.net_g_ms != None:
+ # self.net_g_ms.to(device)
+
+ def load_checkpoint(self, path):
+ self.hps_ms = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
+ self.target_sample = self.hps_ms.data.sampling_rate
+ self.hop_size = self.hps_ms.data.hop_length
+ self.spk2id = self.hps_ms.spk
+
+ self.net_g_ms = SynthesizerTrn(
+ self.hps_ms.data.filter_length // 2 + 1,
+ self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
+ **self.hps_ms.model
+ )
+ _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.net_g_ms, None)
+ _ = self.net_g_ms.eval().to(self.dev)
+
+ # 加载hubert
+ self.hubert_model = utils.get_hubert_model().to(self.dev)
+ # self.load_model()
+ cluster_model_path = f"checkpoints/{path}/kmeans_10000.pt"
+ if os.path.exists(cluster_model_path):
+ self.cluster_model = cluster.get_cluster_model(cluster_model_path)
+ else:
+ self.cluster_model = None
+
+ def load_model(self):
+ # 获取模型配置
+ self.net_g_ms = SynthesizerTrn(
+ self.hps_ms.data.filter_length // 2 + 1,
+ self.hps_ms.train.segment_size // self.hps_ms.data.hop_length,
+ **self.hps_ms.model)
+ _ = utils.load_checkpoint(self.net_g_path, self.net_g_ms, None)
+ if "half" in self.net_g_path and torch.cuda.is_available():
+ _ = self.net_g_ms.half().eval().to(self.dev)
+ else:
+ _ = self.net_g_ms.eval().to(self.dev)
+
+ def get_unit_f0(self, in_path, tran, cluster_infer_ratio, speaker):
+
+ wav, sr = librosa.load(in_path, sr=self.target_sample)
+
+ f0 = utils.compute_f0_parselmouth(wav, sampling_rate=self.target_sample, hop_length=self.hop_size)
+ f0, uv = utils.interpolate_f0(f0)
+ f0 = torch.FloatTensor(f0)
+ uv = torch.FloatTensor(uv)
+ f0 = f0 * 2 ** (tran / 12)
+ f0 = f0.unsqueeze(0).to(self.dev)
+ uv = uv.unsqueeze(0).to(self.dev)
+
+ wav16k = librosa.resample(wav, orig_sr=self.target_sample, target_sr=16000)
+ wav16k = torch.from_numpy(wav16k).to(self.dev)
+ c = utils.get_hubert_content(self.hubert_model, wav_16k_tensor=wav16k)
+ c = utils.repeat_expand_2d(c.squeeze(0), f0.shape[1])
+
+ if self.cluster_model is not None and cluster_infer_ratio != 0:
+ cluster_c = cluster.get_cluster_center_result(self.cluster_model, c.cpu().numpy().T, speaker).T
+ cluster_c = torch.FloatTensor(cluster_c).to(self.dev)
+ c = cluster_infer_ratio * cluster_c + (1 - cluster_infer_ratio) * c
+
+ c = c.unsqueeze(0)
+ return c, f0, uv
+
+ def infer(self, speaker, tran, raw_path,
+ cluster_infer_ratio=0.0,
+ auto_predict_f0=False,
+ noice_scale=0.4):
+ speaker_id = self.spk2id[speaker]
+ sid = torch.LongTensor([int(speaker_id)]).to(self.dev).unsqueeze(0)
+ c, f0, uv = self.get_unit_f0(raw_path, tran, cluster_infer_ratio, speaker)
+
+ with torch.no_grad():
+ start = time.time()
+ audio = self.net_g_ms.infer(c, f0=f0, g=sid, uv=uv, predict_f0=auto_predict_f0, noice_scale=noice_scale)[
+ 0, 0].data.float()
+ use_time = time.time() - start
+ print("VITS use time:{}".format(use_time))
+ return audio, audio.shape[-1]
+
+ def slice_inference(self, raw_audio_path, spk, tran, slice_db, cluster_infer_ratio, auto_predict_f0, noice_scale,
+ pad_seconds=0.5):
+ wav_path = raw_audio_path
+ chunks = slicer.cut(wav_path, db_thresh=slice_db)
+ audio_data, audio_sr = slicer.chunks2audio(wav_path, chunks)
+
+ audio = []
+ for (slice_tag, data) in audio_data:
+ print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
+ # padd
+ pad_len = int(audio_sr * pad_seconds)
+ data = np.concatenate([np.zeros([pad_len]), data, np.zeros([pad_len])])
+ length = int(np.ceil(len(data) / audio_sr * self.target_sample))
+ raw_path = io.BytesIO()
+ soundfile.write(raw_path, data, audio_sr, format="wav")
+ raw_path.seek(0)
+ if slice_tag:
+ print('jump empty segment')
+ _audio = np.zeros(length)
+ else:
+ out_audio, out_sr = self.infer(spk, tran, raw_path,
+ cluster_infer_ratio=cluster_infer_ratio,
+ auto_predict_f0=auto_predict_f0,
+ noice_scale=noice_scale
+ )
+ _audio = out_audio.cpu().numpy()
+
+ pad_len = int(self.target_sample * pad_seconds)
+ _audio = _audio[pad_len:-pad_len]
+ audio.extend(list(_audio))
+ return np.array(audio)
+
+ def inference(self, srcaudio, chara, tran, slice_db, ns):
+ self.noice_scale = ns
+ sampling_rate, audio = srcaudio
+ audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+ if len(audio.shape) > 1:
+ audio = librosa.to_mono(audio.transpose(1, 0))
+ if sampling_rate != 16000:
+ audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
+ soundfile.write("tmpwav.wav", audio, 16000, format="wav")
+
+ chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db)
+ audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks)
+ audio = []
+ for (slice_tag, data) in audio_data:
+ print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
+ # padd
+ pad_len = int(audio_sr * self.pad_seconds)
+ data = np.concatenate([np.zeros([pad_len]), data, np.zeros([pad_len])])
+ length = int(np.ceil(len(data) / audio_sr * self.target_sample))
+ raw_path = io.BytesIO()
+ soundfile.write(raw_path, data, audio_sr, format="wav")
+ raw_path.seek(0)
+ if slice_tag:
+ print('jump empty segment')
+ _audio = np.zeros(length)
+ else:
+ out_audio, out_sr = self.infer(chara, tran, raw_path,
+ cluster_infer_ratio=self.cluster_infer_ratio,
+ auto_predict_f0=self.auto_predict_f0,
+ noice_scale=self.noice_scale
+ )
+ _audio = out_audio.cpu().numpy()
+
+ pad_len = int(self.target_sample * self.pad_seconds)
+ _audio = _audio[pad_len:-pad_len]
+ audio.extend(list(_audio))
+ audio = (np.array(audio) * 32768).astype('int16')
+ return self.hps_ms.data.sampling_rate, audio
diff --git a/inference/infer_tool_grad.py b/inference/infer_tool_grad.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a28486a4bcd1cfbbc5a0f581a382bd1269084ba
--- /dev/null
+++ b/inference/infer_tool_grad.py
@@ -0,0 +1,163 @@
+import hashlib
+import json
+import logging
+import os
+import time
+from pathlib import Path
+import io
+import librosa
+import maad
+import numpy as np
+from inference import slicer
+import parselmouth
+import soundfile
+import torch
+import torchaudio
+
+from hubert import hubert_model
+import utils
+from models import SynthesizerTrn
+
+logging.getLogger('numba').setLevel(logging.WARNING)
+logging.getLogger('matplotlib').setLevel(logging.WARNING)
+
+
+def resize2d_f0(x, target_len):
+ source = np.array(x)
+ source[source < 0.001] = np.nan
+ target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
+ source)
+ res = np.nan_to_num(target)
+ return res
+
+
+def get_f0(x, p_len, f0_up_key=0):
+ time_step = 160 / 16000 * 1000
+ f0_min = 50
+ f0_max = 1100
+ f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+ f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+ f0 = parselmouth.Sound(x, 16000).to_pitch_ac(
+ time_step=time_step / 1000, voicing_threshold=0.6,
+ pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
+
+ pad_size = (p_len - len(f0) + 1) // 2
+ if (pad_size > 0 or p_len - len(f0) - pad_size > 0):
+ f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode='constant')
+
+ f0 *= pow(2, f0_up_key / 12)
+ f0_mel = 1127 * np.log(1 + f0 / 700)
+ f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (f0_mel_max - f0_mel_min) + 1
+ f0_mel[f0_mel <= 1] = 1
+ f0_mel[f0_mel > 255] = 255
+ f0_coarse = np.rint(f0_mel).astype(np.int)
+ return f0_coarse, f0
+
+
+def clean_pitch(input_pitch):
+ num_nan = np.sum(input_pitch == 1)
+ if num_nan / len(input_pitch) > 0.9:
+ input_pitch[input_pitch != 1] = 1
+ return input_pitch
+
+
+def plt_pitch(input_pitch):
+ input_pitch = input_pitch.astype(float)
+ input_pitch[input_pitch == 1] = np.nan
+ return input_pitch
+
+
+def f0_to_pitch(ff):
+ f0_pitch = 69 + 12 * np.log2(ff / 440)
+ return f0_pitch
+
+
+def fill_a_to_b(a, b):
+ if len(a) < len(b):
+ for _ in range(0, len(b) - len(a)):
+ a.append(a[0])
+
+
+def mkdir(paths: list):
+ for path in paths:
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+
+class VitsSvc(object):
+ def __init__(self):
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.SVCVITS = None
+ self.hps = None
+ self.speakers = None
+ self.hubert_soft = utils.get_hubert_model()
+
+ def set_device(self, device):
+ self.device = torch.device(device)
+ self.hubert_soft.to(self.device)
+ if self.SVCVITS != None:
+ self.SVCVITS.to(self.device)
+
+ def loadCheckpoint(self, path):
+ self.hps = utils.get_hparams_from_file(f"checkpoints/{path}/config.json")
+ self.SVCVITS = SynthesizerTrn(
+ self.hps.data.filter_length // 2 + 1,
+ self.hps.train.segment_size // self.hps.data.hop_length,
+ **self.hps.model)
+ _ = utils.load_checkpoint(f"checkpoints/{path}/model.pth", self.SVCVITS, None)
+ _ = self.SVCVITS.eval().to(self.device)
+ self.speakers = self.hps.spk
+
+ def get_units(self, source, sr):
+ source = source.unsqueeze(0).to(self.device)
+ with torch.inference_mode():
+ units = self.hubert_soft.units(source)
+ return units
+
+ def get_unit_pitch(self, in_path, tran):
+ source, sr = torchaudio.load(in_path)
+ source = torchaudio.functional.resample(source, sr, 16000)
+ if len(source.shape) == 2 and source.shape[1] >= 2:
+ source = torch.mean(source, dim=0).unsqueeze(0)
+ soft = self.get_units(source, sr).squeeze(0).cpu().numpy()
+ f0_coarse, f0 = get_f0(source.cpu().numpy()[0], soft.shape[0] * 2, tran)
+ return soft, f0
+
+ def infer(self, speaker_id, tran, raw_path):
+ speaker_id = self.speakers[speaker_id]
+ sid = torch.LongTensor([int(speaker_id)]).to(self.device).unsqueeze(0)
+ soft, pitch = self.get_unit_pitch(raw_path, tran)
+ f0 = torch.FloatTensor(clean_pitch(pitch)).unsqueeze(0).to(self.device)
+ stn_tst = torch.FloatTensor(soft)
+ with torch.no_grad():
+ x_tst = stn_tst.unsqueeze(0).to(self.device)
+ x_tst = torch.repeat_interleave(x_tst, repeats=2, dim=1).transpose(1, 2)
+ audio = self.SVCVITS.infer(x_tst, f0=f0, g=sid)[0, 0].data.float()
+ return audio, audio.shape[-1]
+
+ def inference(self, srcaudio, chara, tran, slice_db):
+ sampling_rate, audio = srcaudio
+ audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+ if len(audio.shape) > 1:
+ audio = librosa.to_mono(audio.transpose(1, 0))
+ if sampling_rate != 16000:
+ audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=16000)
+ soundfile.write("tmpwav.wav", audio, 16000, format="wav")
+
+ chunks = slicer.cut("tmpwav.wav", db_thresh=slice_db)
+ audio_data, audio_sr = slicer.chunks2audio("tmpwav.wav", chunks)
+ audio = []
+ for (slice_tag, data) in audio_data:
+ length = int(np.ceil(len(data) / audio_sr * self.hps.data.sampling_rate))
+ raw_path = io.BytesIO()
+ soundfile.write(raw_path, data, audio_sr, format="wav")
+ raw_path.seek(0)
+ if slice_tag:
+ _audio = np.zeros(length)
+ else:
+ out_audio, out_sr = self.infer(chara, tran, raw_path)
+ _audio = out_audio.cpu().numpy()
+ audio.extend(list(_audio))
+ audio = (np.array(audio) * 32768.0).astype('int16')
+ return (self.hps.data.sampling_rate, audio)
diff --git a/inference/slicer.py b/inference/slicer.py
new file mode 100644
index 0000000000000000000000000000000000000000..76f7055bc97582991173cc77af957b966e813b77
--- /dev/null
+++ b/inference/slicer.py
@@ -0,0 +1,142 @@
+import librosa
+import torch
+import torchaudio
+
+
+class Slicer:
+ def __init__(self,
+ sr: int,
+ threshold: float = -40,
+ min_length: int = 5000,
+ min_interval: int = 300,
+ hop_size: int = 20,
+ max_sil_kept: int = 5000):
+ if not min_length >= min_interval >= hop_size:
+ raise ValueError('The following condition must be satisfied: min_length >= min_interval >= hop_size')
+ if not max_sil_kept >= hop_size:
+ raise ValueError('The following condition must be satisfied: max_sil_kept >= hop_size')
+ min_interval = sr * min_interval / 1000
+ self.threshold = 10 ** (threshold / 20.)
+ self.hop_size = round(sr * hop_size / 1000)
+ self.win_size = min(round(min_interval), 4 * self.hop_size)
+ self.min_length = round(sr * min_length / 1000 / self.hop_size)
+ self.min_interval = round(min_interval / self.hop_size)
+ self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size)
+
+ def _apply_slice(self, waveform, begin, end):
+ if len(waveform.shape) > 1:
+ return waveform[:, begin * self.hop_size: min(waveform.shape[1], end * self.hop_size)]
+ else:
+ return waveform[begin * self.hop_size: min(waveform.shape[0], end * self.hop_size)]
+
+ # @timeit
+ def slice(self, waveform):
+ if len(waveform.shape) > 1:
+ samples = librosa.to_mono(waveform)
+ else:
+ samples = waveform
+ if samples.shape[0] <= self.min_length:
+ return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
+ rms_list = librosa.feature.rms(y=samples, frame_length=self.win_size, hop_length=self.hop_size).squeeze(0)
+ sil_tags = []
+ silence_start = None
+ clip_start = 0
+ for i, rms in enumerate(rms_list):
+ # Keep looping while frame is silent.
+ if rms < self.threshold:
+ # Record start of silent frames.
+ if silence_start is None:
+ silence_start = i
+ continue
+ # Keep looping while frame is not silent and silence start has not been recorded.
+ if silence_start is None:
+ continue
+ # Clear recorded silence start if interval is not enough or clip is too short
+ is_leading_silence = silence_start == 0 and i > self.max_sil_kept
+ need_slice_middle = i - silence_start >= self.min_interval and i - clip_start >= self.min_length
+ if not is_leading_silence and not need_slice_middle:
+ silence_start = None
+ continue
+ # Need slicing. Record the range of silent frames to be removed.
+ if i - silence_start <= self.max_sil_kept:
+ pos = rms_list[silence_start: i + 1].argmin() + silence_start
+ if silence_start == 0:
+ sil_tags.append((0, pos))
+ else:
+ sil_tags.append((pos, pos))
+ clip_start = pos
+ elif i - silence_start <= self.max_sil_kept * 2:
+ pos = rms_list[i - self.max_sil_kept: silence_start + self.max_sil_kept + 1].argmin()
+ pos += i - self.max_sil_kept
+ pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
+ pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
+ if silence_start == 0:
+ sil_tags.append((0, pos_r))
+ clip_start = pos_r
+ else:
+ sil_tags.append((min(pos_l, pos), max(pos_r, pos)))
+ clip_start = max(pos_r, pos)
+ else:
+ pos_l = rms_list[silence_start: silence_start + self.max_sil_kept + 1].argmin() + silence_start
+ pos_r = rms_list[i - self.max_sil_kept: i + 1].argmin() + i - self.max_sil_kept
+ if silence_start == 0:
+ sil_tags.append((0, pos_r))
+ else:
+ sil_tags.append((pos_l, pos_r))
+ clip_start = pos_r
+ silence_start = None
+ # Deal with trailing silence.
+ total_frames = rms_list.shape[0]
+ if silence_start is not None and total_frames - silence_start >= self.min_interval:
+ silence_end = min(total_frames, silence_start + self.max_sil_kept)
+ pos = rms_list[silence_start: silence_end + 1].argmin() + silence_start
+ sil_tags.append((pos, total_frames + 1))
+ # Apply and return slices.
+ if len(sil_tags) == 0:
+ return {"0": {"slice": False, "split_time": f"0,{len(waveform)}"}}
+ else:
+ chunks = []
+ # 第一段静音并非从头开始，补上有声片段
+ if sil_tags[0][0]:
+ chunks.append(
+ {"slice": False, "split_time": f"0,{min(waveform.shape[0], sil_tags[0][0] * self.hop_size)}"})
+ for i in range(0, len(sil_tags)):
+ # 标识有声片段（跳过第一段）
+ if i:
+ chunks.append({"slice": False,
+ "split_time": f"{sil_tags[i - 1][1] * self.hop_size},{min(waveform.shape[0], sil_tags[i][0] * self.hop_size)}"})
+ # 标识所有静音片段
+ chunks.append({"slice": True,
+ "split_time": f"{sil_tags[i][0] * self.hop_size},{min(waveform.shape[0], sil_tags[i][1] * self.hop_size)}"})
+ # 最后一段静音并非结尾，补上结尾片段
+ if sil_tags[-1][1] * self.hop_size < len(waveform):
+ chunks.append({"slice": False, "split_time": f"{sil_tags[-1][1] * self.hop_size},{len(waveform)}"})
+ chunk_dict = {}
+ for i in range(len(chunks)):
+ chunk_dict[str(i)] = chunks[i]
+ return chunk_dict
+
+
+def cut(audio_path, db_thresh=-30, min_len=5000):
+ audio, sr = librosa.load(audio_path, sr=None)
+ slicer = Slicer(
+ sr=sr,
+ threshold=db_thresh,
+ min_length=min_len
+ )
+ chunks = slicer.slice(audio)
+ return chunks
+
+
+def chunks2audio(audio_path, chunks):
+ chunks = dict(chunks)
+ audio, sr = torchaudio.load(audio_path)
+ if len(audio.shape) == 2 and audio.shape[1] >= 2:
+ audio = torch.mean(audio, dim=0).unsqueeze(0)
+ audio = audio.cpu().numpy()[0]
+ result = []
+ for k, v in chunks.items():
+ tag = v["split_time"].split(",")
+ if tag[0] != tag[1]:
+ result.append((v["slice"], audio[int(tag[0]):int(tag[1])]))
+ return result, sr
diff --git a/log/.gitignore b/log/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/main.py b/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..422a3ffdcaa2c07fa0c283eabfacfd38bcea8fc4
--- /dev/null
+++ b/main.py
@@ -0,0 +1,159 @@
+from inference.infer_tool import Svc
+from vextract.vocal_extract import VEX
+import gradio as gr
+import os
+
+
+# os.environ['CUDA_VISIBLE_DEVICES'] = '1,2'
+
+
+class VitsGradio:
+ def __init__(self):
+ self.so = Svc()
+ self.v = VEX()
+ self.lspk = []
+ self.modelPaths = []
+ for root, dirs, files in os.walk("checkpoints"):
+ for dir in dirs:
+ self.modelPaths.append(dir)
+ with gr.Blocks(title="Sovits歌声合成工具") as self.Vits:
+ gr.Markdown(
+ """
+ # 歌声合成工具
+ - 请依次选择语音模型、设备以及运行模式，然后点击"载入模型"
+ - 输入音频需要是干净的人声
+ """
+ )
+ with gr.Tab("人声提取"):
+ with gr.Row():
+ with gr.Column():
+ sample_audio = gr.Audio(label="输入音频")
+ extractAudioBtn = gr.Button("提取人声")
+ with gr.Row():
+ with gr.Column():
+ self.sample_vocal_output = gr.Audio(label="输出音频")
+ self.sample_accompaniment_output = gr.Audio()
+ extractAudioBtn.click(self.v.separate, inputs=[sample_audio],
+ outputs=[self.sample_vocal_output, self.sample_accompaniment_output],
+ show_progress=True, api_name="extract")
+ with gr.Tab("歌声合成"):
+ with gr.Row(visible=False) as self.VoiceConversion:
+ with gr.Column():
+ with gr.Row():
+ with gr.Column():
+ self.srcaudio = gr.Audio(label="输入音频")
+ self.btnVC = gr.Button("说话人转换")
+ with gr.Column():
+ with gr.Row():
+ with gr.Column():
+ self.dsid0 = gr.Dropdown(label="目标角色", choices=self.lspk)
+ self.tran = gr.Slider(label="升降调", maximum=60, minimum=-60, step=1, value=0)
+ self.th = gr.Slider(label="切片阈值", maximum=32767, minimum=-32768, step=0.1,
+ value=-40)
+ self.ns = gr.Slider(label="噪音级别", maximum=1.0, minimum=0.0, step=0.1,
+ value=0.4)
+ with gr.Row():
+ self.VCOutputs = gr.Audio()
+ self.btnVC.click(self.so.inference, inputs=[self.srcaudio, self.dsid0, self.tran, self.th, self.ns],
+ outputs=[self.VCOutputs], show_progress=True, api_name="run")
+
+ with gr.Row(visible=False) as self.VoiceBatchConversion:
+ with gr.Column():
+ with gr.Row():
+ with gr.Column():
+ self.srcaudio = gr.Files(label="上传多个音频文件", file_types=['.wav'],
+ interactive=True)
+ self.btnVC = gr.Button("说话人转换")
+ with gr.Column():
+ with gr.Row():
+ with gr.Column():
+ self.dsid1 = gr.Dropdown(label="目标角色", choices=self.lspk)
+ self.tran = gr.Slider(label="升降调", maximum=60, minimum=-60, step=1, value=0)
+ self.th = gr.Slider(label="切片阈值", maximum=32767, minimum=-32768, step=0.1,
+ value=-40)
+ self.ns = gr.Slider(label="噪音级别", maximum=1.0, minimum=0.0, step=0.1,
+ value=0.4)
+ with gr.Row():
+ self.VCOutputs = gr.File(label="Output Zip File", interactive=False)
+ self.btnVC.click(self.batch_inference, inputs=[self.srcaudio, self.dsid1, self.tran, self.th, self.ns],
+ outputs=[self.VCOutputs], show_progress=True, api_name="batch")
+
+ with gr.Row():
+ with gr.Column():
+ modelstrs = gr.Dropdown(label="模型", choices=self.modelPaths, value=self.modelPaths[0],
+ type="value")
+ devicestrs = gr.Dropdown(label="设备", choices=["cpu", "cuda"], value="cuda", type="value")
+ isbatchmod = gr.Radio(label="运行模式", choices=["single", "batch"], value="single",
+ info="single: 单个文件处理. batch:批量处理支持上传多个文件")
+ btnMod = gr.Button("载入模型")
+ btnMod.click(self.loadModel, inputs=[modelstrs, devicestrs, isbatchmod],
+ outputs=[self.dsid0, self.dsid1, self.VoiceConversion, self.VoiceBatchConversion],
+ show_progress=True, api_name="switch")
+
+ def batch_inference(self, files, chara, tran, slice_db, ns, progress=gr.Progress()):
+ from zipfile import ZipFile
+ from scipy.io import wavfile
+ import uuid
+
+ temp_directory = "temp"
+ if not os.path.exists(temp_directory):
+ os.mkdir(temp_directory)
+
+ progress(0.00, desc="初始化文件夹")
+ tmp_workdir_name = f"{temp_directory}/batch_{uuid.uuid4()}"
+ if not os.path.exists(tmp_workdir_name):
+ os.mkdir(tmp_workdir_name)
+
+ progress(0.10, desc="初始化文件夹")
+
+ output_files = []
+
+ for idx, file in enumerate(files):
+ filename = os.path.basename(file.name)
+ progress(0.10 + (0.70 / float(len(files))) * (idx + 1.00), desc=f"处理音频{(idx + 1)}/{len(files)}：{filename}")
+ print(f"{idx}, {file}, {filename}")
+ sampling_rate, audio = wavfile.read(file.name)
+ output_sampling_rate, output_audio = self.so.inference((sampling_rate, audio), chara=chara, tran=tran,
+ slice_db=slice_db, ns=ns)
+ new_filepath = f"{tmp_workdir_name}/{filename}"
+ wavfile.write(filename=new_filepath, rate=output_sampling_rate, data=output_audio)
+ output_files.append(new_filepath)
+
+ progress(0.70, desc="音频处理完毕")
+
+ zipfilename = f"{tmp_workdir_name}/output.zip"
+ with ZipFile(zipfilename, "w") as zip_obj:
+ for idx, filepath in enumerate(output_files):
+ zip_obj.write(filepath, os.path.basename(filepath))
+ progress(0.80, desc="压缩完毕")
+ # todo: remove data
+ progress(1.00, desc="清理空间")
+ return zipfilename
+
+ def loadModel(self, path, device, process_mode):
+ self.lspk = []
+ print(f"path: {path}, device: {device}")
+ self.so.set_device(device)
+ print(f"device set.")
+ self.so.load_checkpoint(path)
+ print(f"checkpoint loaded")
+ for spk, sid in self.so.hps_ms.spk.items():
+ self.lspk.append(spk)
+ print(f"LSPK: {self.lspk}")
+ if process_mode == "single":
+ VChange = gr.update(visible=True)
+ VBChange = gr.update(visible=False)
+ else:
+ VChange = gr.update(visible=False)
+ VBChange = gr.update(visible=True)
+ SD0Change = gr.update(choices=self.lspk, value=self.lspk[0])
+ SD1Change = gr.update(choices=self.lspk, value=self.lspk[0])
+ print("allset update display")
+ return [SD0Change, SD1Change, VChange, VBChange]
+
+
+if __name__ == "__main__":
+ grVits = VitsGradio()
+ grVits.Vits\
+ .queue(concurrency_count=20, status_update_rate=5.0)\
+ .launch(server_port=7870, share=True, show_api=True)
diff --git a/maint.py b/maint.py
new file mode 100644
index 0000000000000000000000000000000000000000..e97a31a5f97882df89fc01b27131ca10fb9d4862
--- /dev/null
+++ b/maint.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import argparse
+import logging
+import logging.handlers
+import os
+import tornado.ioloop
+import tornado.web
+import config
+import api.main
+import api.svc
+import api.tool
+import api.vm
+
+logger = logging.getLogger(__name__)
+
+BASE_PATH = os.path.dirname(os.path.realpath(__file__))
+WEB_ROOT = os.path.join(BASE_PATH, 'frontend', 'dist')
+LOG_FILE_NAME = os.path.join(BASE_PATH, 'log', 'sovits-webui.log')
+
+routes = [
+ (r'/api/svc/model', api.svc.ModelListHandler),
+ (r'/api/svc/switch', api.svc.SwitchHandler),
+ (r'/api/svc/run', api.svc.SingleInferenceHandler),
+ (r'/api/svc/batch', api.svc.BatchInferenceHandler),
+
+ (r'/api/vm/run', api.vm.VocalRemoverHandler),
+
+ (r'/api/tool/norm', api.tool.AudioNormalizerHandler),
+
+ (r'/(.*)', api.main.MainHandler, {'path': WEB_ROOT, 'default_filename': 'index.html'})
+]
+
+
+def main():
+ args = parse_args()
+
+ init_logging(args.debug)
+ config.init()
+ api.svc.init()
+
+ run_server(args.host, args.port, args.debug)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="SOVITS WEBUI")
+ parser.add_argument('--host', help='Server host, default to 127.0.0.1', default='127.0.0.1')
+ parser.add_argument('--port', help='Server port, default to 7870', type=int, default=7870)
+ parser.add_argument('--debug', help='debug mode', action='store_true')
+ return parser.parse_args()
+
+
+def init_logging(debug):
+ stream_handler = logging.StreamHandler()
+ file_handler = logging.handlers.TimedRotatingFileHandler(
+ LOG_FILE_NAME, encoding='utf-8', when='midnight', backupCount=7, delay=True
+ )
+ # noinspection PyArgumentList
+ logging.basicConfig(
+ format='{asctime} {levelname} [{name}]: {message}',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ style='{',
+ level=logging.INFO if not debug else logging.DEBUG,
+ handlers=[stream_handler, file_handler]
+ )
+
+ # 屏蔽访问日志
+ logging.getLogger('tornado.access').setLevel(logging.WARNING)
+
+
+def run_server(host, port, debug):
+ app = tornado.web.Application(
+ routes,
+ websocket_ping_interval=10,
+ debug=debug,
+ autoreload=True
+ )
+ cfg = config.get_config()
+ try:
+ app.listen(
+ port=port,
+ address=host,
+ xheaders=cfg.tornado_xheaders
+ )
+ except OSError:
+ logger.warning('Address is used %s:%d', host, port)
+ return
+ # finally:
+ # url = f'http://{host}/' if port == 80 else f'http://{host}:{port}/'
+ # # 防止更新版本后浏览器加载缓存
+ # url += '?_v=' + update.VERSION
+ # webbrowser.open(url)
+ logger.info('Server started: %s:%d', host, port)
+ try:
+ tornado.ioloop.IOLoop.current().start()
+ except KeyboardInterrupt:
+ tornado.ioloop.IOLoop.current().instance().stop()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/models.py b/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d6b3072fe037206ac1afc0f986272e881fc56b1
--- /dev/null
+++ b/models.py
@@ -0,0 +1,420 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import modules.attentions as attentions
+import modules.commons as commons
+import modules.modules as modules
+
+from torch.nn import Conv1d, Conv2d
+from torch.nn.utils import weight_norm, spectral_norm
+
+import utils
+from modules.commons import get_padding
+from vdecoder.hifigan.models import Generator
+from utils import f0_to_coarse
+
+
+class ResidualCouplingBlock(nn.Module):
+ def __init__(self,
+ channels,
+ hidden_channels,
+ kernel_size,
+ dilation_rate,
+ n_layers,
+ n_flows=4,
+ gin_channels=0):
+ super().__init__()
+ self.channels = channels
+ self.hidden_channels = hidden_channels
+ self.kernel_size = kernel_size
+ self.dilation_rate = dilation_rate
+ self.n_layers = n_layers
+ self.n_flows = n_flows
+ self.gin_channels = gin_channels
+
+ self.flows = nn.ModuleList()
+ for i in range(n_flows):
+ self.flows.append(
+ modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers,
+ gin_channels=gin_channels, mean_only=True))
+ self.flows.append(modules.Flip())
+
+ def forward(self, x, x_mask, g=None, reverse=False):
+ if not reverse:
+ for flow in self.flows:
+ x, _ = flow(x, x_mask, g=g, reverse=reverse)
+ else:
+ for flow in reversed(self.flows):
+ x = flow(x, x_mask, g=g, reverse=reverse)
+ return x
+
+
+class Encoder(nn.Module):
+ def __init__(self,
+ in_channels,
+ out_channels,
+ hidden_channels,
+ kernel_size,
+ dilation_rate,
+ n_layers,
+ gin_channels=0):
+ super().__init__()
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+ self.hidden_channels = hidden_channels
+ self.kernel_size = kernel_size
+ self.dilation_rate = dilation_rate
+ self.n_layers = n_layers
+ self.gin_channels = gin_channels
+
+ self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
+ self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
+ self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+ def forward(self, x, x_lengths, g=None):
+ # print(x.shape,x_lengths.shape)
+ x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
+ x = self.pre(x) * x_mask
+ x = self.enc(x, x_mask, g=g)
+ stats = self.proj(x) * x_mask
+ m, logs = torch.split(stats, self.out_channels, dim=1)
+ z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
+ return z, m, logs, x_mask
+
+
+class TextEncoder(nn.Module):
+ def __init__(self,
+ out_channels,
+ hidden_channels,
+ kernel_size,
+ n_layers,
+ gin_channels=0,
+ filter_channels=None,
+ n_heads=None,
+ p_dropout=None):
+ super().__init__()
+ self.out_channels = out_channels
+ self.hidden_channels = hidden_channels
+ self.kernel_size = kernel_size
+ self.n_layers = n_layers
+ self.gin_channels = gin_channels
+ self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+ self.f0_emb = nn.Embedding(256, hidden_channels)
+
+ self.enc_ = attentions.Encoder(
+ hidden_channels,
+ filter_channels,
+ n_heads,
+ n_layers,
+ kernel_size,
+ p_dropout)
+
+ def forward(self, x, x_mask, f0=None, noice_scale=1):
+ x = x + self.f0_emb(f0).transpose(1, 2)
+ x = self.enc_(x * x_mask, x_mask)
+ stats = self.proj(x) * x_mask
+ m, logs = torch.split(stats, self.out_channels, dim=1)
+ z = (m + torch.randn_like(m) * torch.exp(logs) * noice_scale) * x_mask
+
+ return z, m, logs, x_mask
+
+
+class DiscriminatorP(torch.nn.Module):
+ def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+ super(DiscriminatorP, self).__init__()
+ self.period = period
+ self.use_spectral_norm = use_spectral_norm
+ norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+ self.convs = nn.ModuleList([
+ norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+ norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+ norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+ norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
+ norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
+ ])
+ self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+ def forward(self, x):
+ fmap = []
+
+ # 1d to 2d
+ b, c, t = x.shape
+ if t % self.period != 0: # pad first
+ n_pad = self.period - (t % self.period)
+ x = F.pad(x, (0, n_pad), "reflect")
+ t = t + n_pad
+ x = x.view(b, c, t // self.period, self.period)
+
+ for l in self.convs:
+ x = l(x)
+ x = F.leaky_relu(x, modules.LRELU_SLOPE)
+ fmap.append(x)
+ x = self.conv_post(x)
+ fmap.append(x)
+ x = torch.flatten(x, 1, -1)
+
+ return x, fmap
+
+
+class DiscriminatorS(torch.nn.Module):
+ def __init__(self, use_spectral_norm=False):
+ super(DiscriminatorS, self).__init__()
+ norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+ self.convs = nn.ModuleList([
+ norm_f(Conv1d(1, 16, 15, 1, padding=7)),
+ norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
+ norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
+ norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
+ norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
+ norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
+ ])
+ self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
+
+ def forward(self, x):
+ fmap = []
+
+ for l in self.convs:
+ x = l(x)
+ x = F.leaky_relu(x, modules.LRELU_SLOPE)
+ fmap.append(x)
+ x = self.conv_post(x)
+ fmap.append(x)
+ x = torch.flatten(x, 1, -1)
+
+ return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+ def __init__(self, use_spectral_norm=False):
+ super(MultiPeriodDiscriminator, self).__init__()
+ periods = [2, 3, 5, 7, 11]
+
+ discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
+ discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
+ self.discriminators = nn.ModuleList(discs)
+
+ def forward(self, y, y_hat):
+ y_d_rs = []
+ y_d_gs = []
+ fmap_rs = []
+ fmap_gs = []
+ for i, d in enumerate(self.discriminators):
+ y_d_r, fmap_r = d(y)
+ y_d_g, fmap_g = d(y_hat)
+ y_d_rs.append(y_d_r)
+ y_d_gs.append(y_d_g)
+ fmap_rs.append(fmap_r)
+ fmap_gs.append(fmap_g)
+
+ return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+class SpeakerEncoder(torch.nn.Module):
+ def __init__(self, mel_n_channels=80, model_num_layers=3, model_hidden_size=256, model_embedding_size=256):
+ super(SpeakerEncoder, self).__init__()
+ self.lstm = nn.LSTM(mel_n_channels, model_hidden_size, model_num_layers, batch_first=True)
+ self.linear = nn.Linear(model_hidden_size, model_embedding_size)
+ self.relu = nn.ReLU()
+
+ def forward(self, mels):
+ self.lstm.flatten_parameters()
+ _, (hidden, _) = self.lstm(mels)
+ embeds_raw = self.relu(self.linear(hidden[-1]))
+ return embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True)
+
+ def compute_partial_slices(self, total_frames, partial_frames, partial_hop):
+ mel_slices = []
+ for i in range(0, total_frames - partial_frames, partial_hop):
+ mel_range = torch.arange(i, i + partial_frames)
+ mel_slices.append(mel_range)
+
+ return mel_slices
+
+ def embed_utterance(self, mel, partial_frames=128, partial_hop=64):
+ mel_len = mel.size(1)
+ last_mel = mel[:, -partial_frames:]
+
+ if mel_len > partial_frames:
+ mel_slices = self.compute_partial_slices(mel_len, partial_frames, partial_hop)
+ mels = list(mel[:, s] for s in mel_slices)
+ mels.append(last_mel)
+ mels = torch.stack(tuple(mels), 0).squeeze(1)
+
+ with torch.no_grad():
+ partial_embeds = self(mels)
+ embed = torch.mean(partial_embeds, axis=0).unsqueeze(0)
+ # embed = embed / torch.linalg.norm(embed, 2)
+ else:
+ with torch.no_grad():
+ embed = self(last_mel)
+
+ return embed
+
+
+class F0Decoder(nn.Module):
+ def __init__(self,
+ out_channels,
+ hidden_channels,
+ filter_channels,
+ n_heads,
+ n_layers,
+ kernel_size,
+ p_dropout,
+ spk_channels=0):
+ super().__init__()
+ self.out_channels = out_channels
+ self.hidden_channels = hidden_channels
+ self.filter_channels = filter_channels
+ self.n_heads = n_heads
+ self.n_layers = n_layers
+ self.kernel_size = kernel_size
+ self.p_dropout = p_dropout
+ self.spk_channels = spk_channels
+
+ self.prenet = nn.Conv1d(hidden_channels, hidden_channels, 3, padding=1)
+ self.decoder = attentions.FFT(
+ hidden_channels,
+ filter_channels,
+ n_heads,
+ n_layers,
+ kernel_size,
+ p_dropout)
+ self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
+ self.f0_prenet = nn.Conv1d(1, hidden_channels, 3, padding=1)
+ self.cond = nn.Conv1d(spk_channels, hidden_channels, 1)
+
+ def forward(self, x, norm_f0, x_mask, spk_emb=None):
+ x = torch.detach(x)
+ if (spk_emb is not None):
+ x = x + self.cond(spk_emb)
+ x += self.f0_prenet(norm_f0)
+ x = self.prenet(x) * x_mask
+ x = self.decoder(x * x_mask, x_mask)
+ x = self.proj(x) * x_mask
+ return x
+
+
+class SynthesizerTrn(nn.Module):
+ """
+ Synthesizer for Training
+ """
+
+ def __init__(self,
+ spec_channels,
+ segment_size,
+ inter_channels,
+ hidden_channels,
+ filter_channels,
+ n_heads,
+ n_layers,
+ kernel_size,
+ p_dropout,
+ resblock,
+ resblock_kernel_sizes,
+ resblock_dilation_sizes,
+ upsample_rates,
+ upsample_initial_channel,
+ upsample_kernel_sizes,
+ gin_channels,
+ ssl_dim,
+ n_speakers,
+ sampling_rate=44100,
+ **kwargs):
+ super().__init__()
+ self.spec_channels = spec_channels
+ self.inter_channels = inter_channels
+ self.hidden_channels = hidden_channels
+ self.filter_channels = filter_channels
+ self.n_heads = n_heads
+ self.n_layers = n_layers
+ self.kernel_size = kernel_size
+ self.p_dropout = p_dropout
+ self.resblock = resblock
+ self.resblock_kernel_sizes = resblock_kernel_sizes
+ self.resblock_dilation_sizes = resblock_dilation_sizes
+ self.upsample_rates = upsample_rates
+ self.upsample_initial_channel = upsample_initial_channel
+ self.upsample_kernel_sizes = upsample_kernel_sizes
+ self.segment_size = segment_size
+ self.gin_channels = gin_channels
+ self.ssl_dim = ssl_dim
+ self.emb_g = nn.Embedding(n_speakers, gin_channels)
+
+ self.pre = nn.Conv1d(ssl_dim, hidden_channels, kernel_size=5, padding=2)
+
+ self.enc_p = TextEncoder(
+ inter_channels,
+ hidden_channels,
+ filter_channels=filter_channels,
+ n_heads=n_heads,
+ n_layers=n_layers,
+ kernel_size=kernel_size,
+ p_dropout=p_dropout
+ )
+ hps = {
+ "sampling_rate": sampling_rate,
+ "inter_channels": inter_channels,
+ "resblock": resblock,
+ "resblock_kernel_sizes": resblock_kernel_sizes,
+ "resblock_dilation_sizes": resblock_dilation_sizes,
+ "upsample_rates": upsample_rates,
+ "upsample_initial_channel": upsample_initial_channel,
+ "upsample_kernel_sizes": upsample_kernel_sizes,
+ "gin_channels": gin_channels,
+ }
+ self.dec = Generator(h=hps)
+ self.enc_q = Encoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
+ self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels)
+ self.f0_decoder = F0Decoder(
+ 1,
+ hidden_channels,
+ filter_channels,
+ n_heads,
+ n_layers,
+ kernel_size,
+ p_dropout,
+ spk_channels=gin_channels
+ )
+ self.emb_uv = nn.Embedding(2, hidden_channels)
+
+ def forward(self, c, f0, uv, spec, g=None, c_lengths=None, spec_lengths=None):
+ g = self.emb_g(g).transpose(1, 2)
+ # ssl prenet
+ x_mask = torch.unsqueeze(commons.sequence_mask(c_lengths, c.size(2)), 1).to(c.dtype)
+ x = self.pre(c) * x_mask + self.emb_uv(uv.long()).transpose(1, 2)
+
+ # f0 predict
+ lf0 = 2595. * torch.log10(1. + f0.unsqueeze(1) / 700.) / 500
+ norm_lf0 = utils.normalize_f0(lf0, x_mask, uv)
+ pred_lf0 = self.f0_decoder(x, norm_lf0, x_mask, spk_emb=g)
+
+ # encoder
+ z_ptemp, m_p, logs_p, _ = self.enc_p(x, x_mask, f0=f0_to_coarse(f0))
+ z, m_q, logs_q, spec_mask = self.enc_q(spec, spec_lengths, g=g)
+
+ # flow
+ z_p = self.flow(z, spec_mask, g=g)
+ z_slice, pitch_slice, ids_slice = commons.rand_slice_segments_with_pitch(z, f0, spec_lengths, self.segment_size)
+
+ # nsf decoder
+ o = self.dec(z_slice, g=g, f0=pitch_slice)
+
+ return o, ids_slice, spec_mask, (z, z_p, m_p, logs_p, m_q, logs_q), pred_lf0, norm_lf0, lf0
+
+ def infer(self, c, f0, uv, g=None, noice_scale=0.35, predict_f0=False):
+ c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device)
+ g = self.emb_g(g).transpose(1, 2)
+ x_mask = torch.unsqueeze(commons.sequence_mask(c_lengths, c.size(2)), 1).to(c.dtype)
+ x = self.pre(c) * x_mask + self.emb_uv(uv.long()).transpose(1, 2)
+
+ if predict_f0:
+ lf0 = 2595. * torch.log10(1. + f0.unsqueeze(1) / 700.) / 500
+ norm_lf0 = utils.normalize_f0(lf0, x_mask, uv, random_scale=False)
+ pred_lf0 = self.f0_decoder(x, norm_lf0, x_mask, spk_emb=g)
+ f0 = (700 * (torch.pow(10, pred_lf0 * 500 / 2595) - 1)).squeeze(1)
+
+ z_p, m_p, logs_p, c_mask = self.enc_p(x, x_mask, f0=f0_to_coarse(f0), noice_scale=noice_scale)
+ z = self.flow(z_p, c_mask, g=g, reverse=True)
+ o = self.dec(z * c_mask, g=g, f0=f0)
+ return o
diff --git a/modules/__init__.py b/modules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/modules/attentions.py b/modules/attentions.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a94a553ab5743747f31aab994a0d52f69d7fb7a
--- /dev/null
+++ b/modules/attentions.py
@@ -0,0 +1,360 @@
+import copy
+import math
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+import modules.commons as commons
+import modules.modules as modules
+from modules.modules import LayerNorm
+
+
+class FFT(nn.Module):
+ def __init__(self, hidden_channels, filter_channels, n_heads, n_layers=1, kernel_size=1, p_dropout=0.,
+ proximal_bias=False, proximal_init=True, **kwargs):
+ super().__init__()
+ self.hidden_channels = hidden_channels
+ self.filter_channels = filter_channels
+ self.n_heads = n_heads
+ self.n_layers = n_layers
+ self.kernel_size = kernel_size
+ self.p_dropout = p_dropout
+ self.proximal_bias = proximal_bias
+ self.proximal_init = proximal_init
+
+ self.drop = nn.Dropout(p_dropout)
+ self.self_attn_layers = nn.ModuleList()
+ self.norm_layers_0 = nn.ModuleList()
+ self.ffn_layers = nn.ModuleList()
+ self.norm_layers_1 = nn.ModuleList()
+ for i in range(self.n_layers):
+ self.self_attn_layers.append(
+ MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout,
+ proximal_bias=proximal_bias,
+ proximal_init=proximal_init))
+ self.norm_layers_0.append(LayerNorm(hidden_channels))
+ self.ffn_layers.append(
+ FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
+
+ def forward(self, x, x_mask):
+ """
+ x: decoder input
+ h: encoder output
+ """
+ self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
+ x = x * x_mask
+ for i in range(self.n_layers):
+ y = self.self_attn_layers[i](x, x, self_attn_mask)
+ y = self.drop(y)
+ x = self.norm_layers_0[i](x + y)
+
+ y = self.ffn_layers[i](x, x_mask)
+ y = self.drop(y)
+ x = self.norm_layers_1[i](x + y)
+ x = x * x_mask
+ return x
+
+
+class Encoder(nn.Module):
+ def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4,
+ **kwargs):
+ super().__init__()
+ self.hidden_channels = hidden_channels
+ self.filter_channels = filter_channels
+ self.n_heads = n_heads
+ self.n_layers = n_layers
+ self.kernel_size = kernel_size
+ self.p_dropout = p_dropout
+ self.window_size = window_size
+
+ self.drop = nn.Dropout(p_dropout)
+ self.attn_layers = nn.ModuleList()
+ self.norm_layers_1 = nn.ModuleList()
+ self.ffn_layers = nn.ModuleList()
+ self.norm_layers_2 = nn.ModuleList()
+ for i in range(self.n_layers):
+ self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout,
+ window_size=window_size))
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
+ self.ffn_layers.append(
+ FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+ def forward(self, x, x_mask):
+ attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+ x = x * x_mask
+ for i in range(self.n_layers):
+ y = self.attn_layers[i](x, x, attn_mask)
+ y = self.drop(y)
+ x = self.norm_layers_1[i](x + y)
+
+ y = self.ffn_layers[i](x, x_mask)
+ y = self.drop(y)
+ x = self.norm_layers_2[i](x + y)
+ x = x * x_mask
+ return x
+
+
+class Decoder(nn.Module):
+ def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0.,
+ proximal_bias=False, proximal_init=True, **kwargs):
+ super().__init__()
+ self.hidden_channels = hidden_channels
+ self.filter_channels = filter_channels
+ self.n_heads = n_heads
+ self.n_layers = n_layers
+ self.kernel_size = kernel_size
+ self.p_dropout = p_dropout
+ self.proximal_bias = proximal_bias
+ self.proximal_init = proximal_init
+
+ self.drop = nn.Dropout(p_dropout)
+ self.self_attn_layers = nn.ModuleList()
+ self.norm_layers_0 = nn.ModuleList()
+ self.encdec_attn_layers = nn.ModuleList()
+ self.norm_layers_1 = nn.ModuleList()
+ self.ffn_layers = nn.ModuleList()
+ self.norm_layers_2 = nn.ModuleList()
+ for i in range(self.n_layers):
+ self.self_attn_layers.append(
+ MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout,
+ proximal_bias=proximal_bias, proximal_init=proximal_init))
+ self.norm_layers_0.append(LayerNorm(hidden_channels))
+ self.encdec_attn_layers.append(
+ MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
+ self.norm_layers_1.append(LayerNorm(hidden_channels))
+ self.ffn_layers.append(
+ FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
+ self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+ def forward(self, x, x_mask, h, h_mask):
+ """
+ x: decoder input
+ h: encoder output
+ """
+ self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
+ encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+ x = x * x_mask
+ for i in range(self.n_layers):
+ y = self.self_attn_layers[i](x, x, self_attn_mask)
+ y = self.drop(y)
+ x = self.norm_layers_0[i](x + y)
+
+ y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+ y = self.drop(y)
+ x = self.norm_layers_1[i](x + y)
+
+ y = self.ffn_layers[i](x, x_mask)
+ y = self.drop(y)
+ x = self.norm_layers_2[i](x + y)
+ x = x * x_mask
+ return x
+
+
+class MultiHeadAttention(nn.Module):
+ def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True,
+ block_length=None, proximal_bias=False, proximal_init=False):
+ super().__init__()
+ assert channels % n_heads == 0
+
+ self.channels = channels
+ self.out_channels = out_channels
+ self.n_heads = n_heads
+ self.p_dropout = p_dropout
+ self.window_size = window_size
+ self.heads_share = heads_share
+ self.block_length = block_length
+ self.proximal_bias = proximal_bias
+ self.proximal_init = proximal_init
+ self.attn = None
+
+ self.k_channels = channels // n_heads
+ self.conv_q = nn.Conv1d(channels, channels, 1)
+ self.conv_k = nn.Conv1d(channels, channels, 1)
+ self.conv_v = nn.Conv1d(channels, channels, 1)
+ self.conv_o = nn.Conv1d(channels, out_channels, 1)
+ self.drop = nn.Dropout(p_dropout)
+
+ if window_size is not None:
+ n_heads_rel = 1 if heads_share else n_heads
+ rel_stddev = self.k_channels ** -0.5
+ self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
+ self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
+
+ nn.init.xavier_uniform_(self.conv_q.weight)
+ nn.init.xavier_uniform_(self.conv_k.weight)
+ nn.init.xavier_uniform_(self.conv_v.weight)
+ if proximal_init:
+ with torch.no_grad():
+ self.conv_k.weight.copy_(self.conv_q.weight)
+ self.conv_k.bias.copy_(self.conv_q.bias)
+
+ def forward(self, x, c, attn_mask=None):
+ q = self.conv_q(x)
+ k = self.conv_k(c)
+ v = self.conv_v(c)
+
+ x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+ x = self.conv_o(x)
+ return x
+
+ def attention(self, query, key, value, mask=None):
+ # reshape [b, d, t] -> [b, n_h, t, d_k]
+ b, d, t_s, t_t = (*key.size(), query.size(2))
+ query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+ key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+ value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+ scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+ if self.window_size is not None:
+ assert t_s == t_t, "Relative attention is only available for self-attention."
+ key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
+ rel_logits = self._matmul_with_relative_keys(query / math.sqrt(self.k_channels), key_relative_embeddings)
+ scores_local = self._relative_position_to_absolute_position(rel_logits)
+ scores = scores + scores_local
+ if self.proximal_bias:
+ assert t_s == t_t, "Proximal bias is only available for self-attention."
+ scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
+ if mask is not None:
+ scores = scores.masked_fill(mask == 0, -1e4)
+ if self.block_length is not None:
+ assert t_s == t_t, "Local attention is only available for self-attention."
+ block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
+ scores = scores.masked_fill(block_mask == 0, -1e4)
+ p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
+ p_attn = self.drop(p_attn)
+ output = torch.matmul(p_attn, value)
+ if self.window_size is not None:
+ relative_weights = self._absolute_position_to_relative_position(p_attn)
+ value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
+ output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
+ output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
+ return output, p_attn
+
+ def _matmul_with_relative_values(self, x, y):
+ """
+ x: [b, h, l, m]
+ y: [h or 1, m, d]
+ ret: [b, h, l, d]
+ """
+ ret = torch.matmul(x, y.unsqueeze(0))
+ return ret
+
+ def _matmul_with_relative_keys(self, x, y):
+ """
+ x: [b, h, l, d]
+ y: [h or 1, m, d]
+ ret: [b, h, l, m]
+ """
+ ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+ return ret
+
+ def _get_relative_embeddings(self, relative_embeddings, length):
+ max_relative_position = 2 * self.window_size + 1
+ # Pad first before slice to avoid using cond ops.
+ pad_length = max(length - (self.window_size + 1), 0)
+ slice_start_position = max((self.window_size + 1) - length, 0)
+ slice_end_position = slice_start_position + 2 * length - 1
+ if pad_length > 0:
+ padded_relative_embeddings = F.pad(
+ relative_embeddings,
+ commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
+ else:
+ padded_relative_embeddings = relative_embeddings
+ used_relative_embeddings = padded_relative_embeddings[:, slice_start_position:slice_end_position]
+ return used_relative_embeddings
+
+ def _relative_position_to_absolute_position(self, x):
+ """
+ x: [b, h, l, 2*l-1]
+ ret: [b, h, l, l]
+ """
+ batch, heads, length, _ = x.size()
+ # Concat columns of pad to shift from relative to absolute indexing.
+ x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
+
+ # Concat extra elements so to add up to shape (len+1, 2*len-1).
+ x_flat = x.view([batch, heads, length * 2 * length])
+ x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]]))
+
+ # Reshape and slice out the padded elements.
+ x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[:, :, :length, length - 1:]
+ return x_final
+
+ def _absolute_position_to_relative_position(self, x):
+ """
+ x: [b, h, l, l]
+ ret: [b, h, l, 2*l-1]
+ """
+ batch, heads, length, _ = x.size()
+ # padd along column
+ x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]]))
+ x_flat = x.view([batch, heads, length ** 2 + length * (length - 1)])
+ # add 0's in the beginning that will skew the elements after reshape
+ x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+ x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+ return x_final
+
+ def _attention_bias_proximal(self, length):
+ """Bias for self-attention to encourage attention to close positions.
+ Args:
+ length: an integer scalar.
+ Returns:
+ a Tensor with shape [1, 1, length, length]
+ """
+ r = torch.arange(length, dtype=torch.float32)
+ diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+ return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+ def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None,
+ causal=False):
+ super().__init__()
+ self.in_channels = in_channels
+ self.out_channels = out_channels
+ self.filter_channels = filter_channels
+ self.kernel_size = kernel_size
+ self.p_dropout = p_dropout
+ self.activation = activation
+ self.causal = causal
+
+ if causal:
+ self.padding = self._causal_padding
+ else:
+ self.padding = self._same_padding
+
+ self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+ self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+ self.drop = nn.Dropout(p_dropout)
+
+ def forward(self, x, x_mask):
+ x = self.conv_1(self.padding(x * x_mask))
+ if self.activation == "gelu":
+ x = x * torch.sigmoid(1.702 * x)
+ else:
+ x = torch.relu(x)
+ x = self.drop(x)
+ x = self.conv_2(self.padding(x * x_mask))
+ return x * x_mask
+
+ def _causal_padding(self, x):
+ if self.kernel_size == 1:
+ return x
+ pad_l = self.kernel_size - 1
+ pad_r = 0
+ padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+ x = F.pad(x, commons.convert_pad_shape(padding))
+ return x
+
+ def _same_padding(self, x):
+ if self.kernel_size == 1:
+ return x
+ pad_l = (self.kernel_size - 1) // 2
+ pad_r = self.kernel_size // 2
+ padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+ x = F.pad(x, commons.convert_pad_shape(padding))
+ return x
diff --git a/modules/commons.py b/modules/commons.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d2ef187b8ce625087bce46c5d8a6319e4ddea89
--- /dev/null
+++ b/modules/commons.py
@@ -0,0 +1,191 @@
+import math
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def slice_pitch_segments(x, ids_str, segment_size=4):
+ ret = torch.zeros_like(x[:, :segment_size])
+ for i in range(x.size(0)):
+ idx_str = ids_str[i]
+ idx_end = idx_str + segment_size
+ ret[i] = x[i, idx_str:idx_end]
+ return ret
+
+
+def rand_slice_segments_with_pitch(x, pitch, x_lengths=None, segment_size=4):
+ b, d, t = x.size()
+ if x_lengths is None:
+ x_lengths = t
+ ids_str_max = x_lengths - segment_size + 1
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
+ ret = slice_segments(x, ids_str, segment_size)
+ ret_pitch = slice_pitch_segments(pitch, ids_str, segment_size)
+ return ret, ret_pitch, ids_str
+
+
+def init_weights(m, mean=0.0, std=0.01):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+ return int((kernel_size * dilation - dilation) / 2)
+
+
+def convert_pad_shape(pad_shape):
+ l = pad_shape[::-1]
+ pad_shape = [item for sublist in l for item in sublist]
+ return pad_shape
+
+
+def intersperse(lst, item):
+ result = [item] * (len(lst) * 2 + 1)
+ result[1::2] = lst
+ return result
+
+
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+ """KL(P||Q)"""
+ kl = (logs_q - logs_p) - 0.5
+ kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2. * logs_q)
+ return kl
+
+
+def rand_gumbel(shape):
+ """Sample from the Gumbel distribution, protect from overflows."""
+ uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
+ return -torch.log(-torch.log(uniform_samples))
+
+
+def rand_gumbel_like(x):
+ g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
+ return g
+
+
+def slice_segments(x, ids_str, segment_size=4):
+ ret = torch.zeros_like(x[:, :, :segment_size])
+ for i in range(x.size(0)):
+ idx_str = ids_str[i]
+ idx_end = idx_str + segment_size
+ ret[i] = x[i, :, idx_str:idx_end]
+ return ret
+
+
+def rand_slice_segments(x, x_lengths=None, segment_size=4):
+ b, d, t = x.size()
+ if x_lengths is None:
+ x_lengths = t
+ ids_str_max = x_lengths - segment_size + 1
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
+ ret = slice_segments(x, ids_str, segment_size)
+ return ret, ids_str
+
+
+def rand_spec_segments(x, x_lengths=None, segment_size=4):
+ b, d, t = x.size()
+ if x_lengths is None:
+ x_lengths = t
+ ids_str_max = x_lengths - segment_size
+ ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
+ ret = slice_segments(x, ids_str, segment_size)
+ return ret, ids_str
+
+
+def get_timing_signal_1d(
+ length, channels, min_timescale=1.0, max_timescale=1.0e4):
+ position = torch.arange(length, dtype=torch.float)
+ num_timescales = channels // 2
+ log_timescale_increment = (
+ math.log(float(max_timescale) / float(min_timescale)) /
+ (num_timescales - 1))
+ inv_timescales = min_timescale * torch.exp(
+ torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
+ scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+ signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+ signal = F.pad(signal, [0, 0, 0, channels % 2])
+ signal = signal.view(1, channels, length)
+ return signal
+
+
+def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
+ b, channels, length = x.size()
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+ return x + signal.to(dtype=x.dtype, device=x.device)
+
+
+def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
+ b, channels, length = x.size()
+ signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+ return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
+
+
+def subsequent_mask(length):
+ mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+ return mask
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+ n_channels_int = n_channels[0]
+ in_act = input_a + input_b
+ t_act = torch.tanh(in_act[:, :n_channels_int, :])
+ s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+ acts = t_act * s_act
+ return acts
+
+
+def convert_pad_shape(pad_shape):
+ l = pad_shape[::-1]
+ pad_shape = [item for sublist in l for item in sublist]
+ return pad_shape
+
+
+def shift_1d(x):
+ x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
+ return x
+
+
+def sequence_mask(length, max_length=None):
+ if max_length is None:
+ max_length = length.max()
+ x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+ return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def generate_path(duration, mask):
+ """
+ duration: [b, 1, t_x]
+ mask: [b, 1, t_y, t_x]
+ """
+ device = duration.device
+
+ b, _, t_y, t_x = mask.shape
+ cum_duration = torch.cumsum(duration, -1)
+
+ cum_duration_flat = cum_duration.view(b * t_x)
+ path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+ path = path.view(b, t_x, t_y)
+ path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
+ path = path.unsqueeze(1).transpose(2, 3) * mask
+ return path
+
+
+def clip_grad_value_(parameters, clip_value, norm_type=2):
+ if isinstance(parameters, torch.Tensor):
+ parameters = [parameters]
+ parameters = list(filter(lambda p: p.grad is not None, parameters))
+ norm_type = float(norm_type)
+ if clip_value is not None:
+ clip_value = float(clip_value)
+
+ total_norm = 0
+ for p in parameters:
+ param_norm = p.grad.data.norm(norm_type)
+ total_norm += param_norm.item() ** norm_type
+ if clip_value is not None:
+ p.grad.data.clamp_(min=-clip_value, max=clip_value)
+ total_norm = total_norm ** (1. / norm_type)
+ return total_norm
diff --git a/modules/modules.py b/modules/modules.py
new file mode 100644
index 0000000000000000000000000000000000000000..87aeec4a1949ab8bcd2ae1997e51100cd5b096ea
--- /dev/null
+++ b/modules/modules.py
@@ -0,0 +1,339 @@
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from torch.nn import Conv1d
+from torch.nn.utils import weight_norm, remove_weight_norm
+
+from modules import commons
+from modules.commons import init_weights, get_padding
+
+LRELU_SLOPE = 0.1
+
+
+class LayerNorm(nn.Module):
+ def __init__(self, channels, eps=1e-5):
+ super().__init__()
+ self.channels = channels
+ self.eps = eps
+
+ self.gamma = nn.Parameter(torch.ones(channels))
+ self.beta = nn.Parameter(torch.zeros(channels))
+
+ def forward(self, x):
+ x = x.transpose(1, -1)
+ x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
+ return x.transpose(1, -1)
+
+
+class ConvReluNorm(nn.Module):
+ def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
+ super().__init__()
+ self.in_channels = in_channels
+ self.hidden_channels = hidden_channels
+ self.out_channels = out_channels
+ self.kernel_size = kernel_size
+ self.n_layers = n_layers
+ self.p_dropout = p_dropout
+ assert n_layers > 1, "Number of layers should be larger than 0."
+
+ self.conv_layers = nn.ModuleList()
+ self.norm_layers = nn.ModuleList()
+ self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size // 2))
+ self.norm_layers.append(LayerNorm(hidden_channels))
+ self.relu_drop = nn.Sequential(
+ nn.ReLU(),
+ nn.Dropout(p_dropout))
+ for _ in range(n_layers - 1):
+ self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size // 2))
+ self.norm_layers.append(LayerNorm(hidden_channels))
+ self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
+ self.proj.weight.data.zero_()
+ self.proj.bias.data.zero_()
+
+ def forward(self, x, x_mask):
+ x_org = x
+ for i in range(self.n_layers):
+ x = self.conv_layers[i](x * x_mask)
+ x = self.norm_layers[i](x)
+ x = self.relu_drop(x)
+ x = x_org + self.proj(x)
+ return x * x_mask
+
+
+class DDSConv(nn.Module):
+ """
+ Dialted and Depth-Separable Convolution
+ """
+
+ def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
+ super().__init__()
+ self.channels = channels
+ self.kernel_size = kernel_size
+ self.n_layers = n_layers
+ self.p_dropout = p_dropout
+
+ self.drop = nn.Dropout(p_dropout)
+ self.convs_sep = nn.ModuleList()
+ self.convs_1x1 = nn.ModuleList()
+ self.norms_1 = nn.ModuleList()
+ self.norms_2 = nn.ModuleList()
+ for i in range(n_layers):
+ dilation = kernel_size ** i
+ padding = (kernel_size * dilation - dilation) // 2
+ self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size,
+ groups=channels, dilation=dilation, padding=padding
+ ))
+ self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
+ self.norms_1.append(LayerNorm(channels))
+ self.norms_2.append(LayerNorm(channels))
+
+ def forward(self, x, x_mask, g=None):
+ if g is not None:
+ x = x + g
+ for i in range(self.n_layers):
+ y = self.convs_sep[i](x * x_mask)
+ y = self.norms_1[i](y)
+ y = F.gelu(y)
+ y = self.convs_1x1[i](y)
+ y = self.norms_2[i](y)
+ y = F.gelu(y)
+ y = self.drop(y)
+ x = x + y
+ return x * x_mask
+
+
+class WN(torch.nn.Module):
+ def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
+ super(WN, self).__init__()
+ assert (kernel_size % 2 == 1)
+ self.hidden_channels = hidden_channels
+ self.kernel_size = kernel_size,
+ self.dilation_rate = dilation_rate
+ self.n_layers = n_layers
+ self.gin_channels = gin_channels
+ self.p_dropout = p_dropout
+
+ self.in_layers = torch.nn.ModuleList()
+ self.res_skip_layers = torch.nn.ModuleList()
+ self.drop = nn.Dropout(p_dropout)
+
+ if gin_channels != 0:
+ cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1)
+ self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
+
+ for i in range(n_layers):
+ dilation = dilation_rate ** i
+ padding = int((kernel_size * dilation - dilation) / 2)
+ in_layer = torch.nn.Conv1d(hidden_channels, 2 * hidden_channels, kernel_size,
+ dilation=dilation, padding=padding)
+ in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
+ self.in_layers.append(in_layer)
+
+ # last one is not necessary
+ if i < n_layers - 1:
+ res_skip_channels = 2 * hidden_channels
+ else:
+ res_skip_channels = hidden_channels
+
+ res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
+ res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
+ self.res_skip_layers.append(res_skip_layer)
+
+ def forward(self, x, x_mask, g=None, **kwargs):
+ output = torch.zeros_like(x)
+ n_channels_tensor = torch.IntTensor([self.hidden_channels])
+
+ if g is not None:
+ g = self.cond_layer(g)
+
+ for i in range(self.n_layers):
+ x_in = self.in_layers[i](x)
+ if g is not None:
+ cond_offset = i * 2 * self.hidden_channels
+ g_l = g[:, cond_offset:cond_offset + 2 * self.hidden_channels, :]
+ else:
+ g_l = torch.zeros_like(x_in)
+
+ acts = commons.fused_add_tanh_sigmoid_multiply(
+ x_in,
+ g_l,
+ n_channels_tensor)
+ acts = self.drop(acts)
+
+ res_skip_acts = self.res_skip_layers[i](acts)
+ if i < self.n_layers - 1:
+ res_acts = res_skip_acts[:, :self.hidden_channels, :]
+ x = (x + res_acts) * x_mask
+ output = output + res_skip_acts[:, self.hidden_channels:, :]
+ else:
+ output = output + res_skip_acts
+ return output * x_mask
+
+ def remove_weight_norm(self):
+ if self.gin_channels != 0:
+ torch.nn.utils.remove_weight_norm(self.cond_layer)
+ for l in self.in_layers:
+ torch.nn.utils.remove_weight_norm(l)
+ for l in self.res_skip_layers:
+ torch.nn.utils.remove_weight_norm(l)
+
+
+class ResBlock1(torch.nn.Module):
+ def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+ super(ResBlock1, self).__init__()
+ self.convs1 = nn.ModuleList([
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+ padding=get_padding(kernel_size, dilation[0]))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+ padding=get_padding(kernel_size, dilation[1]))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+ padding=get_padding(kernel_size, dilation[2])))
+ ])
+ self.convs1.apply(init_weights)
+
+ self.convs2 = nn.ModuleList([
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+ padding=get_padding(kernel_size, 1))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+ padding=get_padding(kernel_size, 1))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+ padding=get_padding(kernel_size, 1)))
+ ])
+ self.convs2.apply(init_weights)
+
+ def forward(self, x, x_mask=None):
+ for c1, c2 in zip(self.convs1, self.convs2):
+ xt = F.leaky_relu(x, LRELU_SLOPE)
+ if x_mask is not None:
+ xt = xt * x_mask
+ xt = c1(xt)
+ xt = F.leaky_relu(xt, LRELU_SLOPE)
+ if x_mask is not None:
+ xt = xt * x_mask
+ xt = c2(xt)
+ x = xt + x
+ if x_mask is not None:
+ x = x * x_mask
+ return x
+
+ def remove_weight_norm(self):
+ for l in self.convs1:
+ remove_weight_norm(l)
+ for l in self.convs2:
+ remove_weight_norm(l)
+
+
+class ResBlock2(torch.nn.Module):
+ def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
+ super(ResBlock2, self).__init__()
+ self.convs = nn.ModuleList([
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+ padding=get_padding(kernel_size, dilation[0]))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+ padding=get_padding(kernel_size, dilation[1])))
+ ])
+ self.convs.apply(init_weights)
+
+ def forward(self, x, x_mask=None):
+ for c in self.convs:
+ xt = F.leaky_relu(x, LRELU_SLOPE)
+ if x_mask is not None:
+ xt = xt * x_mask
+ xt = c(xt)
+ x = xt + x
+ if x_mask is not None:
+ x = x * x_mask
+ return x
+
+ def remove_weight_norm(self):
+ for l in self.convs:
+ remove_weight_norm(l)
+
+
+class Log(nn.Module):
+ def forward(self, x, x_mask, reverse=False, **kwargs):
+ if not reverse:
+ y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
+ logdet = torch.sum(-y, [1, 2])
+ return y, logdet
+ else:
+ x = torch.exp(x) * x_mask
+ return x
+
+
+class Flip(nn.Module):
+ def forward(self, x, *args, reverse=False, **kwargs):
+ x = torch.flip(x, [1])
+ if not reverse:
+ logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
+ return x, logdet
+ else:
+ return x
+
+
+class ElementwiseAffine(nn.Module):
+ def __init__(self, channels):
+ super().__init__()
+ self.channels = channels
+ self.m = nn.Parameter(torch.zeros(channels, 1))
+ self.logs = nn.Parameter(torch.zeros(channels, 1))
+
+ def forward(self, x, x_mask, reverse=False, **kwargs):
+ if not reverse:
+ y = self.m + torch.exp(self.logs) * x
+ y = y * x_mask
+ logdet = torch.sum(self.logs * x_mask, [1, 2])
+ return y, logdet
+ else:
+ x = (x - self.m) * torch.exp(-self.logs) * x_mask
+ return x
+
+
+class ResidualCouplingLayer(nn.Module):
+ def __init__(self,
+ channels,
+ hidden_channels,
+ kernel_size,
+ dilation_rate,
+ n_layers,
+ p_dropout=0,
+ gin_channels=0,
+ mean_only=False):
+ assert channels % 2 == 0, "channels should be divisible by 2"
+ super().__init__()
+ self.channels = channels
+ self.hidden_channels = hidden_channels
+ self.kernel_size = kernel_size
+ self.dilation_rate = dilation_rate
+ self.n_layers = n_layers
+ self.half_channels = channels // 2
+ self.mean_only = mean_only
+
+ self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
+ self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout,
+ gin_channels=gin_channels)
+ self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
+ self.post.weight.data.zero_()
+ self.post.bias.data.zero_()
+
+ def forward(self, x, x_mask, g=None, reverse=False):
+ x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+ h = self.pre(x0) * x_mask
+ h = self.enc(h, x_mask, g=g)
+ stats = self.post(h) * x_mask
+ if not self.mean_only:
+ m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+ else:
+ m = stats
+ logs = torch.zeros_like(m)
+
+ if not reverse:
+ x1 = m + x1 * torch.exp(logs) * x_mask
+ x = torch.cat([x0, x1], 1)
+ logdet = torch.sum(logs, [1, 2])
+ return x, logdet
+ else:
+ x1 = (x1 - m) * torch.exp(-logs) * x_mask
+ x = torch.cat([x0, x1], 1)
+ return x
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c49d2e92d57ff7313167e763397dc2258c34aec8
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,21 @@
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch==1.10.0+cu113
+--extra-index-url https://download.pytorch.org/whl/cu113
+torchaudio==0.10.0+cu113
+# torch==1.10.0
+# torchaudio==0.10.0
+scipy
+numpy
+praat-parselmouth
+scikit-maad
+torchvision
+fairseq
+pyworld
+aiohttp
+tornado
+spleeter
+pymusic-dl
+gradio
+librosa==0.8.1
+setuptools==59.5.0
+SoundFile
\ No newline at end of file
diff --git a/requirements.txt.bkup b/requirements.txt.bkup
new file mode 100644
index 0000000000000000000000000000000000000000..e3ae9c637a4c7d0ee823f85fc50e392f075bfbe1
--- /dev/null
+++ b/requirements.txt.bkup
@@ -0,0 +1,21 @@
+gradio==3.20.0
+librosa==0.10.0
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch==1.10.0+cu113
+--extra-index-url https://download.pytorch.org/whl/cu113
+torchaudio==0.10.0+cu113
+# torch==1.10.0
+# torchaudio==0.10.0
+scipy==1.7.3
+numpy
+SoundFile==0.12.1
+praat-parselmouth
+scikit-maad
+torchvision
+fairseq
+pyworld
+aiohttp==3.8.4
+tornado
+setuptools==59.5.0
+spleeter
+pymusic-dl
\ No newline at end of file
diff --git a/service/tool.py b/service/tool.py
new file mode 100644
index 0000000000000000000000000000000000000000..df9a106c059057009cabff83e783b7c2c78d02df
--- /dev/null
+++ b/service/tool.py
@@ -0,0 +1,60 @@
+import os
+import subprocess
+import tempfile
+import logging
+import wave
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+FFMPEG_CMD = "ffmpeg"
+
+
+async def audio_normalize(full_filename, file_data):
+ try:
+ file_name = os.path.splitext(full_filename)[0]
+ file_extension = os.path.splitext(full_filename)[1].lower()
+
+ logger.debug(f"normalizing {full_filename}")
+
+ with tempfile.NamedTemporaryFile(suffix=file_extension, delete=False) as temp_file:
+ temp_file.write(file_data)
+ temp_file.flush()
+ temp_out_filename = f"norm_{file_name}.wav"
+
+ subprocess.call(
+ [FFMPEG_CMD,
+ '-i', temp_file.name,
+ '-ac', '1',
+ '-ar', '44100',
+ '-acodec', 'pcm_s16le',
+ '-f', 'wav',
+ temp_out_filename])
+
+ if os.path.exists(temp_out_filename):
+ return temp_out_filename
+ else:
+ raise FileNotFoundError("Unable to make the file")
+ except Exception as e:
+ logger.exception(e)
+ return None
+
+
+def read_wav_file_to_numpy_array(filename):
+ with wave.open(filename, 'rb') as wav_file:
+ sampling_rate = wav_file.getframerate()
+ bytes_data = wav_file.readframes(wav_file.getnframes())
+ sample_width = wav_file.getsampwidth()
+
+ if sample_width == 1:
+ data_type = np.uint8
+ elif sample_width == 2:
+ data_type = np.int16
+ elif sample_width == 4:
+ data_type = np.int32
+ else:
+ raise ValueError(f"Unsupported sample width: {sample_width}")
+
+ numpy_data = np.frombuffer(bytes_data, dtype=data_type)
+
+ return sampling_rate, numpy_data
diff --git a/utils.py b/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b896c211acb77f22425f4a935d8639136b3ae88
--- /dev/null
+++ b/utils.py
@@ -0,0 +1,492 @@
+import os
+import glob
+import re
+import sys
+import argparse
+import logging
+import json
+import subprocess
+
+import numpy as np
+from scipy.io.wavfile import read
+import torch
+
+MATPLOTLIB_FLAG = False
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+logger = logging
+
+f0_bin = 256
+f0_max = 1100.0
+f0_min = 50.0
+f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+
+def normalize_f0(f0, x_mask, uv, random_scale=True):
+ # calculate means based on x_mask
+ uv_sum = torch.sum(uv, dim=1, keepdim=True)
+ uv_sum[uv_sum == 0] = 9999
+ means = torch.sum(f0[:, 0, :] * uv, dim=1, keepdim=True) / uv_sum
+
+ if random_scale:
+ factor = torch.Tensor(f0.shape[0], 1).uniform_(0.8, 1.2).to(f0.device)
+ else:
+ factor = torch.ones(f0.shape[0], 1).to(f0.device)
+ # normalize f0 based on means and factor
+ f0_norm = (f0 - means.unsqueeze(-1)) * factor.unsqueeze(-1)
+ if torch.isnan(f0_norm).any():
+ exit(0)
+ return f0_norm * x_mask
+
+
+def plot_data_to_numpy(x, y):
+ global MATPLOTLIB_FLAG
+ if not MATPLOTLIB_FLAG:
+ import matplotlib
+ matplotlib.use("Agg")
+ MATPLOTLIB_FLAG = True
+ mpl_logger = logging.getLogger('matplotlib')
+ mpl_logger.setLevel(logging.WARNING)
+ import matplotlib.pylab as plt
+ import numpy as np
+
+ fig, ax = plt.subplots(figsize=(10, 2))
+ plt.plot(x)
+ plt.plot(y)
+ plt.tight_layout()
+
+ fig.canvas.draw()
+ data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+ data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+ plt.close()
+ return data
+
+
+def interpolate_f0(f0):
+ """
+ 对F0进行插值处理
+ """
+
+ data = np.reshape(f0, (f0.size, 1))
+
+ vuv_vector = np.zeros((data.size, 1), dtype=np.float32)
+ vuv_vector[data > 0.0] = 1.0
+ vuv_vector[data <= 0.0] = 0.0
+
+ ip_data = data
+
+ frame_number = data.size
+ last_value = 0.0
+ for i in range(frame_number):
+ if data[i] <= 0.0:
+ j = i + 1
+ for j in range(i + 1, frame_number):
+ if data[j] > 0.0:
+ break
+ if j < frame_number - 1:
+ if last_value > 0.0:
+ step = (data[j] - data[i - 1]) / float(j - i)
+ for k in range(i, j):
+ ip_data[k] = data[i - 1] + step * (k - i + 1)
+ else:
+ for k in range(i, j):
+ ip_data[k] = data[j]
+ else:
+ for k in range(i, frame_number):
+ ip_data[k] = last_value
+ else:
+ ip_data[i] = data[i]
+ last_value = data[i]
+
+ return ip_data[:, 0], vuv_vector[:, 0]
+
+
+def compute_f0_parselmouth(wav_numpy, p_len=None, sampling_rate=44100, hop_length=512):
+ import parselmouth
+ x = wav_numpy
+ if p_len is None:
+ p_len = x.shape[0] // hop_length
+ else:
+ assert abs(p_len - x.shape[0] // hop_length) < 4, "pad length error"
+ time_step = hop_length / sampling_rate * 1000
+ f0_min = 50
+ f0_max = 1100
+ f0 = parselmouth.Sound(x, sampling_rate).to_pitch_ac(
+ time_step=time_step / 1000, voicing_threshold=0.6,
+ pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
+
+ pad_size = (p_len - len(f0) + 1) // 2
+ if (pad_size > 0 or p_len - len(f0) - pad_size > 0):
+ f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode='constant')
+ return f0
+
+
+def resize_f0(x, target_len):
+ source = np.array(x)
+ source[source < 0.001] = np.nan
+ target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)),
+ source)
+ res = np.nan_to_num(target)
+ return res
+
+
+def compute_f0_dio(wav_numpy, p_len=None, sampling_rate=44100, hop_length=512):
+ import pyworld
+ if p_len is None:
+ p_len = wav_numpy.shape[0] // hop_length
+ f0, t = pyworld.dio(
+ wav_numpy.astype(np.double),
+ fs=sampling_rate,
+ f0_ceil=800,
+ frame_period=1000 * hop_length / sampling_rate,
+ )
+ f0 = pyworld.stonemask(wav_numpy.astype(np.double), f0, t, sampling_rate)
+ for index, pitch in enumerate(f0):
+ f0[index] = round(pitch, 1)
+ return resize_f0(f0, p_len)
+
+
+def f0_to_coarse(f0):
+ is_torch = isinstance(f0, torch.Tensor)
+ f0_mel = 1127 * (1 + f0 / 700).log() if is_torch else 1127 * np.log(1 + f0 / 700)
+ f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (f0_bin - 2) / (f0_mel_max - f0_mel_min) + 1
+
+ f0_mel[f0_mel <= 1] = 1
+ f0_mel[f0_mel > f0_bin - 1] = f0_bin - 1
+ f0_coarse = (f0_mel + 0.5).long() if is_torch else np.rint(f0_mel).astype(np.int)
+ assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (f0_coarse.max(), f0_coarse.min())
+ return f0_coarse
+
+
+def get_hubert_model():
+ vec_path = "hubert/checkpoint_best_legacy_500.pt"
+ print("load model(s) from {}".format(vec_path))
+ from fairseq import checkpoint_utils
+ models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(
+ [vec_path],
+ suffix="",
+ )
+ model = models[0]
+ model.eval()
+ return model
+
+
+def get_hubert_content(hmodel, wav_16k_tensor):
+ feats = wav_16k_tensor
+ if feats.dim() == 2: # double channels
+ feats = feats.mean(-1)
+ assert feats.dim() == 1, feats.dim()
+ feats = feats.view(1, -1)
+ padding_mask = torch.BoolTensor(feats.shape).fill_(False)
+ inputs = {
+ "source": feats.to(wav_16k_tensor.device),
+ "padding_mask": padding_mask.to(wav_16k_tensor.device),
+ "output_layer": 9, # layer 9
+ }
+ with torch.no_grad():
+ logits = hmodel.extract_features(**inputs)
+ feats = hmodel.final_proj(logits[0])
+ return feats.transpose(1, 2)
+
+
+def get_content(cmodel, y):
+ with torch.no_grad():
+ c = cmodel.extract_features(y.squeeze(1))[0]
+ c = c.transpose(1, 2)
+ return c
+
+
+def load_checkpoint(checkpoint_path, model, optimizer=None, skip_optimizer=False):
+ assert os.path.isfile(checkpoint_path)
+ checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
+ iteration = checkpoint_dict['iteration']
+ learning_rate = checkpoint_dict['learning_rate']
+ if optimizer is not None and not skip_optimizer and checkpoint_dict['optimizer'] is not None:
+ optimizer.load_state_dict(checkpoint_dict['optimizer'])
+ saved_state_dict = checkpoint_dict['model']
+ if hasattr(model, 'module'):
+ state_dict = model.module.state_dict()
+ else:
+ state_dict = model.state_dict()
+ new_state_dict = {}
+ for k, v in state_dict.items():
+ try:
+ # assert "dec" in k or "disc" in k
+ # print("load", k)
+ new_state_dict[k] = saved_state_dict[k]
+ assert saved_state_dict[k].shape == v.shape, (saved_state_dict[k].shape, v.shape)
+ except:
+ print("error, %s is not in the checkpoint" % k)
+ logger.info("%s is not in the checkpoint" % k)
+ new_state_dict[k] = v
+ if hasattr(model, 'module'):
+ model.module.load_state_dict(new_state_dict)
+ else:
+ model.load_state_dict(new_state_dict)
+ print("load ")
+ logger.info("Loaded checkpoint '{}' (iteration {})".format(
+ checkpoint_path, iteration))
+ return model, optimizer, learning_rate, iteration
+
+
+def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path, val_steps, current_step):
+ logger.info("Saving model and optimizer state at iteration {} to {}".format(
+ iteration, checkpoint_path))
+ if hasattr(model, 'module'):
+ state_dict = model.module.state_dict()
+ else:
+ state_dict = model.state_dict()
+ torch.save({'model': state_dict,
+ 'iteration': iteration,
+ 'optimizer': optimizer.state_dict(),
+ 'learning_rate': learning_rate}, checkpoint_path)
+ if current_step >= val_steps * 3:
+ to_del_ckptname = checkpoint_path.replace(str(current_step), str(current_step - val_steps * 3))
+ if os.path.exists(to_del_ckptname):
+ os.remove(to_del_ckptname)
+ print("Removing ", to_del_ckptname)
+
+
+def clean_checkpoints(path_to_models='logs/48k/', n_ckpts_to_keep=2, sort_by_time=True):
+ """Freeing up space by deleting saved ckpts
+
+ Arguments:
+ path_to_models -- Path to the model directory
+ n_ckpts_to_keep -- Number of ckpts to keep, excluding G_0.pth and D_0.pth
+ sort_by_time -- True -> chronologically delete ckpts
+ False -> lexicographically delete ckpts
+ """
+ ckpts_files = [f for f in os.listdir(path_to_models) if os.path.isfile(os.path.join(path_to_models, f))]
+ name_key = (lambda _f: int(re.compile('._(\d+)\.pth').match(_f).group(1)))
+ time_key = (lambda _f: os.path.getmtime(os.path.join(path_to_models, _f)))
+ sort_key = time_key if sort_by_time else name_key
+ x_sorted = lambda _x: sorted([f for f in ckpts_files if f.startswith(_x) and not f.endswith('_0.pth')],
+ key=sort_key)
+ to_del = [os.path.join(path_to_models, fn) for fn in
+ (x_sorted('G')[:-n_ckpts_to_keep] + x_sorted('D')[:-n_ckpts_to_keep])]
+ del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}")
+ del_routine = lambda x: [os.remove(x), del_info(x)]
+ rs = [del_routine(fn) for fn in to_del]
+
+
+def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050):
+ for k, v in scalars.items():
+ writer.add_scalar(k, v, global_step)
+ for k, v in histograms.items():
+ writer.add_histogram(k, v, global_step)
+ for k, v in images.items():
+ writer.add_image(k, v, global_step, dataformats='HWC')
+ for k, v in audios.items():
+ writer.add_audio(k, v, global_step, audio_sampling_rate)
+
+
+def latest_checkpoint_path(dir_path, regex="G_*.pth"):
+ f_list = glob.glob(os.path.join(dir_path, regex))
+ f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
+ x = f_list[-1]
+ print(x)
+ return x
+
+
+def plot_spectrogram_to_numpy(spectrogram):
+ global MATPLOTLIB_FLAG
+ if not MATPLOTLIB_FLAG:
+ import matplotlib
+ matplotlib.use("Agg")
+ MATPLOTLIB_FLAG = True
+ mpl_logger = logging.getLogger('matplotlib')
+ mpl_logger.setLevel(logging.WARNING)
+ import matplotlib.pylab as plt
+ import numpy as np
+
+ fig, ax = plt.subplots(figsize=(10, 2))
+ im = ax.imshow(spectrogram, aspect="auto", origin="lower",
+ interpolation='none')
+ plt.colorbar(im, ax=ax)
+ plt.xlabel("Frames")
+ plt.ylabel("Channels")
+ plt.tight_layout()
+
+ fig.canvas.draw()
+ data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+ data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+ plt.close()
+ return data
+
+
+def plot_alignment_to_numpy(alignment, info=None):
+ global MATPLOTLIB_FLAG
+ if not MATPLOTLIB_FLAG:
+ import matplotlib
+ matplotlib.use("Agg")
+ MATPLOTLIB_FLAG = True
+ mpl_logger = logging.getLogger('matplotlib')
+ mpl_logger.setLevel(logging.WARNING)
+ import matplotlib.pylab as plt
+ import numpy as np
+
+ fig, ax = plt.subplots(figsize=(6, 4))
+ im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower',
+ interpolation='none')
+ fig.colorbar(im, ax=ax)
+ xlabel = 'Decoder timestep'
+ if info is not None:
+ xlabel += '\n\n' + info
+ plt.xlabel(xlabel)
+ plt.ylabel('Encoder timestep')
+ plt.tight_layout()
+
+ fig.canvas.draw()
+ data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
+ data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+ plt.close()
+ return data
+
+
+def load_wav_to_torch(full_path):
+ sampling_rate, data = read(full_path)
+ return torch.FloatTensor(data.astype(np.float32)), sampling_rate
+
+
+def load_filepaths_and_text(filename, split="|"):
+ with open(filename, encoding='utf-8') as f:
+ filepaths_and_text = [line.strip().split(split) for line in f]
+ return filepaths_and_text
+
+
+def get_hparams(init=True):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-c', '--config', type=str, default="./configs/base.json",
+ help='JSON file for configuration')
+ parser.add_argument('-m', '--model', type=str, required=True,
+ help='Model name')
+
+ args = parser.parse_args()
+ model_dir = os.path.join("./logs", args.model)
+
+ if not os.path.exists(model_dir):
+ os.makedirs(model_dir)
+
+ config_path = args.config
+ config_save_path = os.path.join(model_dir, "config.json")
+ if init:
+ with open(config_path, "r") as f:
+ data = f.read()
+ with open(config_save_path, "w") as f:
+ f.write(data)
+ else:
+ with open(config_save_path, "r") as f:
+ data = f.read()
+ config = json.loads(data)
+
+ hparams = HParams(**config)
+ hparams.model_dir = model_dir
+ return hparams
+
+
+def get_hparams_from_dir(model_dir):
+ config_save_path = os.path.join(model_dir, "config.json")
+ with open(config_save_path, "r") as f:
+ data = f.read()
+ config = json.loads(data)
+
+ hparams = HParams(**config)
+ hparams.model_dir = model_dir
+ return hparams
+
+
+def get_hparams_from_file(config_path):
+ with open(config_path, "r") as f:
+ data = f.read()
+ config = json.loads(data)
+
+ hparams = HParams(**config)
+
+ logger.info("Loaded config '{}' (config: {})".format(
+ config_path, hparams))
+ return hparams
+
+
+def check_git_hash(model_dir):
+ source_dir = os.path.dirname(os.path.realpath(__file__))
+ if not os.path.exists(os.path.join(source_dir, ".git")):
+ logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format(
+ source_dir
+ ))
+ return
+
+ cur_hash = subprocess.getoutput("git rev-parse HEAD")
+
+ path = os.path.join(model_dir, "githash")
+ if os.path.exists(path):
+ saved_hash = open(path).read()
+ if saved_hash != cur_hash:
+ logger.warn("git hash values are different. {}(saved) != {}(current)".format(
+ saved_hash[:8], cur_hash[:8]))
+ else:
+ open(path, "w").write(cur_hash)
+
+
+def get_logger(model_dir, filename="train.log"):
+ global logger
+ logger = logging.getLogger(os.path.basename(model_dir))
+ logger.setLevel(logging.DEBUG)
+
+ formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
+ if not os.path.exists(model_dir):
+ os.makedirs(model_dir)
+ h = logging.FileHandler(os.path.join(model_dir, filename))
+ h.setLevel(logging.DEBUG)
+ h.setFormatter(formatter)
+ logger.addHandler(h)
+ return logger
+
+
+def repeat_expand_2d(content, target_len):
+ # content : [h, t]
+
+ src_len = content.shape[-1]
+ target = torch.zeros([content.shape[0], target_len], dtype=torch.float).to(content.device)
+ temp = torch.arange(src_len + 1) * target_len / src_len
+ current_pos = 0
+ for i in range(target_len):
+ if i < temp[current_pos + 1]:
+ target[:, i] = content[:, current_pos]
+ else:
+ current_pos += 1
+ target[:, i] = content[:, current_pos]
+
+ return target
+
+
+class HParams():
+ def __init__(self, **kwargs):
+ for k, v in kwargs.items():
+ if type(v) == dict:
+ v = HParams(**v)
+ self[k] = v
+
+ def keys(self):
+ return self.__dict__.keys()
+
+ def items(self):
+ return self.__dict__.items()
+
+ def values(self):
+ return self.__dict__.values()
+
+ def __len__(self):
+ return len(self.__dict__)
+
+ def __getitem__(self, key):
+ return getattr(self, key)
+
+ def __setitem__(self, key, value):
+ return setattr(self, key, value)
+
+ def __contains__(self, key):
+ return key in self.__dict__
+
+ def __repr__(self):
+ return self.__dict__.__repr__()
diff --git a/vdecoder/__init__.py b/vdecoder/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/vdecoder/hifigan/env.py b/vdecoder/hifigan/env.py
new file mode 100644
index 0000000000000000000000000000000000000000..2bdbc95d4f7a8bad8fd4f5eef657e2b51d946056
--- /dev/null
+++ b/vdecoder/hifigan/env.py
@@ -0,0 +1,15 @@
+import os
+import shutil
+
+
+class AttrDict(dict):
+ def __init__(self, *args, **kwargs):
+ super(AttrDict, self).__init__(*args, **kwargs)
+ self.__dict__ = self
+
+
+def build_env(config, config_name, path):
+ t_path = os.path.join(path, config_name)
+ if config != t_path:
+ os.makedirs(path, exist_ok=True)
+ shutil.copyfile(config, os.path.join(path, config_name))
diff --git a/vdecoder/hifigan/models.py b/vdecoder/hifigan/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..9747301f350bb269e62601017fe4633ce271b27e
--- /dev/null
+++ b/vdecoder/hifigan/models.py
@@ -0,0 +1,503 @@
+import os
+import json
+from .env import AttrDict
+import numpy as np
+import torch
+import torch.nn.functional as F
+import torch.nn as nn
+from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
+from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
+from .utils import init_weights, get_padding
+
+LRELU_SLOPE = 0.1
+
+
+def load_model(model_path, device='cuda'):
+ config_file = os.path.join(os.path.split(model_path)[0], 'config.json')
+ with open(config_file) as f:
+ data = f.read()
+
+ global h
+ json_config = json.loads(data)
+ h = AttrDict(json_config)
+
+ generator = Generator(h).to(device)
+
+ cp_dict = torch.load(model_path)
+ generator.load_state_dict(cp_dict['generator'])
+ generator.eval()
+ generator.remove_weight_norm()
+ del cp_dict
+ return generator, h
+
+
+class ResBlock1(torch.nn.Module):
+ def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5)):
+ super(ResBlock1, self).__init__()
+ self.h = h
+ self.convs1 = nn.ModuleList([
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+ padding=get_padding(kernel_size, dilation[0]))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+ padding=get_padding(kernel_size, dilation[1]))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
+ padding=get_padding(kernel_size, dilation[2])))
+ ])
+ self.convs1.apply(init_weights)
+
+ self.convs2 = nn.ModuleList([
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+ padding=get_padding(kernel_size, 1))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+ padding=get_padding(kernel_size, 1))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
+ padding=get_padding(kernel_size, 1)))
+ ])
+ self.convs2.apply(init_weights)
+
+ def forward(self, x):
+ for c1, c2 in zip(self.convs1, self.convs2):
+ xt = F.leaky_relu(x, LRELU_SLOPE)
+ xt = c1(xt)
+ xt = F.leaky_relu(xt, LRELU_SLOPE)
+ xt = c2(xt)
+ x = xt + x
+ return x
+
+ def remove_weight_norm(self):
+ for l in self.convs1:
+ remove_weight_norm(l)
+ for l in self.convs2:
+ remove_weight_norm(l)
+
+
+class ResBlock2(torch.nn.Module):
+ def __init__(self, h, channels, kernel_size=3, dilation=(1, 3)):
+ super(ResBlock2, self).__init__()
+ self.h = h
+ self.convs = nn.ModuleList([
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
+ padding=get_padding(kernel_size, dilation[0]))),
+ weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
+ padding=get_padding(kernel_size, dilation[1])))
+ ])
+ self.convs.apply(init_weights)
+
+ def forward(self, x):
+ for c in self.convs:
+ xt = F.leaky_relu(x, LRELU_SLOPE)
+ xt = c(xt)
+ x = xt + x
+ return x
+
+ def remove_weight_norm(self):
+ for l in self.convs:
+ remove_weight_norm(l)
+
+
+def padDiff(x):
+ return F.pad(F.pad(x, (0,0,-1,1), 'constant', 0) - x, (0,0,0,-1), 'constant', 0)
+
+class SineGen(torch.nn.Module):
+ """ Definition of sine generator
+ SineGen(samp_rate, harmonic_num = 0,
+ sine_amp = 0.1, noise_std = 0.003,
+ voiced_threshold = 0,
+ flag_for_pulse=False)
+ samp_rate: sampling rate in Hz
+ harmonic_num: number of harmonic overtones (default 0)
+ sine_amp: amplitude of sine-wavefrom (default 0.1)
+ noise_std: std of Gaussian noise (default 0.003)
+ voiced_thoreshold: F0 threshold for U/V classification (default 0)
+ flag_for_pulse: this SinGen is used inside PulseGen (default False)
+ Note: when flag_for_pulse is True, the first time step of a voiced
+ segment is always sin(np.pi) or cos(0)
+ """
+
+ def __init__(self, samp_rate, harmonic_num=0,
+ sine_amp=0.1, noise_std=0.003,
+ voiced_threshold=0,
+ flag_for_pulse=False):
+ super(SineGen, self).__init__()
+ self.sine_amp = sine_amp
+ self.noise_std = noise_std
+ self.harmonic_num = harmonic_num
+ self.dim = self.harmonic_num + 1
+ self.sampling_rate = samp_rate
+ self.voiced_threshold = voiced_threshold
+ self.flag_for_pulse = flag_for_pulse
+
+ def _f02uv(self, f0):
+ # generate uv signal
+ uv = (f0 > self.voiced_threshold).type(torch.float32)
+ return uv
+
+ def _f02sine(self, f0_values):
+ """ f0_values: (batchsize, length, dim)
+ where dim indicates fundamental tone and overtones
+ """
+ # convert to F0 in rad. The interger part n can be ignored
+ # because 2 * np.pi * n doesn't affect phase
+ rad_values = (f0_values / self.sampling_rate) % 1
+
+ # initial phase noise (no noise for fundamental component)
+ rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], \
+ device=f0_values.device)
+ rand_ini[:, 0] = 0
+ rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
+
+ # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad)
+ if not self.flag_for_pulse:
+ # for normal case
+
+ # To prevent torch.cumsum numerical overflow,
+ # it is necessary to add -1 whenever \sum_k=1^n rad_value_k > 1.
+ # Buffer tmp_over_one_idx indicates the time step to add -1.
+ # This will not change F0 of sine because (x-1) * 2*pi = x * 2*pi
+ tmp_over_one = torch.cumsum(rad_values, 1) % 1
+ tmp_over_one_idx = (padDiff(tmp_over_one)) < 0
+ cumsum_shift = torch.zeros_like(rad_values)
+ cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0
+
+ sines = torch.sin(torch.cumsum(rad_values + cumsum_shift, dim=1)
+ * 2 * np.pi)
+ else:
+ # If necessary, make sure that the first time step of every
+ # voiced segments is sin(pi) or cos(0)
+ # This is used for pulse-train generation
+
+ # identify the last time step in unvoiced segments
+ uv = self._f02uv(f0_values)
+ uv_1 = torch.roll(uv, shifts=-1, dims=1)
+ uv_1[:, -1, :] = 1
+ u_loc = (uv < 1) * (uv_1 > 0)
+
+ # get the instantanouse phase
+ tmp_cumsum = torch.cumsum(rad_values, dim=1)
+ # different batch needs to be processed differently
+ for idx in range(f0_values.shape[0]):
+ temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :]
+ temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :]
+ # stores the accumulation of i.phase within
+ # each voiced segments
+ tmp_cumsum[idx, :, :] = 0
+ tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum
+
+ # rad_values - tmp_cumsum: remove the accumulation of i.phase
+ # within the previous voiced segment.
+ i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1)
+
+ # get the sines
+ sines = torch.cos(i_phase * 2 * np.pi)
+ return sines
+
+ def forward(self, f0):
+ """ sine_tensor, uv = forward(f0)
+ input F0: tensor(batchsize=1, length, dim=1)
+ f0 for unvoiced steps should be 0
+ output sine_tensor: tensor(batchsize=1, length, dim)
+ output uv: tensor(batchsize=1, length, 1)
+ """
+ with torch.no_grad():
+ f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim,
+ device=f0.device)
+ # fundamental component
+ fn = torch.multiply(f0, torch.FloatTensor([[range(1, self.harmonic_num + 2)]]).to(f0.device))
+
+ # generate sine waveforms
+ sine_waves = self._f02sine(fn) * self.sine_amp
+
+ # generate uv signal
+ # uv = torch.ones(f0.shape)
+ # uv = uv * (f0 > self.voiced_threshold)
+ uv = self._f02uv(f0)
+
+ # noise: for unvoiced should be similar to sine_amp
+ # std = self.sine_amp/3 -> max value ~ self.sine_amp
+ # . for voiced regions is self.noise_std
+ noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3
+ noise = noise_amp * torch.randn_like(sine_waves)
+
+ # first: set the unvoiced part to 0 by uv
+ # then: additive noise
+ sine_waves = sine_waves * uv + noise
+ return sine_waves, uv, noise
+
+
+class SourceModuleHnNSF(torch.nn.Module):
+ """ SourceModule for hn-nsf
+ SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1,
+ add_noise_std=0.003, voiced_threshod=0)
+ sampling_rate: sampling_rate in Hz
+ harmonic_num: number of harmonic above F0 (default: 0)
+ sine_amp: amplitude of sine source signal (default: 0.1)
+ add_noise_std: std of additive Gaussian noise (default: 0.003)
+ note that amplitude of noise in unvoiced is decided
+ by sine_amp
+ voiced_threshold: threhold to set U/V given F0 (default: 0)
+ Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
+ F0_sampled (batchsize, length, 1)
+ Sine_source (batchsize, length, 1)
+ noise_source (batchsize, length 1)
+ uv (batchsize, length, 1)
+ """
+
+ def __init__(self, sampling_rate, harmonic_num=0, sine_amp=0.1,
+ add_noise_std=0.003, voiced_threshod=0):
+ super(SourceModuleHnNSF, self).__init__()
+
+ self.sine_amp = sine_amp
+ self.noise_std = add_noise_std
+
+ # to produce sine waveforms
+ self.l_sin_gen = SineGen(sampling_rate, harmonic_num,
+ sine_amp, add_noise_std, voiced_threshod)
+
+ # to merge source harmonics into a single excitation
+ self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)
+ self.l_tanh = torch.nn.Tanh()
+
+ def forward(self, x):
+ """
+ Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
+ F0_sampled (batchsize, length, 1)
+ Sine_source (batchsize, length, 1)
+ noise_source (batchsize, length 1)
+ """
+ # source for harmonic branch
+ sine_wavs, uv, _ = self.l_sin_gen(x)
+ sine_merge = self.l_tanh(self.l_linear(sine_wavs))
+
+ # source for noise branch, in the same shape as uv
+ noise = torch.randn_like(uv) * self.sine_amp / 3
+ return sine_merge, noise, uv
+
+
+class Generator(torch.nn.Module):
+ def __init__(self, h):
+ super(Generator, self).__init__()
+ self.h = h
+
+ self.num_kernels = len(h["resblock_kernel_sizes"])
+ self.num_upsamples = len(h["upsample_rates"])
+ self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(h["upsample_rates"]))
+ self.m_source = SourceModuleHnNSF(
+ sampling_rate=h["sampling_rate"],
+ harmonic_num=8)
+ self.noise_convs = nn.ModuleList()
+ self.conv_pre = weight_norm(Conv1d(h["inter_channels"], h["upsample_initial_channel"], 7, 1, padding=3))
+ resblock = ResBlock1 if h["resblock"] == '1' else ResBlock2
+ self.ups = nn.ModuleList()
+ for i, (u, k) in enumerate(zip(h["upsample_rates"], h["upsample_kernel_sizes"])):
+ c_cur = h["upsample_initial_channel"] // (2 ** (i + 1))
+ self.ups.append(weight_norm(
+ ConvTranspose1d(h["upsample_initial_channel"] // (2 ** i), h["upsample_initial_channel"] // (2 ** (i + 1)),
+ k, u, padding=(k - u) // 2)))
+ if i + 1 < len(h["upsample_rates"]): #
+ stride_f0 = np.prod(h["upsample_rates"][i + 1:])
+ self.noise_convs.append(Conv1d(
+ 1, c_cur, kernel_size=stride_f0 * 2, stride=stride_f0, padding=stride_f0 // 2))
+ else:
+ self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))
+ self.resblocks = nn.ModuleList()
+ for i in range(len(self.ups)):
+ ch = h["upsample_initial_channel"] // (2 ** (i + 1))
+ for j, (k, d) in enumerate(zip(h["resblock_kernel_sizes"], h["resblock_dilation_sizes"])):
+ self.resblocks.append(resblock(h, ch, k, d))
+
+ self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3))
+ self.ups.apply(init_weights)
+ self.conv_post.apply(init_weights)
+ self.cond = nn.Conv1d(h['gin_channels'], h['upsample_initial_channel'], 1)
+
+ def forward(self, x, f0, g=None):
+ # print(1,x.shape,f0.shape,f0[:, None].shape)
+ f0 = self.f0_upsamp(f0[:, None]).transpose(1, 2) # bs,n,t
+ # print(2,f0.shape)
+ har_source, noi_source, uv = self.m_source(f0)
+ har_source = har_source.transpose(1, 2)
+ x = self.conv_pre(x)
+ x = x + self.cond(g)
+ # print(124,x.shape,har_source.shape)
+ for i in range(self.num_upsamples):
+ x = F.leaky_relu(x, LRELU_SLOPE)
+ # print(3,x.shape)
+ x = self.ups[i](x)
+ x_source = self.noise_convs[i](har_source)
+ # print(4,x_source.shape,har_source.shape,x.shape)
+ x = x + x_source
+ xs = None
+ for j in range(self.num_kernels):
+ if xs is None:
+ xs = self.resblocks[i * self.num_kernels + j](x)
+ else:
+ xs += self.resblocks[i * self.num_kernels + j](x)
+ x = xs / self.num_kernels
+ x = F.leaky_relu(x)
+ x = self.conv_post(x)
+ x = torch.tanh(x)
+
+ return x
+
+ def remove_weight_norm(self):
+ print('Removing weight norm...')
+ for l in self.ups:
+ remove_weight_norm(l)
+ for l in self.resblocks:
+ l.remove_weight_norm()
+ remove_weight_norm(self.conv_pre)
+ remove_weight_norm(self.conv_post)
+
+
+class DiscriminatorP(torch.nn.Module):
+ def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+ super(DiscriminatorP, self).__init__()
+ self.period = period
+ norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+ self.convs = nn.ModuleList([
+ norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+ norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+ norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+ norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(5, 1), 0))),
+ norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(2, 0))),
+ ])
+ self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+ def forward(self, x):
+ fmap = []
+
+ # 1d to 2d
+ b, c, t = x.shape
+ if t % self.period != 0: # pad first
+ n_pad = self.period - (t % self.period)
+ x = F.pad(x, (0, n_pad), "reflect")
+ t = t + n_pad
+ x = x.view(b, c, t // self.period, self.period)
+
+ for l in self.convs:
+ x = l(x)
+ x = F.leaky_relu(x, LRELU_SLOPE)
+ fmap.append(x)
+ x = self.conv_post(x)
+ fmap.append(x)
+ x = torch.flatten(x, 1, -1)
+
+ return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+ def __init__(self, periods=None):
+ super(MultiPeriodDiscriminator, self).__init__()
+ self.periods = periods if periods is not None else [2, 3, 5, 7, 11]
+ self.discriminators = nn.ModuleList()
+ for period in self.periods:
+ self.discriminators.append(DiscriminatorP(period))
+
+ def forward(self, y, y_hat):
+ y_d_rs = []
+ y_d_gs = []
+ fmap_rs = []
+ fmap_gs = []
+ for i, d in enumerate(self.discriminators):
+ y_d_r, fmap_r = d(y)
+ y_d_g, fmap_g = d(y_hat)
+ y_d_rs.append(y_d_r)
+ fmap_rs.append(fmap_r)
+ y_d_gs.append(y_d_g)
+ fmap_gs.append(fmap_g)
+
+ return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+class DiscriminatorS(torch.nn.Module):
+ def __init__(self, use_spectral_norm=False):
+ super(DiscriminatorS, self).__init__()
+ norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+ self.convs = nn.ModuleList([
+ norm_f(Conv1d(1, 128, 15, 1, padding=7)),
+ norm_f(Conv1d(128, 128, 41, 2, groups=4, padding=20)),
+ norm_f(Conv1d(128, 256, 41, 2, groups=16, padding=20)),
+ norm_f(Conv1d(256, 512, 41, 4, groups=16, padding=20)),
+ norm_f(Conv1d(512, 1024, 41, 4, groups=16, padding=20)),
+ norm_f(Conv1d(1024, 1024, 41, 1, groups=16, padding=20)),
+ norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
+ ])
+ self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
+
+ def forward(self, x):
+ fmap = []
+ for l in self.convs:
+ x = l(x)
+ x = F.leaky_relu(x, LRELU_SLOPE)
+ fmap.append(x)
+ x = self.conv_post(x)
+ fmap.append(x)
+ x = torch.flatten(x, 1, -1)
+
+ return x, fmap
+
+
+class MultiScaleDiscriminator(torch.nn.Module):
+ def __init__(self):
+ super(MultiScaleDiscriminator, self).__init__()
+ self.discriminators = nn.ModuleList([
+ DiscriminatorS(use_spectral_norm=True),
+ DiscriminatorS(),
+ DiscriminatorS(),
+ ])
+ self.meanpools = nn.ModuleList([
+ AvgPool1d(4, 2, padding=2),
+ AvgPool1d(4, 2, padding=2)
+ ])
+
+ def forward(self, y, y_hat):
+ y_d_rs = []
+ y_d_gs = []
+ fmap_rs = []
+ fmap_gs = []
+ for i, d in enumerate(self.discriminators):
+ if i != 0:
+ y = self.meanpools[i - 1](y)
+ y_hat = self.meanpools[i - 1](y_hat)
+ y_d_r, fmap_r = d(y)
+ y_d_g, fmap_g = d(y_hat)
+ y_d_rs.append(y_d_r)
+ fmap_rs.append(fmap_r)
+ y_d_gs.append(y_d_g)
+ fmap_gs.append(fmap_g)
+
+ return y_d_rs, y_d_gs, fmap_rs, fmap_gs
+
+
+def feature_loss(fmap_r, fmap_g):
+ loss = 0
+ for dr, dg in zip(fmap_r, fmap_g):
+ for rl, gl in zip(dr, dg):
+ loss += torch.mean(torch.abs(rl - gl))
+
+ return loss * 2
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+ loss = 0
+ r_losses = []
+ g_losses = []
+ for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+ r_loss = torch.mean((1 - dr) ** 2)
+ g_loss = torch.mean(dg ** 2)
+ loss += (r_loss + g_loss)
+ r_losses.append(r_loss.item())
+ g_losses.append(g_loss.item())
+
+ return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+ loss = 0
+ gen_losses = []
+ for dg in disc_outputs:
+ l = torch.mean((1 - dg) ** 2)
+ gen_losses.append(l)
+ loss += l
+
+ return loss, gen_losses
diff --git a/vdecoder/hifigan/nvSTFT.py b/vdecoder/hifigan/nvSTFT.py
new file mode 100644
index 0000000000000000000000000000000000000000..9942c3a7f49e6e0ae37d6d47790e4b44283f1a29
--- /dev/null
+++ b/vdecoder/hifigan/nvSTFT.py
@@ -0,0 +1,124 @@
+import math
+import os
+
+os.environ["LRU_CACHE_CAPACITY"] = "3"
+import random
+import torch
+import torch.utils.data
+import numpy as np
+import librosa
+from librosa.util import normalize
+from librosa.filters import mel as librosa_mel_fn
+from scipy.io.wavfile import read
+import soundfile as sf
+
+
+def load_wav_to_torch(full_path, target_sr=None, return_empty_on_exception=False):
+ sampling_rate = None
+ try:
+ data, sampling_rate = sf.read(full_path, always_2d=True) # than soundfile.
+ except Exception as ex:
+ print(f"'{full_path}' failed to load.\nException:")
+ print(ex)
+ if return_empty_on_exception:
+ return [], sampling_rate or target_sr or 32000
+ else:
+ raise Exception(ex)
+
+ if len(data.shape) > 1:
+ data = data[:, 0]
+ assert len(
+ data) > 2 # check duration of audio file is > 2 samples (because otherwise the slice operation was on the wrong dimension)
+
+ if np.issubdtype(data.dtype, np.integer): # if audio data is type int
+ max_mag = -np.iinfo(data.dtype).min # maximum magnitude = min possible value of intXX
+ else: # if audio data is type fp32
+ max_mag = max(np.amax(data), -np.amin(data))
+ max_mag = (2 ** 31) + 1 if max_mag > (2 ** 15) else ((
+ 2 ** 15) + 1 if max_mag > 1.01 else 1.0) # data should be either 16-bit INT, 32-bit INT or [-1 to 1] float32
+
+ data = torch.FloatTensor(data.astype(np.float32)) / max_mag
+
+ if (torch.isinf(data) | torch.isnan(
+ data)).any() and return_empty_on_exception: # resample will crash with inf/NaN inputs. return_empty_on_exception will return empty arr instead of except
+ return [], sampling_rate or target_sr or 32000
+ if target_sr is not None and sampling_rate != target_sr:
+ data = torch.from_numpy(librosa.core.resample(data.numpy(), orig_sr=sampling_rate, target_sr=target_sr))
+ sampling_rate = target_sr
+
+ return data, sampling_rate
+
+
+def dynamic_range_compression(x, C=1, clip_val=1e-5):
+ return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
+
+
+def dynamic_range_decompression(x, C=1):
+ return np.exp(x) / C
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+ return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+ return torch.exp(x) / C
+
+
+class STFT():
+ def __init__(self, sr=22050, n_mels=80, n_fft=1024, win_size=1024, hop_length=256, fmin=20, fmax=11025,
+ clip_val=1e-5):
+ self.target_sr = sr
+
+ self.n_mels = n_mels
+ self.n_fft = n_fft
+ self.win_size = win_size
+ self.hop_length = hop_length
+ self.fmin = fmin
+ self.fmax = fmax
+ self.clip_val = clip_val
+ self.mel_basis = {}
+ self.hann_window = {}
+
+ def get_mel(self, y, center=False):
+ sampling_rate = self.target_sr
+ n_mels = self.n_mels
+ n_fft = self.n_fft
+ win_size = self.win_size
+ hop_length = self.hop_length
+ fmin = self.fmin
+ fmax = self.fmax
+ clip_val = self.clip_val
+
+ if torch.min(y) < -1.:
+ print('min value is ', torch.min(y))
+ if torch.max(y) > 1.:
+ print('max value is ', torch.max(y))
+
+ if fmax not in self.mel_basis:
+ mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax)
+ self.mel_basis[str(fmax) + '_' + str(y.device)] = torch.from_numpy(mel).float().to(y.device)
+ self.hann_window[str(y.device)] = torch.hann_window(self.win_size).to(y.device)
+
+ y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft - hop_length) / 2), int((n_fft - hop_length) / 2)),
+ mode='reflect')
+ y = y.squeeze(1)
+
+ spec = torch.stft(y, n_fft, hop_length=hop_length, win_length=win_size, window=self.hann_window[str(y.device)],
+ center=center, pad_mode='reflect', normalized=False, onesided=True)
+ # print(111,spec)
+ spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
+ # print(222,spec)
+ spec = torch.matmul(self.mel_basis[str(fmax) + '_' + str(y.device)], spec)
+ # print(333,spec)
+ spec = dynamic_range_compression_torch(spec, clip_val=clip_val)
+ # print(444,spec)
+ return spec
+
+ def __call__(self, audiopath):
+ audio, sr = load_wav_to_torch(audiopath, target_sr=self.target_sr)
+ spect = self.get_mel(audio.unsqueeze(0)).squeeze(0)
+ return spect
+
+
+stft = STFT()
diff --git a/vdecoder/hifigan/utils.py b/vdecoder/hifigan/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..d62a40484718d504e2ef4d43972482c7975aa113
--- /dev/null
+++ b/vdecoder/hifigan/utils.py
@@ -0,0 +1,67 @@
+import glob
+import os
+import matplotlib
+import torch
+from torch.nn.utils import weight_norm
+matplotlib.use("Agg")
+import matplotlib.pylab as plt
+
+
+def plot_spectrogram(spectrogram):
+ fig, ax = plt.subplots(figsize=(10, 2))
+ im = ax.imshow(spectrogram, aspect="auto", origin="lower",
+ interpolation='none')
+ plt.colorbar(im, ax=ax)
+
+ fig.canvas.draw()
+ plt.close()
+
+ return fig
+
+
+def init_weights(m, mean=0.0, std=0.01):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ m.weight.data.normal_(mean, std)
+
+
+def apply_weight_norm(m):
+ classname = m.__class__.__name__
+ if classname.find("Conv") != -1:
+ weight_norm(m)
+
+
+def get_padding(kernel_size, dilation=1):
+ return int((kernel_size*dilation - dilation)/2)
+
+
+def load_checkpoint(filepath, device):
+ assert os.path.isfile(filepath)
+ print("Loading '{}'".format(filepath))
+ checkpoint_dict = torch.load(filepath, map_location=device)
+ print("Complete.")
+ return checkpoint_dict
+
+
+def save_checkpoint(filepath, obj):
+ print("Saving checkpoint to {}".format(filepath))
+ torch.save(obj, filepath)
+ print("Complete.")
+
+
+def del_old_checkpoints(cp_dir, prefix, n_models=2):
+ pattern = os.path.join(cp_dir, prefix + '????????')
+ cp_list = glob.glob(pattern) # get checkpoint paths
+ cp_list = sorted(cp_list)# sort by iter
+ if len(cp_list) > n_models: # if more than n_models models are found
+ for cp in cp_list[:-n_models]:# delete the oldest models other than lastest n_models
+ open(cp, 'w').close()# empty file contents
+ os.unlink(cp)# delete file (move to trash when using Colab)
+
+
+def scan_checkpoint(cp_dir, prefix):
+ pattern = os.path.join(cp_dir, prefix + '????????')
+ cp_list = glob.glob(pattern)
+ if len(cp_list) == 0:
+ return None
+ return sorted(cp_list)[-1]
diff --git a/vextract/__init__.py b/vextract/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/vextract/vocal_extract.py b/vextract/vocal_extract.py
new file mode 100644
index 0000000000000000000000000000000000000000..125782437e381cf70d1b92b432b75ca3b8adb29e
--- /dev/null
+++ b/vextract/vocal_extract.py
@@ -0,0 +1,95 @@
+import io
+import os
+import subprocess
+import tempfile
+import wave
+
+import librosa
+import numpy as np
+import soundfile
+from scipy.io import wavfile
+from scipy.io.wavfile import read
+
+
+class VEX(object):
+ def __init__(self):
+ pass
+
+ def load(self, audio_file):
+ with wave.open(audio_file, 'rb') as wav_file:
+ num_frames = wav_file.getnframes()
+ audiofile_body = wav_file.readframes(num_frames)
+ with io.BytesIO(audiofile_body) as file_stream:
+ with wave.open(file_stream, 'rb') as wave_file:
+ audio_data = wave_file.readframes(-1)
+ sampling_rate = wave_file.getframerate()
+ num_channels = wave_file.getnchannels()
+
+ audio_array = np.frombuffer(audio_data, dtype=np.int16)
+ audio_array = np.reshape(audio_array, (-1, num_channels))
+ return sampling_rate, audio_array
+
+ def separate(self, srcaudio=None):
+ import uuid
+ sampling_rate, audio = srcaudio
+
+ # Make sure the NumPy array has an integer data type
+ if not np.issubdtype(audio.dtype, np.integer):
+ raise ValueError("The input NumPy array must have an integer data type.")
+
+ temp_filebasename = f"splt_{uuid.uuid4()}"
+ temp_filename = f"{temp_filebasename}.wav"
+ wavfile.write(temp_filename, sampling_rate, audio)
+
+ subprocess.run([
+ 'spleeter', 'separate',
+ '-p', 'spleeter:2stems',
+ '-o', 'output',
+ temp_filename
+ ])
+
+ temp_dir = f"output/{temp_filebasename}"
+ vocal_file = f"output/{temp_filebasename}/vocals.wav"
+ accompaniment_file = f"output/{temp_filebasename}/accompaniment.wav"
+
+ vocal_sampling_rate, vocal_audio = wavfile.read(vocal_file)
+
+ # with wave.open(vocal_file, 'rb') as wav_file:
+ # num_frames = wav_file.getnframes()
+ # audiofile_body = wav_file.readframes(num_frames)
+ # with io.BytesIO(audiofile_body) as file_stream:
+ # with wave.open(file_stream, 'rb') as wave_file:
+ # audio_data = wave_file.readframes(-1)
+ # vocal_sampling_rate = wave_file.getframerate()
+ # num_channels = wave_file.getnchannels()
+ #
+ # vocal_audio = np.frombuffer(audio_data, dtype=np.int16)
+ # vocal_audio = np.reshape(vocal_audio, (-1, num_channels))
+
+ accompaniment_sampling_rate, accompaniment_audio = wavfile.read(accompaniment_file)
+
+ # with wave.open(accompaniment_file, 'rb') as wav_file:
+ # num_frames = wav_file.getnframes()
+ # audiofile_body = wav_file.readframes(num_frames)
+ # with io.BytesIO(audiofile_body) as file_stream:
+ # with wave.open(file_stream, 'rb') as wave_file:
+ # audio_data = wave_file.readframes(-1)
+ # accompaniment_sampling_rate = wave_file.getframerate()
+ # num_channels = wave_file.getnchannels()
+ #
+ # accompaniment_audio = np.frombuffer(audio_data, dtype=np.int16)
+ # accompaniment_audio = np.reshape(accompaniment_audio, (-1, num_channels))
+
+ os.remove(temp_filename)
+ os.remove(vocal_file)
+ os.remove(accompaniment_file)
+ os.rmdir(temp_dir)
+
+ return [(vocal_sampling_rate, vocal_audio), (accompaniment_sampling_rate, accompaniment_audio)]
+
+
+if __name__ == "__main__":
+ v = VEX()
+ # srcaudio = v.load("")
+ v.separate()
+
+
+
+
+
+
+ 单个文件处理需要提供音频文件，输出会是单个音频文件；
+ 批量处理支持上传多个音频文件，输出会是一个zip压缩包。 +
+
+
+
+
+
+
+ 运行设备，默认选择cuda，即GPU计算；如果没有支持的GPU设备可以选择cpu。
+
+
+
+
+
+
+
+ 选择一个语音模型
+
+
+
+
+
+
+ 加载模型
+
+
+
+ 基本设置
++ 批量处理支持上传多个音频文件，输出会是一个zip压缩包。 +
+
+
+
+ 选择音频文件并开始制作！
+
+
+
+
+
+
+
+ 制作一个歌曲。
+你使用的语音模型是 {{initForm.model}}
+
+ 需要是wav文件哦
+
+
+
+
+
+
+ 选择音频文件
+
+
+ 开始制作！
+
+
+
+
+
+
+ 制作多个歌曲。
+你使用的语音模型是 {{initForm.model}}
+
+ 需要是wav文件哦
+
+
+