fumiama/Retrieval-based-Voice-Conversion-WebUI
1
0
Fork 0
mirror of https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI.git synced 2026-06-08 03:55:47 +08:00
Code Activity

optimize(rmvpe): move rmvpe into rvc.f0

Browse Source
This commit is contained in:
源文雨
2024-06-13 00:42:42 +09:00
parent 77b371d615
commit 8ac5597a3f
12 changed files with 96 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
infer/lib/rmvpe.py
View File

@@ -1,171 +0,0 @@
from io import BytesIO
import os
from typing import List, Optional, Tuple, Union
import numpy as np
import torch
from infer.lib import jit
import torch.nn.functional as F
import logging
logger = logging.getLogger(__name__)
from rvc.f0.mel import MelSpectrogram
from rvc.f0.e2e import E2E
class RMVPE:
def __init__(self, model_path: str, is_half, device=None, use_jit=False):
self.resample_kernel = {}
self.resample_kernel = {}
self.is_half = is_half
if device is None:
device = "cuda:0" if torch.cuda.is_available() else "cpu"
self.device = device
self.mel_extractor = MelSpectrogram(
is_half=is_half,
n_mel_channels=128,
sampling_rate=16000,
win_length=1024,
hop_length=160,
mel_fmin=30,
mel_fmax=8000,
device=device,
).to(device)
if "privateuseone" in str(device):
import onnxruntime as ort
ort_session = ort.InferenceSession(
"%s/rmvpe.onnx" % os.environ["rmvpe_root"],
providers=["DmlExecutionProvider"],
)
self.model = ort_session
else:
if str(self.device) == "cuda":
self.device = torch.device("cuda:0")
def get_jit_model():
jit_model_path = model_path.rstrip(".pth")
jit_model_path += ".half.jit" if is_half else ".jit"
ckpt = None
if os.path.exists(jit_model_path):
ckpt = jit.load(jit_model_path)
model_device = ckpt["device"]
if model_device != str(self.device):
del ckpt
ckpt = None
if ckpt is None:
ckpt = jit.rmvpe_jit_export(
model_path=model_path,
mode="script",
inputs_path=None,
save_path=jit_model_path,
device=device,
is_half=is_half,
)
model = torch.jit.load(BytesIO(ckpt["model"]), map_location=device)
return model
def get_default_model():
model = E2E(4, 1, (2, 2))
ckpt = torch.load(model_path, map_location="cpu")
model.load_state_dict(ckpt)
model.eval()
if is_half:
model = model.half()
else:
model = model.float()
return model
if use_jit:
if is_half and "cpu" in str(self.device):
logger.warning(
"Use default rmvpe model. \
Jit is not supported on the CPU for half floating point"
)
self.model = get_default_model()
else:
self.model = get_jit_model()
else:
self.model = get_default_model()
self.model = self.model.to(device)
cents_mapping = 20 * np.arange(360) + 1997.3794084376191
self.cents_mapping = np.pad(cents_mapping, (4, 4)) # 368
def mel2hidden(self, mel):
with torch.no_grad():
n_frames = mel.shape[-1]
n_pad = 32 * ((n_frames - 1) // 32 + 1) - n_frames
if n_pad > 0:
mel = F.pad(mel, (0, n_pad), mode="constant")
if "privateuseone" in str(self.device):
onnx_input_name = self.model.get_inputs()[0].name
onnx_outputs_names = self.model.get_outputs()[0].name
hidden = self.model.run(
[onnx_outputs_names],
input_feed={onnx_input_name: mel.cpu().numpy()},
)[0]
else:
mel = mel.half() if self.is_half else mel.float()
hidden = self.model(mel)
return hidden[:, :n_frames]
def decode(self, hidden, thred=0.03):
cents_pred = self.to_local_average_cents(hidden, threshold=thred)
f0 = 10 * (2 ** (cents_pred / 1200))
f0[f0 == 10] = 0
# f0 = np.array([10 * (2 ** (cent_pred / 1200)) if cent_pred else 0 for cent_pred in cents_pred])
return f0
def infer_from_audio(self, audio, threshold=0.03):
# torch.cuda.synchronize()
# t0 = ttime()
if not torch.is_tensor(audio):
audio = torch.from_numpy(audio)
mel = self.mel_extractor(
audio.float().to(self.device).unsqueeze(0), center=True
)
# print(123123123,mel.device.type)
# torch.cuda.synchronize()
# t1 = ttime()
hidden = self.mel2hidden(mel)
# torch.cuda.synchronize()
# t2 = ttime()
# print(234234,hidden.device.type)
if "privateuseone" not in str(self.device):
hidden = hidden.squeeze(0).cpu().numpy()
else:
hidden = hidden[0]
if self.is_half == True:
hidden = hidden.astype("float32")
f0 = self.decode(hidden, thred=threshold)
# torch.cuda.synchronize()
# t3 = ttime()
# print("hmvpe:%s\t%s\t%s\t%s"%(t1-t0,t2-t1,t3-t2,t3-t0))
return f0
def to_local_average_cents(self, salience, threshold=0.05):
center = np.argmax(salience, axis=1) # 帧长#index
salience = np.pad(salience, ((0, 0), (4, 4))) # 帧长,368
center += 4
todo_salience = []
todo_cents_mapping = []
starts = center - 4
ends = center + 5
for idx in range(salience.shape[0]):
todo_salience.append(salience[:, starts[idx] : ends[idx]][idx])
todo_cents_mapping.append(self.cents_mapping[starts[idx] : ends[idx]])
todo_salience = np.array(todo_salience) # 帧长，9
todo_cents_mapping = np.array(todo_cents_mapping) # 帧长，9
product_sum = np.sum(todo_salience * todo_cents_mapping, 1)
weight_sum = np.sum(todo_salience, 1) # 帧长
devided = product_sum / weight_sum # 帧长
maxx = np.max(salience, axis=1) # 帧长
devided[maxx <= threshold] = 0
return devided

4
infer/lib/rtrvc.py
View File

@@ -313,7 +313,7 @@ class RVC:
def get_f0_rmvpe(self, x, f0_up_key):
if hasattr(self, "model_rmvpe") == False:
from infer.lib.rmvpe import RMVPE
from rvc.f0 import RMVPE
printt("Loading rmvpe model")
self.model_rmvpe = RMVPE(
@@ -322,7 +322,7 @@ class RVC:
device=self.device,
use_jit=self.config.use_jit,
)
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
f0 = self.model_rmvpe.compute_f0(x, thred=0.03)
f0 *= pow(2, f0_up_key / 12)
return self.get_f0_post(f0)

4
infer/modules/train/extract/extract_f0_print.py
View File

@@ -83,13 +83,13 @@ class FeatureInput(object):
f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.fs)
elif f0_method == "rmvpe":
if hasattr(self, "model_rmvpe") == False:
from infer.lib.rmvpe import RMVPE
from rvc.f0.rmvpe import RMVPE
print("Loading rmvpe model")
self.model_rmvpe = RMVPE(
"assets/rmvpe/rmvpe.pt", is_half=False, device="cpu"
)
f0 = self.model_rmvpe.infer_from_audio(x, threshold=0.03)
f0 = self.model_rmvpe.compute_f0(x, filter_radius=0.03)
return f0
def coarse_f0(self, f0):

4
infer/modules/train/extract/extract_f0_rmvpe.py
View File

@@ -46,13 +46,13 @@ class FeatureInput(object):
# p_len = x.shape[0] // self.hop
if f0_method == "rmvpe":
if hasattr(self, "model_rmvpe") == False:
from infer.lib.rmvpe import RMVPE
from rvc.f0.rmvpe import RMVPE
print("Loading rmvpe model")
self.model_rmvpe = RMVPE(
"assets/rmvpe/rmvpe.pt", is_half=is_half, device="cuda"
)
f0 = self.model_rmvpe.infer_from_audio(x, threshold=0.03)
f0 = self.model_rmvpe.compute_f0(x, filter_radius=0.03)
return f0
def coarse_f0(self, f0):

4
infer/modules/train/extract/extract_f0_rmvpe_dml.py
View File

@@ -44,13 +44,13 @@ class FeatureInput(object):
# p_len = x.shape[0] // self.hop
if f0_method == "rmvpe":
if hasattr(self, "model_rmvpe") == False:
from infer.lib.rmvpe import RMVPE
from rvc.f0.rmvpe import RMVPE
print("Loading rmvpe model")
self.model_rmvpe = RMVPE(
"assets/rmvpe/rmvpe.pt", is_half=False, device=device
)
f0 = self.model_rmvpe.infer_from_audio(x, threshold=0.03)
f0 = self.model_rmvpe.compute_f0(x, filter_radius=0.03)
return f0
def coarse_f0(self, f0):

15
infer/modules/vc/pipeline.py
View File

@@ -16,7 +16,7 @@ import torch.nn.functional as F
import torchcrepe
from scipy import signal
from rvc.f0 import PM, Harvest
from rvc.f0 import PM, Harvest, RMVPE
now_dir = os.getcwd()
sys.path.append(now_dir)
@@ -108,24 +108,23 @@ class Pipeline(object):
f0[pd < 0.1] = 0
f0 = f0[0].cpu().numpy()
elif f0_method == "rmvpe":
if not hasattr(self, "model_rmvpe"):
from infer.lib.rmvpe import RMVPE
if not hasattr(self, "rmvpe"):
logger.info(
"Loading rmvpe model %s" % "%s/rmvpe.pt" % os.environ["rmvpe_root"]
)
self.model_rmvpe = RMVPE(
self.rmvpe = RMVPE(
"%s/rmvpe.pt" % os.environ["rmvpe_root"],
is_half=self.is_half,
device=self.device,
# use_jit=self.config.use_jit,
)
f0 = self.model_rmvpe.infer_from_audio(x, threshold=0.03)
f0 = self.rmvpe.compute_f0(x, filter_radius=0.03)
if "privateuseone" in str(self.device): # clean ortruntime memory
del self.model_rmvpe.model
del self.model_rmvpe
del self.rmvpe.model
del self.rmvpe
logger.info("Cleaning ortruntime memory")
elif f0_method == "fcpe":
if not hasattr(self, "model_fcpe"):
from torchfcpe import spawn_bundled_infer_model