fumiama/Retrieval-based-Voice-Conversion-WebUI
1
0
Fork 0
mirror of https://github.com/fumiama/Retrieval-based-Voice-Conversion-WebUI.git synced 2026-06-07 19:40:44 +08:00
Code Activity

feat(audio): use PyAV instead of ffmpeg (#31)

Browse Source 
* feat(audio): use PyAV instead of ffmpeg

replaced usage of ffmpeg in favor of PyAV (`av`)

* refactor(audio): store all of the audio related functions in the `infer.lib.audio`

refactors previous commit to have singular functions for each task, all located in `infer.lib.audio`

* fix(audio): remove downsample_audio from mdxnet.py

it is no longer needed, since it's imported from infer.lib.audio

* docs: remove every ffmpeg mention in the documentation to avoid confusion

* chore(requirements): remove ffmpeg-python and ffmpy from all requirements

* fix(audio): fix loading for UVR

wrapped gathering of META info from the stream into a function

fixes loading for UVR

* fix(audio): use np.frombuffer() instead of direct conversion of the resampled frames

this fixes traceback on preprocessing

* feat(audio): pre-allocate decoded_audio array in the load_audio function

this should improve performance, even if just a little

* Revert "docs: remove every ffmpeg mention in the documentation to avoid confusion"

This reverts commit 1e05bbce03.

* chore(format): run black on dev

* fix(requirements): revert removal of ffmpeg in unitest.yml and Dockerfile

* Revert "fix(requirements): revert removal of ffmpeg in unitest.yml and Dockerfile"

This reverts commit e28a0eebb2.

* feat(audio): pre-allocate numpy array to store the AudioFrame data in ndarray of dtype float32

* chore(format): run black on dev

* fix(audio): fix the decoded_audio size estimation

in estimated_total_samples we multiply by `sr` instead of `container.streams.audio[0].rate` since we want to estimate size of the OUTPUT file, not the input one. - Added dynamic resizing, in case something goes wrong and the size of decoded_audio is estimated incorrectly

Fixed function `load_audio` when the input audio's samplerate does not match the desired samplerate (`sr`)

* chore(format): run black on dev

* refactor(audio): remove `clean_path()` function as it serves no purpose anymore

* docs: remove everything related to ffmpeg

this includes everything except for formats support specification in the training_tips docs, since it has nothing to do with what ffmpeg does/did but rather what audio formats are supported (all the ones that ffmpeg supports!)

* docs: fix order of the steps in preparation in the READMEs

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Alex Murkoff
2024-06-12 18:13:26 +07:00
committed by GitHub
parent aec56ec0b4
commit 1e22d468ea
28 changed files with 233 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
infer/lib/audio.py
View File

@@ -1,9 +1,10 @@
from io import BufferedWriter, BytesIO
from pathlib import Path
from typing import Dict
import ffmpeg
from typing import Dict, Tuple
import numpy as np
import av
import os
from av.audio.resampler import AudioResampler
video_format_dict: Dict[str, str] = {
"m4a": "mp4",
@@ -39,20 +40,112 @@ def load_audio(file: str, sr: int) -> np.ndarray:
raise FileNotFoundError(f"File not found: {file}")
try:
# https://github.com/openai/whisper/blob/main/whisper/audio.py#L26
# This launches a subprocess to decode audio while down-mixing and resampling as necessary.
# Requires the ffmpeg CLI and `ffmpeg-python` package to be installed.
file = str(clean_path(file)) # 防止小白拷路径头尾带了空格和"和回车
out, _ = (
ffmpeg.input(file, threads=0)
.output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sr)
.run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True)
)
container = av.open(file)
resampler = AudioResampler(format="fltp", layout="mono", rate=sr)
# Estimated maximum total number of samples to pre-allocate the array
audio_duration_sec: float = (
container.duration / 1_000_000
) # AV stores length in microseconds by default
estimated_total_samples = int(audio_duration_sec * sr + 0.5)
decoded_audio = np.zeros(estimated_total_samples + 1, dtype=np.float32)
offset = 0
for frame in container.decode(audio=0):
frame.pts = None # Clear presentation timestamp to avoid resampling issues
resampled_frames = resampler.resample(frame)
for resampled_frame in resampled_frames:
frame_data = np.array(resampled_frame.to_ndarray()).flatten()
end_index = offset + len(frame_data)
# Check if decoded_audio has enough space, and resize if necessary
if end_index > decoded_audio.shape[0]:
decoded_audio = np.resize(decoded_audio, end_index + 1)
decoded_audio[offset:end_index] = frame_data
offset += len(frame_data)
# Truncate the array to the actual size
decoded_audio = decoded_audio[:offset]
except Exception as e:
raise RuntimeError(f"Failed to load audio: {e}")
return np.frombuffer(out, np.float32).flatten()
return decoded_audio
def clean_path(path: str) -> Path:
return Path(path.strip(' "\n')).resolve()
def downsample_audio(input_path: str, output_path: str, format: str) -> None:
if not os.path.exists(input_path):
return
input_container = av.open(input_path)
output_container = av.open(output_path, "w")
# Create a stream in the output container
input_stream = input_container.streams.audio[0]
output_stream = output_container.add_stream(format)
output_stream.bit_rate = 128_000 # 128kb/s (equivalent to -q:a 2)
# Copy packets from the input file to the output file
for packet in input_container.demux(input_stream):
for frame in packet.decode():
for out_packet in output_stream.encode(frame):
output_container.mux(out_packet)
for packet in output_stream.encode():
output_container.mux(packet)
# Close the containers
input_container.close()
output_container.close()
try: # Remove the original file
os.remove(input_path)
except Exception as e:
print(f"Failed to remove the original file: {e}")
def resample_audio(
input_path: str, output_path: str, codec: str, format: str, sr: int, layout: str
) -> None:
if not os.path.exists(input_path):
return
input_container = av.open(input_path)
output_container = av.open(output_path, "w")
# Create a stream in the output container
input_stream = input_container.streams.audio[0]
output_stream = output_container.add_stream(codec, rate=sr, layout=layout)
resampler = AudioResampler(format, layout, sr)
# Copy packets from the input file to the output file
for packet in input_container.demux(input_stream):
for frame in packet.decode():
frame.pts = None # Clear presentation timestamp to avoid resampling issues
out_frames = resampler.resample(frame)
for out_frame in out_frames:
for out_packet in output_stream.encode(out_frame):
output_container.mux(out_packet)
for packet in output_stream.encode():
output_container.mux(packet)
# Close the containers
input_container.close()
output_container.close()
try: # Remove the original file
os.remove(input_path)
except Exception as e:
print(f"Failed to remove the original file: {e}")
def get_audio_properties(input_path: str) -> Tuple:
container = av.open(input_path)
audio_stream = next(s for s in container.streams if s.type == "audio")
channels = 1 if audio_stream.layout == "mono" else 2
rate = audio_stream.base_rate
container.close()
return channels, rate

19
infer/modules/uvr5/mdxnet.py
View File

@@ -8,6 +8,9 @@ import numpy as np
import soundfile as sf
import torch
from tqdm import tqdm
import av
from infer.lib.audio import downsample_audio
cpu = torch.device("cpu")
@@ -218,20 +221,8 @@ class Predictor:
sf.write(path_other, opt, rate)
opt_path_vocal = path_vocal[:-4] + ".%s" % format
opt_path_other = path_other[:-4] + ".%s" % format
if os.path.exists(path_vocal):
os.system(f'ffmpeg -i "{path_vocal}" -vn "{opt_path_vocal}" -q:a 2 -y')
if os.path.exists(opt_path_vocal):
try:
os.remove(path_vocal)
except:
pass
if os.path.exists(path_other):
os.system(f'ffmpeg -i "{path_other}" -vn "{opt_path_other}" -q:a 2 -y')
if os.path.exists(opt_path_other):
try:
os.remove(path_other)
except:
pass
downsample_audio(path_vocal, opt_path_vocal, format)
downsample_audio(path_other, opt_path_other, format)
class MDXNetDereverb:

21
infer/modules/uvr5/modules.py
View File

@@ -4,7 +4,7 @@ import logging
logger = logging.getLogger(__name__)
import ffmpeg
from infer.lib.audio import resample_audio, get_audio_properties
import torch
from configs import Config
@@ -46,27 +46,24 @@ def uvr(model_name, inp_root, save_root_vocal, paths, save_root_ins, agg, format
need_reformat = 1
done = 0
try:
info = ffmpeg.probe(inp_path, cmd="ffprobe")
if (
info["streams"][0]["channels"] == 2
and info["streams"][0]["sample_rate"] == "44100"
):
need_reformat = 0
channels, rate = get_audio_properties(inp_path)
# Check the audio stream's properties
if channels == 2 and rate == 44100:
pre_fun._path_audio_(
inp_path, save_root_ins, save_root_vocal, format0, is_hp3=is_hp3
)
need_reformat = 0
done = 1
except:
except Exception as e:
need_reformat = 1
traceback.print_exc()
print(f"Exception {e} occured. Will reformat")
if need_reformat == 1:
tmp_path = "%s/%s.reformatted.wav" % (
os.path.join(os.environ["TEMP"]),
os.path.basename(inp_path),
)
os.system(
f'ffmpeg -i "{inp_path}" -vn -acodec pcm_s16le -ac 2 -ar 44100 "{tmp_path}" -y'
)
resample_audio(inp_path, tmp_path, "pcm_s16le", "s16", 44100, "stereo")
inp_path = tmp_path
try:
if done == 0:

36
infer/modules/uvr5/vr.py
View File

@@ -6,6 +6,7 @@ logger = logging.getLogger(__name__)
import librosa
import numpy as np
import soundfile as sf
from infer.lib.audio import downsample_audio
import torch
from infer.lib.uvr5_pack.lib_v5 import nets_123821KB as Nets
@@ -60,7 +61,7 @@ class AudioPre:
(
X_wave[d],
_,
) = librosa.core.load( # 理论上librosa读取可能对某些音频有bug，应该上ffmpeg读取，但是太麻烦了弃坑
) = librosa.core.load( # 理论上librosa读取可能对某些音频有bug，应该上av读取，但是太麻烦了弃坑
music_file,
bp["sr"],
False,
@@ -146,12 +147,7 @@ class AudioPre:
)
if os.path.exists(path):
opt_format_path = path[:-4] + ".%s" % format
os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
if os.path.exists(opt_format_path):
try:
os.remove(path)
except:
pass
downsample_audio(path, opt_format_path, format)
if vocal_root is not None:
if is_hp3 == True:
head = "instrument_"
@@ -185,14 +181,8 @@ class AudioPre:
(np.array(wav_vocals) * 32768).astype("int16"),
self.mp.param["sr"],
)
if os.path.exists(path):
opt_format_path = path[:-4] + ".%s" % format
os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
if os.path.exists(opt_format_path):
try:
os.remove(path)
except:
pass
opt_format_path = path[:-4] + ".%s" % format
downsample_audio(path, opt_format_path, format)
class AudioPreDeEcho:
@@ -241,7 +231,7 @@ class AudioPreDeEcho:
(
X_wave[d],
_,
) = librosa.core.load( # 理论上librosa读取可能对某些音频有bug，应该上ffmpeg读取，但是太麻烦了弃坑
) = librosa.core.load( # 理论上librosa读取可能对某些音频有bug，应该上av读取，但是太麻烦了弃坑
music_file,
bp["sr"],
False,
@@ -323,12 +313,7 @@ class AudioPreDeEcho:
)
if os.path.exists(path):
opt_format_path = path[:-4] + ".%s" % format
os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
if os.path.exists(opt_format_path):
try:
os.remove(path)
except:
pass
downsample_audio(path, opt_format_path, format)
if vocal_root is not None:
if self.data["high_end_process"].startswith("mirroring"):
input_high_end_ = spec_utils.mirroring(
@@ -360,9 +345,4 @@ class AudioPreDeEcho:
)
if os.path.exists(path):
opt_format_path = path[:-4] + ".%s" % format
os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
if os.path.exists(opt_format_path):
try:
os.remove(path)
except:
pass
downsample_audio(path, opt_format_path, format)