feat(audio): use PyAV instead of ffmpeg (#31)

* feat(audio): use PyAV instead of ffmpeg

replaced usage of ffmpeg in favor of PyAV (`av`)

* refactor(audio): store all of the audio related functions in the `infer.lib.audio`

refactors previous commit to have singular functions for each task, all located in `infer.lib.audio`

* fix(audio): remove downsample_audio from mdxnet.py

it is no longer needed, since it's imported from infer.lib.audio

* docs: remove every ffmpeg mention in the documentation to avoid confusion

* chore(requirements): remove ffmpeg-python and ffmpy from all requirements

* fix(audio): fix loading for UVR

wrapped gathering of META info from the stream into a function

fixes loading for UVR

* fix(audio): use np.frombuffer() instead of direct conversion of the resampled frames

this fixes traceback on preprocessing

* feat(audio): pre-allocate decoded_audio array in the load_audio function

this should improve performance, even if just a little

* Revert "docs: remove every ffmpeg mention in the documentation to avoid confusion"

This reverts commit 1e05bbce03.

* chore(format): run black on dev

* fix(requirements): revert removal of ffmpeg in unitest.yml and Dockerfile

* Revert "fix(requirements): revert removal of ffmpeg in unitest.yml and Dockerfile"

This reverts commit e28a0eebb2.

* feat(audio): pre-allocate numpy array to store the AudioFrame data in ndarray of dtype float32

* chore(format): run black on dev

* fix(audio): fix the decoded_audio size estimation

in estimated_total_samples we multiply by `sr` instead of `container.streams.audio[0].rate` since we want to estimate size of the OUTPUT file, not the input one. - Added dynamic resizing, in case something goes wrong and the size of decoded_audio is estimated incorrectly

Fixed function `load_audio` when the input audio's samplerate does not match the desired samplerate (`sr`)

* chore(format): run black on dev

* refactor(audio): remove `clean_path()` function as it serves no purpose anymore

* docs: remove everything related to ffmpeg

this includes everything except for formats support specification in the training_tips docs, since it has nothing to do with what ffmpeg does/did but rather what audio formats are supported (all the ones that ffmpeg supports!)

* docs: fix order of the steps in preparation in the READMEs

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

infer/modules/uvr5/mdxnet.py

@@ -8,6 +8,9 @@ import numpy as np
 import soundfile as sf
 import torch
 from tqdm import tqdm
+import av
+
+from infer.lib.audio import downsample_audio
 
 cpu = torch.device("cpu")
 
@@ -218,20 +221,8 @@ class Predictor:
             sf.write(path_other, opt, rate)
             opt_path_vocal = path_vocal[:-4] + ".%s" % format
             opt_path_other = path_other[:-4] + ".%s" % format
-            if os.path.exists(path_vocal):
-                os.system(f'ffmpeg -i "{path_vocal}" -vn "{opt_path_vocal}" -q:a 2 -y')
-                if os.path.exists(opt_path_vocal):
-                    try:
-                        os.remove(path_vocal)
-                    except:
-                        pass
-            if os.path.exists(path_other):
-                os.system(f'ffmpeg -i "{path_other}" -vn "{opt_path_other}" -q:a 2 -y')
-                if os.path.exists(opt_path_other):
-                    try:
-                        os.remove(path_other)
-                    except:
-                        pass
+            downsample_audio(path_vocal, opt_path_vocal, format)
+            downsample_audio(path_other, opt_path_other, format)
 
 
 class MDXNetDereverb:

infer/modules/uvr5/modules.py

@@ -4,7 +4,7 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-import ffmpeg
+from infer.lib.audio import resample_audio, get_audio_properties
 import torch
 
 from configs import Config
@@ -46,27 +46,24 @@ def uvr(model_name, inp_root, save_root_vocal, paths, save_root_ins, agg, format
             need_reformat = 1
             done = 0
             try:
-                info = ffmpeg.probe(inp_path, cmd="ffprobe")
-                if (
-                    info["streams"][0]["channels"] == 2
-                    and info["streams"][0]["sample_rate"] == "44100"
-                ):
-                    need_reformat = 0
+                channels, rate = get_audio_properties(inp_path)
+
+                # Check the audio stream's properties
+                if channels == 2 and rate == 44100:
                     pre_fun._path_audio_(
                         inp_path, save_root_ins, save_root_vocal, format0, is_hp3=is_hp3
                     )
+                    need_reformat = 0
                     done = 1
-            except:
+            except Exception as e:
                 need_reformat = 1
-                traceback.print_exc()
+                print(f"Exception {e} occured. Will reformat")
             if need_reformat == 1:
                 tmp_path = "%s/%s.reformatted.wav" % (
                     os.path.join(os.environ["TEMP"]),
                     os.path.basename(inp_path),
                 )
-                os.system(
-                    f'ffmpeg -i "{inp_path}" -vn -acodec pcm_s16le -ac 2 -ar 44100 "{tmp_path}" -y'
-                )
+                resample_audio(inp_path, tmp_path, "pcm_s16le", "s16", 44100, "stereo")
                 inp_path = tmp_path
             try:
                 if done == 0:

infer/modules/uvr5/vr.py

@@ -6,6 +6,7 @@ logger = logging.getLogger(__name__)
 import librosa
 import numpy as np
 import soundfile as sf
+from infer.lib.audio import downsample_audio
 import torch
 
 from infer.lib.uvr5_pack.lib_v5 import nets_123821KB as Nets
@@ -60,7 +61,7 @@ class AudioPre:
                 (
                     X_wave[d],
                     _,
-                ) = librosa.core.load(  # 理论上librosa读取可能对某些音频有bug，应该上ffmpeg读取，但是太麻烦了弃坑
+                ) = librosa.core.load(  # 理论上librosa读取可能对某些音频有bug，应该上av读取，但是太麻烦了弃坑
                     music_file,
                     bp["sr"],
                     False,
@@ -146,12 +147,7 @@ class AudioPre:
                 )
                 if os.path.exists(path):
                     opt_format_path = path[:-4] + ".%s" % format
-                    os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
-                    if os.path.exists(opt_format_path):
-                        try:
-                            os.remove(path)
-                        except:
-                            pass
+                    downsample_audio(path, opt_format_path, format)
         if vocal_root is not None:
             if is_hp3 == True:
                 head = "instrument_"
@@ -185,14 +181,8 @@ class AudioPre:
                     (np.array(wav_vocals) * 32768).astype("int16"),
                     self.mp.param["sr"],
                 )
-                if os.path.exists(path):
-                    opt_format_path = path[:-4] + ".%s" % format
-                    os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
-                    if os.path.exists(opt_format_path):
-                        try:
-                            os.remove(path)
-                        except:
-                            pass
+                opt_format_path = path[:-4] + ".%s" % format
+                downsample_audio(path, opt_format_path, format)
 
 
 class AudioPreDeEcho:
@@ -241,7 +231,7 @@ class AudioPreDeEcho:
                 (
                     X_wave[d],
                     _,
-                ) = librosa.core.load(  # 理论上librosa读取可能对某些音频有bug，应该上ffmpeg读取，但是太麻烦了弃坑
+                ) = librosa.core.load(  # 理论上librosa读取可能对某些音频有bug，应该上av读取，但是太麻烦了弃坑
                     music_file,
                     bp["sr"],
                     False,
@@ -323,12 +313,7 @@ class AudioPreDeEcho:
                 )
                 if os.path.exists(path):
                     opt_format_path = path[:-4] + ".%s" % format
-                    os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
-                    if os.path.exists(opt_format_path):
-                        try:
-                            os.remove(path)
-                        except:
-                            pass
+                    downsample_audio(path, opt_format_path, format)
         if vocal_root is not None:
             if self.data["high_end_process"].startswith("mirroring"):
                 input_high_end_ = spec_utils.mirroring(
@@ -360,9 +345,4 @@ class AudioPreDeEcho:
                 )
                 if os.path.exists(path):
                     opt_format_path = path[:-4] + ".%s" % format
-                    os.system(f'ffmpeg -i "{path}" -vn "{opt_format_path}" -q:a 2 -y')
-                    if os.path.exists(opt_format_path):
-                        try:
-                            os.remove(path)
-                        except:
-                            pass
+                    downsample_audio(path, opt_format_path, format)