From 4ed830f785ad4d4b59c59db204f896052cb24e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?= <41315874+fumiama@users.noreply.github.com> Date: Sun, 21 Apr 2024 00:16:10 +0900 Subject: [PATCH] feat(web): check assets integrity on start --- .env | 40 +++++++++ .github/workflows/unitest.yml | 5 +- configs/config.py | 5 ++ infer-web.py | 6 ++ infer/lib/rvcmd.py | 150 ++++++++++++++++++++++++++++++++++ run.sh | 7 +- 6 files changed, 206 insertions(+), 7 deletions(-) create mode 100644 infer/lib/rvcmd.py diff --git a/.env b/.env index 3e17125..39ccfcc 100644 --- a/.env +++ b/.env @@ -7,3 +7,43 @@ weight_uvr5_root = assets/uvr5_weights index_root = logs outside_index_root = assets/indices rmvpe_root = assets/rmvpe + +sha256_hubert_base_pt = f54b40fd2802423a5643779c4861af1e9ee9c1564dc9d32f54f20b5ffba7db96 +sha256_rmvpe_pt = 6d62215f4306e3ca278246188607209f09af3dc77ed4232efdd069798c4ec193 +sha256_rmvpe_onnx = 5370e71ac80af8b4b7c793d27efd51fd8bf962de3a7ede0766dac0befa3660fd + +sha256_v1_D32k_pth = 2ab20645829460fdad0d3c44254f1ab53c32cae50c22a66c926ae5aa30abda6f +sha256_v1_D40k_pth = 547f66dbbcd9023b9051ed244d12ab043ba8a4e854b154cc28761ac7c002909b +sha256_v1_D48k_pth = 8cc013fa60ed9c3f902f5bd99f48c7e3b9352d763d4d3cd6bc241c37b0bfd9ad +sha256_v1_G32k_pth = 81817645cde7ed2e2d83f23ef883f33dda564924b497e84d792743912eca4c23 +sha256_v1_G40k_pth = e428573bda1124b0ae0ae843fd8dcded6027d3993444790b3e9b0100938b2113 +sha256_v1_G48k_pth = 3862a67ea6313e8ffefc05cee6bee656ef3e089442e9ecf4a6618d60721f3e95 +sha256_v1_f0D32k_pth = 294db3087236e2c75260d6179056791c9231245daf5d0485545d9e54c4057c77 +sha256_v1_f0D40k_pth = 7d4f5a441594b470d67579958b2fd4c6b992852ded28ff9e72eda67abcebe423 +sha256_v1_f0D48k_pth = 1b84c8bf347ad1e539c842e8f2a4c36ecd9e7fb23c16041189e4877e9b07925c +sha256_v1_f0G32k_pth = 285f524bf48bb692c76ad7bd0bc654c12bd9e5edeb784dddf7f61a789a608574 +sha256_v1_f0G40k_pth = 9115654aeef1995f7dd3c6fc4140bebbef0ca9760bed798105a2380a34299831 +sha256_v1_f0G48k_pth = 78bc9cab27e34bcfc194f93029374d871d8b3e663ddedea32a9709e894cc8fe8 + +sha256_v2_D32k_pth = d8043378cc6619083d385f5a045de09b83fb3bf8de45c433ca863b71723ac3ca +sha256_v2_D40k_pth = 471378e894e7191f89a94eda8288c5947b16bbe0b10c3f1f17efdb7a1d998242 +sha256_v2_D48k_pth = db01094a93c09868a278e03dafe8bb781bfcc1a5ba8df168c948bf9168c84d82 +sha256_v2_G32k_pth = 869b26a47f75168d6126f64ac39e6de5247017a8658cfd68aca600f7323efb9f +sha256_v2_G40k_pth = a3843da7fde33db1dab176146c70d6c2df06eafe9457f4e3aa10024e9c6a4b69 +sha256_v2_G48k_pth = 2e2b1581a436d07a76b10b9d38765f64aa02836dc65c7dee1ce4140c11ea158b +sha256_v2_f0D32k_pth = bd7134e7793674c85474d5145d2d982e3c5d8124fc7bb6c20f710ed65808fa8a +sha256_v2_f0D40k_pth = 6b6ab091e70801b28e3f41f335f2fc5f3f35c75b39ae2628d419644ec2b0fa09 +sha256_v2_f0D48k_pth = 2269b73c7a4cf34da09aea99274dabf99b2ddb8a42cbfb065fb3c0aa9a2fc748 +sha256_v2_f0G32k_pth = 2332611297b8d88c7436de8f17ef5f07a2119353e962cd93cda5806d59a1133d +sha256_v2_f0G40k_pth = 3b2c44035e782c4b14ddc0bede9e2f4a724d025cd073f736d4f43708453adfcb +sha256_v2_f0G48k_pth = b5d51f589cc3632d4eae36a315b4179397695042edc01d15312e1bddc2b764a4 + +sha256_uvr5_HP2-人声vocals+非人声instrumentals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05 +sha256_uvr5_HP2_all_vocals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05 +sha256_uvr5_HP3_all_vocals_pth = 45e6b65199e781b4a6542002699be9f19cd3d1cb7d1558bc2bfbcd84674dfe28 +sha256_uvr5_HP5-主旋律人声vocals+其他instrumentals_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee +sha256_uvr5_HP5_only_main_vocal_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee +sha256_uvr5_VR-DeEchoAggressive_pth = 8c8fd1582f9aabc363e47af62ddb88df6cae7e064cae75bbf041a067a5e0aee2 +sha256_uvr5_VR-DeEchoDeReverb_pth = 01376dd2a571bf3cb9cced680732726d2d732609d09216a610b0d110f133febe +sha256_uvr5_VR-DeEchoNormal_pth = 56aba59db3bcdd14a14464e62f3129698ecdea62eee0f003b9360923eb3ac79e +sha256_uvr5_vocals_onnx = 233bb5c6aaa365e568659a0a81211746fa881f8f47f82d9e864fce1f7692db80 diff --git a/.github/workflows/unitest.yml b/.github/workflows/unitest.yml index 9901f9f..d38a76a 100644 --- a/.github/workflows/unitest.yml +++ b/.github/workflows/unitest.yml @@ -19,13 +19,14 @@ jobs: run: | sudo apt update sudo apt -y install ffmpeg - sudo apt -y install -qq aria2 - aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt -d ./ -o hubert_base.pt + wget https://github.com/RVC-Project/RVC-Models-Downloader/releases/download/v0.2.1/rvcmd_linux_amd64.deb + sudo apt -y install ./rvcmd_linux_amd64.deb python -m pip install --upgrade pip python -m pip install --upgrade setuptools python -m pip install --upgrade wheel pip install torch torchvision torchaudio pip install -r requirements.txt + rvcmd -notrs -w 1 -notui assets/all - name: Test step 1 & 2 run: | mkdir -p logs/mi-test diff --git a/configs/config.py b/configs/config.py index a330fb5..78bfdae 100644 --- a/configs/config.py +++ b/configs/config.py @@ -57,6 +57,7 @@ class Config: self.noparallel, self.noautoopen, self.dml, + self.nocheck, ) = self.arg_parse() self.instead = "" self.preprocess_per = 3.7 @@ -93,6 +94,9 @@ class Config: action="store_true", help="torch_dml", ) + parser.add_argument( + "--nocheck", action="store_true", help="Run without checking assets" + ) cmd_opts = parser.parse_args() cmd_opts.port = cmd_opts.port if 0 <= cmd_opts.port <= 65535 else 7865 @@ -104,6 +108,7 @@ class Config: cmd_opts.noparallel, cmd_opts.noautoopen, cmd_opts.dml, + cmd_opts.nocheck, ) # has_mps is only available in nightly pytorch (for now) and MasOS 12.3+. diff --git a/infer-web.py b/infer-web.py index d0106a1..7e28423 100644 --- a/infer-web.py +++ b/infer-web.py @@ -13,6 +13,7 @@ from infer.lib.train.process_ckpt import ( merge, show_info, ) +from infer.lib.rvcmd import check_all_assets, download_all_assets from i18n.i18n import I18nAuto from configs.config import Config from sklearn.cluster import MiniBatchKMeans @@ -53,6 +54,11 @@ torch.manual_seed(114514) config = Config() vc = VC(config) +if not config.nocheck and not check_all_assets(): + download_all_assets(tmpdir=tmp) + if not check_all_assets(): + logging.error("counld not satisfy all assets needed.") + exit(1) if config.dml == True: diff --git a/infer/lib/rvcmd.py b/infer/lib/rvcmd.py new file mode 100644 index 0000000..2f916a5 --- /dev/null +++ b/infer/lib/rvcmd.py @@ -0,0 +1,150 @@ +import os +from pathlib import Path +import hashlib +import requests +from io import BytesIO +import logging + +logger = logging.getLogger(__name__) + +def sha256(f) -> str: + sha256_hash = hashlib.sha256() + # Read and update hash in chunks of 4M + for byte_block in iter(lambda: f.read(4*1024*1024), b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + +def check_model(dir_name: Path, model_name: str, hash: str) -> bool: + target = dir_name / model_name + relname = str(target) + relname = relname[relname.rindex("assets/"):] + logger.debug(f"checking {relname}...") + if not os.path.exists(target): + logger.info(f"{target} not exist.") + return False + with open(target, "rb") as f: + digest = sha256(f) + if digest != hash: + logger.info(f"{target} sha256 hash mismatch.") + logger.info(f"expected: {hash}") + logger.info(f"real val: {digest}") + return False + return True + +def check_all_assets() -> bool: + BASE_DIR = Path(__file__).resolve().parent.parent.parent + + logger.info("checking hubret & rmvpe...") + + if not check_model(BASE_DIR / "assets/hubert", "hubert_base.pt", os.environ["sha256_hubert_base_pt"]): + return False + if not check_model(BASE_DIR / "assets/rmvpe", "rmvpe.pt", os.environ["sha256_rmvpe_pt"]): + return False + if not check_model(BASE_DIR / "assets/rmvpe", "rmvpe.onnx", os.environ["sha256_rmvpe_onnx"]): + return False + + rvc_models_dir = BASE_DIR / "assets/pretrained" + logger.info("checking pretrained models...") + model_names = [ + "D32k.pth", + "D40k.pth", + "D48k.pth", + "G32k.pth", + "G40k.pth", + "G48k.pth", + "f0D32k.pth", + "f0D40k.pth", + "f0D48k.pth", + "f0G32k.pth", + "f0G40k.pth", + "f0G48k.pth", + ] + for model in model_names: + menv = model.replace(".", "_") + if not check_model(rvc_models_dir, model, os.environ[f"sha256_v1_{menv}"]): + return False + + rvc_models_dir = BASE_DIR / "assets/pretrained_v2" + logger.info("checking pretrained models v2...") + for model in model_names: + menv = model.replace(".", "_") + if not check_model(rvc_models_dir, model, os.environ[f"sha256_v2_{menv}"]): + return False + + logger.info("checking uvr5_weights...") + rvc_models_dir = BASE_DIR / "assets/uvr5_weights" + model_names = [ + "HP2-人声vocals+非人声instrumentals.pth", + "HP2_all_vocals.pth", + "HP3_all_vocals.pth", + "HP5-主旋律人声vocals+其他instrumentals.pth", + "HP5_only_main_vocal.pth", + "VR-DeEchoAggressive.pth", + "VR-DeEchoDeReverb.pth", + "VR-DeEchoNormal.pth", + ] + for model in model_names: + menv = model.replace(".", "_") + if not check_model(rvc_models_dir, model, os.environ[f"sha256_uvr5_{menv}"]): + return False + if not check_model( + BASE_DIR / "assets/uvr5_weights/onnx_dereverb_By_FoxJoy", + "vocals.onnx", + os.environ[f"sha256_uvr5_vocals_onnx"], + ): return False + + logger.info("all assets are already latest.") + return True + +def download_and_extract_tar_gz(url: str, folder: str): + import tarfile + logger.info(f"downloading {url}") + response = requests.get(url, stream=True) + with BytesIO() as out_file: + out_file.write(response.content) + out_file.seek(0) + logger.info(f"downloaded.") + with tarfile.open(fileobj=out_file, mode="r:gz") as tar: + tar.extractall(folder) + logger.info(f"extracted into {folder}") + +def download_and_extract_zip(url: str, folder: str): + import zipfile + logger.info(f"downloading {url}") + response = requests.get(url) + with BytesIO() as out_file: + out_file.write(response.content) + out_file.seek(0) + logger.info(f"downloaded.") + with zipfile.ZipFile(out_file) as zip_ref: + zip_ref.extractall(folder) + logger.info(f"extracted into {folder}") + +def download_all_assets(tmpdir: str, version="0.2.1"): + import subprocess + import platform + + archs = { + "aarch64": "arm64", "armv8l": "arm64", "arm64": "arm64", + "x86": "386", "i386": "386", "i686": "386", "386": "386", + "x86_64": "amd64", "x64": "amd64", "amd64": "amd64", + } + system_type = platform.system().lower() + architecture = platform.machine().lower() + is_win = architecture == "windows" + + architecture = archs.get(architecture, None) + if not architecture: + logger.error(f"architecture {architecture} is not supported") + exit(1) + BASE_URL = "https://github.com/RVC-Project/RVC-Models-Downloader/releases/download/" + suffix = "zip" if is_win else "tar.gz" + RVCMD_URL = BASE_URL+f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}" + cmdfile = tmpdir + "/rvcmd" + if is_win: + download_and_extract_zip(RVCMD_URL, tmpdir) + cmdfile += ".exe" + else: + download_and_extract_tar_gz(RVCMD_URL, tmpdir) + os.chmod(cmdfile, 0o755) + subprocess.run([cmdfile, "-notui", "-w", "0", "assets/all"]) diff --git a/run.sh b/run.sh index f239307..f034d23 100755 --- a/run.sh +++ b/run.sh @@ -4,9 +4,6 @@ if [ "$(uname)" = "Darwin" ]; then # macOS specific env: export PYTORCH_ENABLE_MPS_FALLBACK=1 export PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 -elif [ "$(uname)" != "Linux" ]; then - echo "Unsupported operating system." - exit 1 fi if [ -d ".venv" ]; then @@ -22,8 +19,8 @@ else if [ "$(uname)" = "Darwin" ] && command -v brew >/dev/null 2>&1; then brew install python@3.8 elif [ "$(uname)" = "Linux" ] && command -v apt-get >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install python3.8 + sudo apt update + sudo apt install -y python3.8 else echo "Please install Python 3.8 manually." exit 1