From a54fb25c7a6b46c29bae73e8c2562e76a8b96b5c Mon Sep 17 00:00:00 2001 From: Escartem Date: Fri, 19 Jul 2024 12:15:09 +0200 Subject: [PATCH] rework all app, add gui, wip --- app.py | 133 ++++++++++++++ extract.py | 512 +++++++++++++++++++++++++---------------------------- gui.ui | 296 +++++++++++++++++++++++++++++++ 3 files changed, 668 insertions(+), 273 deletions(-) create mode 100644 app.py create mode 100644 gui.ui diff --git a/app.py b/app.py new file mode 100644 index 0000000..713af63 --- /dev/null +++ b/app.py @@ -0,0 +1,133 @@ +import os +import sys +import json +import time +import mapper +import extract +from PyQt5 import uic +from PyQt5.QtGui import QTextCursor +from PyQt5.QtWidgets import QMessageBox, QMainWindow, QApplication, QFileDialog +from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QThread, QMetaType + +QMetaType.type('QTextCursor') + +class TextEditStream(QObject): + append_text = pyqtSignal(str) + + def __init__(self, text_edit): + super().__init__() + self.text_edit = text_edit + self.append_text.connect(self._append_text) + + def write(self, text): + self.append_text.emit(text) + + def flush(self): + pass + + def _append_text(self, text): + self.text_edit.moveCursor(QTextCursor.End) + self.text_edit.insertPlainText(text) + self.text_edit.moveCursor(QTextCursor.End) + +class ExtractionWorker(QObject): + finished = pyqtSignal() + progress = pyqtSignal(list) + + def __init__(self, folders, _map, _format): + super().__init__() + self.folders = folders + self.map = _map + self.format = _format + + def run(self): + _map = None + if self.map: + _map = mapper.Mapper(os.path.join(os.getcwd(), f"maps/{self.map}")) + print("\n==========\n") + self.progress.emit(["total", 5]) + extracter = extract.WwiseExtract(_map, self.format, *self.folders.values(), progress=self.progress.emit) + extracter.extract() + self.finished.emit() + +class AnimeWwise(QMainWindow): + def __init__(self): + super(AnimeWwise, self).__init__() + uic.loadUi("gui.ui", self) + # self.setupUi(self) + self.maps = self.getMaps() + self.folders = { + "input": "", + "output": "", + "diff": "" + } + self.setupActions() + sys.stdout = TextEditStream(self.console) + + # utils + self.selectFolder = lambda: QFileDialog.getExistingDirectory(self, "Select Folder") + + def setFolder(self, elem, folder): + path = self.selectFolder() + self.folders[folder] = path + elem.setText(path) + + def setupActions(self): + self.changeInput.clicked.connect(lambda: self.setFolder(self.inputPath, "input")) + self.changeAltInput.clicked.connect(lambda: self.setFolder(self.altInputPath, "diff")) + self.changeOutput.clicked.connect(lambda: self.setFolder(self.outputPath, "output")) + + self.outputFormat.addItems(["mp3", "ogg"]) + self.assetMap.addItems(["No map", *[f'{e["game"]} - v{e["version"]}' for e in self.maps["maps"]]]) + + self.startButton.clicked.connect(lambda: self.start()) + + def getMaps(self): + with open("maps/index.json", "r") as f: + maps = json.loads(f.read()) + f.close() + + return maps + + @pyqtSlot(list) + def progressBarSlot(self, progress): + if progress[0] == "total": + self.progress.setValue(progress[1]) + elif progress[0] == "task": + self.taskProgress.setValue(progress[1]) + + def start(self): + if "" in [self.folders["input"], self.folders["output"]]: + QMessageBox.warning(None, "Warning", "Missing input/output folder !", QMessageBox.Ok) + return + + _map = self.assetMap.currentIndex() + if _map != 0: + _map = self.maps["maps"][_map-1]["name"] + else: + _map = None + + self.extractThread = QThread() + self.extractWorker = ExtractionWorker(self.folders, _map, self.outputFormat.currentText()) + self.extractWorker.moveToThread(self.extractThread) + self.extractThread.started.connect(self.extractWorker.run) + self.extractWorker.finished.connect(self.extractThread.quit) + self.extractWorker.finished.connect(self.extractWorker.deleteLater) + self.extractThread.finished.connect(self.extractThread.deleteLater) + + self.extractWorker.progress.connect(self.progressBarSlot) + self.extractThread.start() + + def _appendText(self, text): + cursor = self.console.textCursor() + cursor.movePosition(cursor.End) + cursor.insertText(text) + self.console.setTextCursor(cursor) + self.console.ensureCursorVisible() + + +if __name__ == "__main__": + app = QApplication(sys.argv) + window = AnimeWwise() + window.show() + sys.exit(app.exec_()) diff --git a/extract.py b/extract.py index 1c1606a..02b5487 100644 --- a/extract.py +++ b/extract.py @@ -1,21 +1,16 @@ import os import sys -import mapper +import time import shutil import filecmp -import argparse +import tempfile import wavescan import subprocess -from halo import Halo -from progress.bar import PixelBar - -print("Setting up...") cwd = os.getcwd() path = lambda path: os.path.join(cwd, path) call = lambda args: subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) -mapper = mapper.Mapper(path("mapping/latest.map")) -spinner = Halo(text="spinner", spinner={'interval': 100, 'frames': ['◜', '◠', '◝', '◞', '◡', '◟']}, placement="right") + skips = "000000000" # used for debugging # 1 - original extract @@ -28,310 +23,281 @@ skips = "000000000" # used for debugging # 8 - clean up # 9 - temp clean up -def main(): - parser = argparse.ArgumentParser() +class WwiseExtract: + def __init__(self, _map, _format, input_folder, output_folder, diff_folder, progress): + self.map = _map + self.format = _format + + self.paths = { + "input": input_folder, + "output": output_folder, + "diff": diff_folder, + "temp": tempfile.TemporaryDirectory() + } + + self.progress = progress + # TODO: add skip / select mapping option - parser.add_argument("--format", nargs="?", type=str, default="mp3", help="Output audio format, can be either mp3 or ogg") - args = parser.parse_args() - formats = ["mp3", "ogg"] - audio_format = "mp3" - if args.format in formats: - audio_format = args.format + def path(self, base, path): + base_path = self.paths[base] + if base == "temp": + base_path = base_path.name + return os.path.join(base_path, path) - print(f'Format: {audio_format}') + def extract(self): + audio_format = self.format + mapper = self.map + _p = self.path # lazy - # Initial cleanup - if os.path.exists("temp") and skips[8] != "1": - shutil.rmtree("temp") + print(f'Format: {audio_format}') - if os.path.exists("output") and len(os.listdir("output")) > 0: - print("The output folder needs to be cleared, continue ? [Y/N]") - select = input(">") - if select.lower() == "y": - shutil.rmtree("output") - else: - print("Aborting") - exit() + # TODO: ui popup + # if os.path.exists("output") and len(os.listdir("output")) > 0: + # print("The output folder needs to be cleared, continue ? [Y/N]") + # select = input(">") + # if select.lower() == "y": + # shutil.rmtree("output") + # else: + # print("Aborting") + # exit() - # Get all files to process - hdiff_files = [f for f in os.listdir("audio") if f.endswith(".pck") and os.path.exists(f"patch/{f}.hdiff")] - alone_files = [f for f in os.listdir("audio") if f.endswith(".pck") and not os.path.exists(f"patch/{f}.hdiff")] - files = [*hdiff_files, *alone_files] + # Get all files to process + hdiff_files = [f for f in os.listdir(self.paths["input"]) if f.endswith(".pck") and os.path.exists(_p("diff", f"{f}.hdiff"))] + alone_files = [f for f in os.listdir(self.paths["input"]) if f.endswith(".pck") and not os.path.exists(_p("diff", f"{f}.hdiff"))] + files = [*hdiff_files, *alone_files] - if len(files) == 0: - print("No files found !") - return + if len(files) == 0: + print("No files found !") + # self.progress(100) + return - print(f"{len(files)} file{'s' if len(files) != 1 else ''} to extract") - iteration = 0 + print(self.paths["temp"].name) - for file in files: - try: - iteration += 1 - filename = file - if file in hdiff_files: - filename = f"{file.split('.')[0]}.hdiff.pck" - print(f"--- {filename} ({iteration}/{len(files)}) ---") + print(f"{len(files)} file{'s' if len(files) != 1 else ''} to extract") + iteration = 0 - alone, steps, curr = False, 8, 1 - if file in alone_files: - alone, steps = True, 5 + for file in files: + try: + iteration += 1 + filename = file + if file in hdiff_files: + filename = f"{file.split('.')[0]}.hdiff.pck" + print(f"--- {filename} ({iteration}/{len(files)}) ---") - ###################################### - ### 1 - Extract original .pck file ### - ###################################### + alone = False #8 steps + if file in alone_files: + alone = True # 5 steps - if skips[0] != "1": - # update files - if os.path.exists("temp"): - shutil.rmtree("temp") - os.makedirs(path("temp"), exist_ok=True) - shutil.copy(f"audio/{file}", f"temp/{file}") + ###################################### + ### 1 - Extract original .pck file ### + ###################################### - output_path = "original_decoded" - if alone: - output_path = "wem" + if skips[0] != "1": + shutil.copy(_p("input", file), _p("temp", file)) - # update spinner and call program - spinner.text = f"[{curr}/{steps}] Extracting" - spinner.start() - wavescan.extract(path(f"temp/{file}"), path(f"temp/{output_path}")) - spinner.stop() - print(f"[{curr}/{steps}] Extracting") + output_path = "original_decoded" + if alone: + output_path = "wem" - if alone: - all_files = os.listdir(path("temp/wem")) + print(f"Extracting") + wavescan.extract(_p("temp", file), _p("temp", output_path)) + self.progress(["total", 15]) - ###################################### - ### 2 - Patch the .pck with .hdiff ### - ###################################### + if alone: + all_files = os.listdir(_p("temp", "wem")) - if skips[1] != "1": - if not alone: - curr += 1 + ###################################### + ### 2 - Patch the .pck with .hdiff ### + ###################################### - # update files - shutil.copy(f"patch/{file}.hdiff", f"temp/{file}.hdiff") - shutil.move(f"temp/{file}", f"temp/{file.split('.')[0]}.original.pck") + if skips[1] != "1": + if not alone: + print(f"Patching") - # prepare args - args = [ - path("tools/hpatchz/hpatchz.exe"), - "-f", - path(f"temp/{file.split('.')[0]}.original.pck"), - path(f"temp/{file}.hdiff"), - path(f"temp/{file}") - ] + # update files + shutil.copy(_p("diff", f"{file}.hdiff"), _p("temp", f"{file}.hdiff")) + shutil.move(_p("temp", file), _p("temp", f"{file.split('.')[0]}.original.pck")) - # update spinner and call program - spinner.text = f"[{curr}/{steps}] Patching" - spinner.start() - call(args) - spinner.stop() - print(f"[{curr}/{steps}] Patching") + # prepare args + args = [ + path("tools/hpatchz/hpatchz.exe"), + "-f", + _p("temp", f"{file.split('.')[0]}.original.pck"), + _p("temp", f"{file}.hdiff"), + _p("temp", file) + ] - ##################################### - ### 3 - Extract patched .pck file ### - ##################################### + call(args) + self.progress(["total", 20]) - if skips[2] != "1": - if not alone: - curr += 1 + ##################################### + ### 3 - Extract patched .pck file ### + ##################################### - # update spinner and call program - spinner.text = f"[{curr}/{steps}] Extracting patch" - spinner.start() - wavescan.extract(path(f"temp/{file}"), path(f"temp/patched_decoded")) - spinner.stop() - print(f"[{curr}/{steps}] Extracting patch") + if skips[2] != "1": + if not alone: + print(f"Extracting patch") + wavescan.extract(path(f"temp/{file}"), path(f"temp/patched_decoded")) + self.progress(["total", 30]) - # cleanup useless files to save storage - os.remove(f"temp/{file}") - os.remove(f"temp/{file}.hdiff") - os.remove(f"temp/{file.split('.')[0]}.original.pck") + # cleanup useless files to save storage + os.remove(_p("temp", file)) + os.remove(_p("temp", f"{file}.hdiff")) + os.remove(_p("temp", f"{file.split('.')[0]}.original.pck")) - #################################### - ### 4 - Search new/changed files ### - #################################### + #################################### + ### 4 - Search new/changed files ### + #################################### - if skips[3] != "1": - if not alone: - curr += 1 + if skips[3] != "1": + if not alone: + print(f"Filtering files") + # compare folders + diff = filecmp.dircmp(_p("temp", "original_decoded"), _p("temp", "patched_decoded")) + new_files, changed_files = diff.right_only, diff.diff_files + all_files = [*new_files, *changed_files] - # update spinner - spinner.text = f"[{curr}/{steps}] Filtering files" - spinner.start() + # merge files + os.makedirs(_p("temp", "wem"), exist_ok=True) - # compare folders - diff = filecmp.dircmp(path("temp/original_decoded"), path("temp/patched_decoded")) - new_files, changed_files = diff.right_only, diff.diff_files - all_files = [*new_files, *changed_files] + for file in all_files: + shutil.move(_p("temp", f"patched_decoded/{file}"), _p("temp", f"wem/{file}")) - # merge files - os.makedirs(path("temp/wem"), exist_ok=True) + # cleanup useless folders to save storage + shutil.rmtree(_p("temp", "original_decoded")) + shutil.rmtree(_p("temp", "patched_decoded")) + self.progress(["total", 35]) + + ###################################### + ### 5 - Convert .wem files to .wav ### + ###################################### + + if skips[4] != "1": + + # updates folders + os.makedirs(_p("temp", "wav"), exist_ok=True) + print(f"Converting to wav") + pos = 0 + # convert each file one by one + for file in all_files: + pos += 1 + args = [ + path("tools/vgmstream/vgmstream-cli.exe"), + "-o", + _p("temp", f"wav/{file.split('.')[0]}.wav"), + _p("temp", f"wem/{file}") + ] + + call(args) + self.progress(["total", round(35 + (pos * 30) / len(all_files))]) + self.progress(["task", round((pos * 100) / len(all_files))]) + + # cleanup + shutil.rmtree(_p("temp", "wem")) + wem_length = len(all_files) + all_files = [f for f in os.listdir(_p("temp", "wav"))] + diff_length = wem_length - len(all_files) + + if diff_length > 0: + print(f": Failed to extract {diff_length} files out of {wem_length} (probably no extractable content)") + + ############################################# + ### 6 - Convert .wav files to .mp3 or ogg ### + ############################################# + + if skips[5] != "1": + # updates folders and progress bar + os.makedirs(_p("temp", audio_format), exist_ok=True) + print(f"Converting to {audio_format}") + + # update file list + all_files = [f"{f.split('.')[0]}.wav" for f in all_files] + pos = 0 + # convert each file one by one + for file in all_files: + pos += 1 + args = [ + path("tools/ffmpeg/ffmpeg.exe"), + "-i", + _p("temp", f"wav/{file}"), + "-acodec", + "libvorbis" if audio_format == "ogg" else "libmp3lame", + "-b:a", + "192k", # 192k | 4k + _p("temp", f"{audio_format}/{file.split('.')[0]}.{audio_format}"), + ] + + call(args) + self.progress(["total", (round(65 + (pos * 30) / len(all_files)))]) + self.progress(["task", round((pos * 100) / len(all_files))]) + + # cleanup + shutil.rmtree(_p("temp", "wav")) + + # update files list + all_files = [f"{f.split('.')[0]}.{audio_format}" for f in all_files] + + if not alone: + new_files = [f"{f.split('.')[0]}.{audio_format}" for f in new_files] + changed_files = [f"{f.split('.')[0]}.{audio_format}" for f in changed_files] + + ######################### + ### 7 - Map filenames ### + ######################### + + if skips[6] != "1" or mapper != None: + print(f"Mapping names") + + os.makedirs(_p("temp", "map/unmapped"), exist_ok=True) + if not alone: + os.makedirs(_p("temp", f"map/new_files/unmapped"), exist_ok=True) + os.makedirs(_p("temp", f"map/changed_files/unmapped"), exist_ok=True) + + lang = None for file in all_files: - shutil.move(f"temp/patched_decoded/{file}", f"temp/wem/{file}") + file_name = file.split(".")[0] + base_path = "map" + if not alone: + if file in new_files: + base_path = "map/new_files" + elif file in changed_files: + base_path = "map/changed_files" - # cleanup useless folders to save storage - shutil.rmtree("temp/original_decoded") - shutil.rmtree("temp/patched_decoded") + key_data = mapper.get_key(file_name, lang is None) - spinner.stop() - print(f"[{curr}/{steps}] Filtering files") + if key_data is not None: + if lang is None: + lang = key_data[1] + # TODO: use language for output path + print(f"\n: {lang} detected") - ###################################### - ### 5 - Convert .wem files to .wav ### - ###################################### + dir_path = _p("temp", f"{base_path}/{key_data[0]}.{audio_format}") + os.makedirs(os.path.dirname(dir_path), exist_ok=True) + shutil.copy(_p("temp", f"{audio_format}/{file}"), dir_path) + else: + shutil.copy(_p("temp", f"{audio_format}/{file}"), _p("temp", f"{base_path}/unmapped/{file}")) - if skips[4] != "1": - curr += 1 + ###################################################### + ### 8 - Clean everything and move result to output ### + ###################################################### - # updates folders and progress bar - os.makedirs(path("temp/wav"), exist_ok=True) - bar = PixelBar(f"[{curr}/{steps}] Converting to wav ", max=len(all_files), suffix='%(percent).1f%% - %(eta)ds left') + if skips[7] != "1": + print(f"Cleaning up") - # convert each file one by one - for file in all_files: - args = [ - path("tools/vgmstream/vgmstream-cli.exe"), - "-o", - path(f"temp/wav/{file.split('.')[0]}.wav"), - path(f"temp/wem/{file}") - ] + filename = filename.split('.')[0] - call(args) - bar.next() - bar.finish() + shutil.move(_p("temp", "map"), _p("output", filename)) - # cleanup - shutil.rmtree("temp/wem") - wem_length = len(all_files) - all_files = [f for f in os.listdir(path("temp/wav"))] - diff_length = wem_length - len(all_files) + self.paths["temp"].cleanup() - if diff_length > 0: - print(f": Failed to extract {diff_length} files out of {wem_length} (probably no extractable content)") + self.progress["total", 100] - ############################################# - ### 6 - Convert .wav files to .mp3 or ogg ### - ############################################# + except Exception as e: + print("") + print("An error occured while processing this file ! Skipping... details of the error bellow :") + print(f"Line {sys.exc_info()[-1].tb_lineno}, {e}") - if skips[5] != "1": - curr += 1 - - # updates folders and progress bar - os.makedirs(path(f"temp/{audio_format}"), exist_ok=True) - bar = PixelBar( - f"[{curr}/{steps}] Converting to {audio_format} ", - max=len(all_files), - suffix="%(percent).1f%% - %(eta)ds left", - ) - - # update file list - all_files = [f"{f.split('.')[0]}.wav" for f in all_files] - - # convert each file one by one - for file in all_files: - args = [ - path("tools/ffmpeg/ffmpeg.exe"), - "-i", - path(f"temp/wav/{file}"), - "-acodec", - "libvorbis" if audio_format == "ogg" else "libmp3lame", - "-b:a", - "192k", - path(f"temp/{audio_format}/{file.split('.')[0]}.{audio_format}"), - ] - - call(args) - bar.next() - bar.finish() - - # cleanup - shutil.rmtree("temp/wav") - - # update files list - all_files = [f"{f.split('.')[0]}.{audio_format}" for f in all_files] - - if not alone: - new_files = [f"{f.split('.')[0]}.{audio_format}" for f in new_files] - changed_files = [f"{f.split('.')[0]}.{audio_format}" for f in changed_files] - - ######################### - ### 7 - Map filenames ### - ######################### - - if skips[6] != "1": - curr += 1 - - # update spinner - spinner.text = f"[{curr}/{steps}] Mapping names" - spinner.start() - - os.makedirs(path(f"temp/map/unmapped"), exist_ok=True) - if not alone: - os.makedirs(path(f"temp/map/new_files/unmapped"), exist_ok=True) - os.makedirs(path(f"temp/map/changed_files/unmapped"), exist_ok=True) - - lang = None - - for file in all_files: - file_name = file.split(".")[0] - base_path = "temp/map" - if not alone: - if file in new_files: - base_path = "temp/map/new_files" - elif file in changed_files: - base_path = "temp/map/changed_files" - - key_data = mapper.get_key(file_name, lang is None) - - if key_data is not None: - if lang is None: - lang = key_data[1] - # TODO: use language for output path - print(f"\n: {lang} detected") - - dir_path = path(f"{base_path}/{key_data[0]}.{audio_format}") - os.makedirs(os.path.dirname(dir_path), exist_ok=True) - shutil.copy(path(f"temp/{audio_format}/{file}"), dir_path) - else: - shutil.copy(path(f"temp/{audio_format}/{file}"), path(f"{base_path}/unmapped/{file}")) - - # stop spinner - spinner.stop() - print(f"[{curr}/{steps}] Mapping names") - - ###################################################### - ### 8 - Clean everything and move result to output ### - ###################################################### - - if skips[7] != "1": - curr += 1 - - # update spinner - spinner.text = f"[{curr}/{steps}] Cleaning up" - spinner.start() - - filename = filename.split('.')[0] - - shutil.move(f"temp/map", f"output/{filename}") - - spinner.stop() - print(f"[{curr}/{steps}] Cleaning up") - - except Exception as e: - print("") - print("An error occured while processing this file ! Skipping to the next one, details of the error bellow :") - print(f"Line {sys.exc_info()[-1].tb_lineno}, {e}") - - # all files processed - if os.path.exists("temp") and skips[8] != "1": - shutil.rmtree("temp") - print("-"*30) - print("Done extracting everything !") - -if __name__ == "__main__": - main() + print("-"*30) + print("Done extracting everything !") diff --git a/gui.ui b/gui.ui new file mode 100644 index 0000000..54df3ec --- /dev/null +++ b/gui.ui @@ -0,0 +1,296 @@ + + + AnimeWwise + + + Qt::NonModal + + + + 0 + 0 + 1100 + 800 + + + + + 1100 + 800 + + + + + 1100 + 800 + + + + AnimeWwise + + + + + + 4 + -1 + 1091 + 791 + + + + 0 + + + true + + + false + + + false + + + false + + + false + + + + true + + + Extract + + + + + 9 + 9 + 1071 + 741 + + + + + + + + + Output folder + + + + + + + Select + + + + + + + Select + + + false + + + false + + + + + + + Select + + + + + + + Input folder + + + + + + + Diff folder (optional) + + + + + + + true + + + + + + true + + + + + + + true + + + + + + + true + + + + + + + + + Qt::Horizontal + + + + + + + + + Output format + + + + + + + Asset map (optional) + + + + + + + + + + + + + + + + + + + + + + + + + + Start + + + + + + + Qt::Horizontal + + + + + + + + + + + Progress + + + + + + + 0 + + + + + + + + + + + Task Progress + + + + + + + 0 + + + + + + + + + + + Qt::Horizontal + + + + + + + + 16777215 + 220 + + + + true + + + + + + + + + Browse + + + + + + + inputPath + changeInput + altInputPath + changeAltInput + outputPath + changeOutput + outputFormat + assetMap + startButton + tabs + + + +