From 6472ce2b0158a46ead5b0df1e9eafe3670f61e19 Mon Sep 17 00:00:00 2001 From: Escartem Date: Tue, 23 Jul 2024 13:01:57 +0200 Subject: [PATCH] lot of fixes and progress --- allocator.py | 25 +++++++ app.py | 77 ++++++++++++++------- extract.py | 38 +++++++---- gui.ui | 184 ++++++++++++++++++++++++++++++--------------------- 4 files changed, 210 insertions(+), 114 deletions(-) create mode 100644 allocator.py diff --git a/allocator.py b/allocator.py new file mode 100644 index 0000000..43b3063 --- /dev/null +++ b/allocator.py @@ -0,0 +1,25 @@ +# memory manager to prevent redundant calls to files and save up disk usage +# may cause massive ram usage if we input too many pck at once, fix required +import os +import mmap + +class Allocator: + def __init__(self): + self.files = {} + + def load_file(self, path): + filename = os.path.basename(path) + with open(path, "r+b") as f: + mmap_object = mmap.mmap(f.fileno(), 0) + + self.files[filename] = mmap_object, + + + def read_at(self, file, offset, size): + data = self.files[file][offset:offset+size] + return data + + def free_mem(self): + for file in list(self.files.keys()): + self.files[file].close() + self.files.clear() diff --git a/app.py b/app.py index 8cc2142..cf810bf 100644 --- a/app.py +++ b/app.py @@ -9,7 +9,7 @@ from PyQt5.QtGui import QTextCursor, QStandardItemModel, QStandardItem from PyQt5.QtWidgets import QMessageBox, QMainWindow, QApplication, QFileDialog, QHeaderView, QAbstractItemView, QTreeWidgetItem from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QThread, QMetaType, Qt -QMetaType.type('QTextCursor') +QMetaType.type("QTextCursor") class TextEditStream(QObject): append_text = pyqtSignal(str) @@ -34,20 +34,25 @@ class BackgroundWorker(QObject): finished = pyqtSignal(dict) progress = pyqtSignal(list) - def __init__(self, action, folders, _map, _format): + def __init__(self, action, extract, data): super().__init__() self.action = action - # TODO: dynamic args depending on action - self.folders = folders - self.map = _map - self.format = _format + self.extract = extract + + if action == "load": + self.input = data["input"] + self.map = data["map"] + if action == "extract": + self.input = data["input"] def run(self): if self.action == "load": print("Loading files and mapping if necessary...") - fileStructure = extract.WwiseExtract(self.map, "mp3", *self.folders.values(), progress=self.progress.emit).load_folder() + fileStructure = self.extract.load_folder(self.map, self.input) print("Done !") self.finished.emit({"action": "load", "content": fileStructure}) + if self.action == "extract": + self.extract.extract_files(self.input) class AnimeWwise(QMainWindow): def __init__(self): @@ -61,6 +66,7 @@ class AnimeWwise(QMainWindow): } self.setupActions() sys.stdout = TextEditStream(self.console) + self.extract = extract.WwiseExtract() # utils self.selectFolder = lambda: QFileDialog.getExistingDirectory(self, "Select Folder") @@ -82,15 +88,15 @@ class AnimeWwise(QMainWindow): self.changeAltInput.clicked.connect(lambda: self.setFolder(self.altInputPath, "diff")) self.changeOutput.clicked.connect(lambda: self.setFolder(self.outputPath, "output")) - self.outputFormat.addItems(["mp3", "ogg"]) + self.outputFormat.addItems(["wem (fastest)", "wav (fast)", "mp3 (slow)", "ogg (slow)"]) self.assetMap.addItems(["No map", *[f'{e["game"]} - v{e["version"]}' for e in self.maps["maps"]]]) self.tabs.setTabEnabled(1, False) self.tabs.setTabEnabled(2, False) - self.loadFilesButton.clicked.connect(lambda: self.start()) + self.loadFilesButton.clicked.connect(lambda: self.loadFiles()) - self.actionClearTreeView.triggered.connect(lambda: self.resetTreeWidget()) + self.actionReset.triggered.connect(lambda: self.resetApp()) self.actionExit.triggered.connect(lambda: self.close()) self.actionExtractSelected.triggered.connect(lambda: self.extractItems(False)) @@ -109,14 +115,15 @@ class AnimeWwise(QMainWindow): if data["action"] == "load": self.fileStructure = data["content"] self.updateTreeWidget() - self.tabs.setTabEnabled(0, True) + self.tabs.setTabEnabled(0, False) self.tabs.setTabEnabled(1, True) + self.tabs.setTabEnabled(2, True) self.tabs.setCurrentIndex(1) # page 1 - config - def start(self): - if "" in [self.folders["input"], self.folders["output"]]: - QMessageBox.warning(None, "Warning", "Missing input/output folder !", QMessageBox.Ok) + def loadFiles(self): + if self.folders["input"] == "": + QMessageBox.warning(None, "Warning", "Missing input folder !", QMessageBox.Ok) return _map = self.assetMap.currentIndex() @@ -128,17 +135,18 @@ class AnimeWwise(QMainWindow): self.tabs.setTabEnabled(0, False) self.resetTreeWidget() - self.extractThread = QThread() - self.extractWorker = BackgroundWorker("load", self.folders, _map, self.outputFormat.currentText()) - self.extractWorker.moveToThread(self.extractThread) - self.extractThread.started.connect(self.extractWorker.run) - self.extractWorker.finished.connect(self.handleFinished) - self.extractWorker.finished.connect(self.extractThread.quit) - self.extractWorker.finished.connect(self.extractWorker.deleteLater) - self.extractThread.finished.connect(self.extractThread.deleteLater) + # why is all this required for threading damnit + self.backgroundThread = QThread() + self.backgroundWorker = BackgroundWorker("load", self.extract, {"input": self.folders["input"], "map": _map}) + self.backgroundWorker.moveToThread(self.backgroundThread) + self.backgroundThread.started.connect(self.backgroundWorker.run) + self.backgroundWorker.finished.connect(self.handleFinished) + self.backgroundWorker.finished.connect(self.backgroundThread.quit) + self.backgroundWorker.finished.connect(self.backgroundWorker.deleteLater) + self.backgroundThread.finished.connect(self.backgroundThread.deleteLater) - self.extractWorker.progress.connect(self.progressBarSlot) - self.extractThread.start() + self.backgroundWorker.progress.connect(self.progressBarSlot) + self.backgroundThread.start() # page 2 - browsing def resetTreeWidget(self): @@ -174,7 +182,7 @@ class AnimeWwise(QMainWindow): self.addItems(folder_item, folder_content) for file in sorted(element.get("files", [])): - file_item = QTreeWidgetItem([str(file[0]), str(file[1]), str(file[2])]) + file_item = QTreeWidgetItem([str(file[0]), str(hex(file[1])), str(file[2])]) file_item.setFlags(file_item.flags() | Qt.ItemIsUserCheckable) file_item.setCheckState(0, Qt.Unchecked) if parent is None: @@ -197,7 +205,26 @@ class AnimeWwise(QMainWindow): print(checked_items) + # yet another block of threading bs + self.backgroundThread = QThread() + self.backgroundWorker = BackgroundWorker("extract", self.extract, {"input": self.folders["input"], "files": checked_items}) + self.backgroundWorker.moveToThread(self.backgroundThread) + self.backgroundThread.started.connect(self.backgroundWorker.run) + self.backgroundWorker.finished.connect(self.handleFinished) + self.backgroundWorker.finished.connect(self.backgroundThread.quit) + self.backgroundWorker.finished.connect(self.backgroundWorker.deleteLater) + self.backgroundThread.finished.connect(self.backgroundThread.deleteLater) + + self.backgroundWorker.progress.connect(self.progressBarSlot) + self.backgroundThread.start() + # misc + def resetApp(self): + self.resetTreeWidget() + self.tabs.setTabEnabled(0, True) + self.tabs.setTabEnabled(1, False) + self.tabs.setTabEnabled(2, False) + def _appendText(self, text): cursor = self.console.textCursor() cursor.movePosition(cursor.End) diff --git a/extract.py b/extract.py index 34ff96d..d8b72b1 100644 --- a/extract.py +++ b/extract.py @@ -9,10 +9,12 @@ import tempfile import wavescan import subprocess from filereader import FileReader +from allocator import Allocator cwd = os.getcwd() path = lambda path: os.path.join(cwd, path) call = lambda args: subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) +allocator = Allocator() skips = "000000000" # used for debugging @@ -27,18 +29,18 @@ skips = "000000000" # used for debugging # 9 - temp clean up class WwiseExtract: - def __init__(self, _map, _format, input_folder, output_folder, diff_folder, progress): - self.map = _map - self.format = _format + def __init__(self): + # self.map = _map + # self.format = _format self.paths = { - "input": input_folder, - "output": output_folder, - "diff": diff_folder, + "input": "", + "output": "", + "diff": "", "temp": tempfile.TemporaryDirectory() } - self.progress = progress + # self.progress = progress def path(self, base, path): base_path = self.paths[base] @@ -46,7 +48,7 @@ class WwiseExtract: base_path = base_path.name return os.path.join(base_path, path) - def extract(self): + def extract2(self): audio_format = self.format mapper = self.map _p = self.path # lazy @@ -305,18 +307,18 @@ class WwiseExtract: print("Done extracting everything !") - ### new content ### + ### loading files ### - def load_folder(self): + def load_folder(self, _map, path): self.mapper = None - if self.map is not None: - self.mapper = Mapper(os.path.join(os.getcwd(), f"maps/{self.map}")) + if _map is not None: + self.mapper = Mapper(os.path.join(os.getcwd(), f"maps/{_map}")) self.file_structure = {"folders": {}, "files": []} - files = [f for f in os.listdir(self.paths["input"]) if f.endswith(".pck")] + files = [f for f in os.listdir(path) if f.endswith(".pck")] for file in files: - self.load_file(os.path.join(self.paths["input"], file)) + self.load_file(os.path.join(path, file)) return self.file_structure @@ -364,3 +366,11 @@ class WwiseExtract: if "files" not in current_level: current_level["files"] = [] current_level["files"].append([parts[-1], meta[0], meta[1]]) + + ### extracting files ### + + def extract_files(self, _input): + # load all pck, then extract each file and apply conversion if required + # allocator.load_file(os.path.join(_input, "2050.pck")) + # allocator.read_at("2050.pck", 0, 0) + pass diff --git a/gui.ui b/gui.ui index 844a1ac..e08efea 100644 --- a/gui.ui +++ b/gui.ui @@ -39,7 +39,7 @@ - 0 + 2 true @@ -75,15 +75,11 @@ - - - - Output folder + + + + false - - - - Select @@ -102,17 +98,13 @@ - - - - Select + + + + false - - - - - - Input folder + + true @@ -123,6 +115,13 @@ + + + + Input folder + + + @@ -136,20 +135,6 @@ - - - - true - - - - - - - true - - - @@ -161,13 +146,6 @@ - - - - Output format - - - @@ -178,13 +156,6 @@ - - - - - - - @@ -231,52 +202,117 @@ Extract - + - 153 - 110 - 521 - 201 + 9 + 9 + 1061 + 601 - + - + - + - Progress + Output folder - - - 0 + + + true + + + + + + + Select - + - + - Task Progress + Output format - - - 0 + + + + + + + + + + + Total progress + + + + + + + 0 + + + + + + + + + + + Per file progress + + + + + + + 0 + + + + + + + + + + + Extract All + + + + + + + Extract Selected + + + + + + + @@ -320,19 +356,19 @@ File - + - false + true Extract - - + + @@ -342,19 +378,19 @@ not working here yet - + All files - + Selected files - + - Clear Table + Reset @@ -368,8 +404,6 @@ changeInput altInputPath changeAltInput - outputPath - changeOutput loadFilesButton tabs