diff --git a/allocator.py b/allocator.py
new file mode 100644
index 0000000..43b3063
--- /dev/null
+++ b/allocator.py
@@ -0,0 +1,25 @@
+# memory manager to prevent redundant calls to files and save up disk usage
+# may cause massive ram usage if we input too many pck at once, fix required
+import os
+import mmap
+
+class Allocator:
+ def __init__(self):
+ self.files = {}
+
+ def load_file(self, path):
+ filename = os.path.basename(path)
+ with open(path, "r+b") as f:
+ mmap_object = mmap.mmap(f.fileno(), 0)
+
+ self.files[filename] = mmap_object,
+
+
+ def read_at(self, file, offset, size):
+ data = self.files[file][offset:offset+size]
+ return data
+
+ def free_mem(self):
+ for file in list(self.files.keys()):
+ self.files[file].close()
+ self.files.clear()
diff --git a/app.py b/app.py
index 8cc2142..cf810bf 100644
--- a/app.py
+++ b/app.py
@@ -9,7 +9,7 @@ from PyQt5.QtGui import QTextCursor, QStandardItemModel, QStandardItem
from PyQt5.QtWidgets import QMessageBox, QMainWindow, QApplication, QFileDialog, QHeaderView, QAbstractItemView, QTreeWidgetItem
from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QThread, QMetaType, Qt
-QMetaType.type('QTextCursor')
+QMetaType.type("QTextCursor")
class TextEditStream(QObject):
append_text = pyqtSignal(str)
@@ -34,20 +34,25 @@ class BackgroundWorker(QObject):
finished = pyqtSignal(dict)
progress = pyqtSignal(list)
- def __init__(self, action, folders, _map, _format):
+ def __init__(self, action, extract, data):
super().__init__()
self.action = action
- # TODO: dynamic args depending on action
- self.folders = folders
- self.map = _map
- self.format = _format
+ self.extract = extract
+
+ if action == "load":
+ self.input = data["input"]
+ self.map = data["map"]
+ if action == "extract":
+ self.input = data["input"]
def run(self):
if self.action == "load":
print("Loading files and mapping if necessary...")
- fileStructure = extract.WwiseExtract(self.map, "mp3", *self.folders.values(), progress=self.progress.emit).load_folder()
+ fileStructure = self.extract.load_folder(self.map, self.input)
print("Done !")
self.finished.emit({"action": "load", "content": fileStructure})
+ if self.action == "extract":
+ self.extract.extract_files(self.input)
class AnimeWwise(QMainWindow):
def __init__(self):
@@ -61,6 +66,7 @@ class AnimeWwise(QMainWindow):
}
self.setupActions()
sys.stdout = TextEditStream(self.console)
+ self.extract = extract.WwiseExtract()
# utils
self.selectFolder = lambda: QFileDialog.getExistingDirectory(self, "Select Folder")
@@ -82,15 +88,15 @@ class AnimeWwise(QMainWindow):
self.changeAltInput.clicked.connect(lambda: self.setFolder(self.altInputPath, "diff"))
self.changeOutput.clicked.connect(lambda: self.setFolder(self.outputPath, "output"))
- self.outputFormat.addItems(["mp3", "ogg"])
+ self.outputFormat.addItems(["wem (fastest)", "wav (fast)", "mp3 (slow)", "ogg (slow)"])
self.assetMap.addItems(["No map", *[f'{e["game"]} - v{e["version"]}' for e in self.maps["maps"]]])
self.tabs.setTabEnabled(1, False)
self.tabs.setTabEnabled(2, False)
- self.loadFilesButton.clicked.connect(lambda: self.start())
+ self.loadFilesButton.clicked.connect(lambda: self.loadFiles())
- self.actionClearTreeView.triggered.connect(lambda: self.resetTreeWidget())
+ self.actionReset.triggered.connect(lambda: self.resetApp())
self.actionExit.triggered.connect(lambda: self.close())
self.actionExtractSelected.triggered.connect(lambda: self.extractItems(False))
@@ -109,14 +115,15 @@ class AnimeWwise(QMainWindow):
if data["action"] == "load":
self.fileStructure = data["content"]
self.updateTreeWidget()
- self.tabs.setTabEnabled(0, True)
+ self.tabs.setTabEnabled(0, False)
self.tabs.setTabEnabled(1, True)
+ self.tabs.setTabEnabled(2, True)
self.tabs.setCurrentIndex(1)
# page 1 - config
- def start(self):
- if "" in [self.folders["input"], self.folders["output"]]:
- QMessageBox.warning(None, "Warning", "Missing input/output folder !", QMessageBox.Ok)
+ def loadFiles(self):
+ if self.folders["input"] == "":
+ QMessageBox.warning(None, "Warning", "Missing input folder !", QMessageBox.Ok)
return
_map = self.assetMap.currentIndex()
@@ -128,17 +135,18 @@ class AnimeWwise(QMainWindow):
self.tabs.setTabEnabled(0, False)
self.resetTreeWidget()
- self.extractThread = QThread()
- self.extractWorker = BackgroundWorker("load", self.folders, _map, self.outputFormat.currentText())
- self.extractWorker.moveToThread(self.extractThread)
- self.extractThread.started.connect(self.extractWorker.run)
- self.extractWorker.finished.connect(self.handleFinished)
- self.extractWorker.finished.connect(self.extractThread.quit)
- self.extractWorker.finished.connect(self.extractWorker.deleteLater)
- self.extractThread.finished.connect(self.extractThread.deleteLater)
+ # why is all this required for threading damnit
+ self.backgroundThread = QThread()
+ self.backgroundWorker = BackgroundWorker("load", self.extract, {"input": self.folders["input"], "map": _map})
+ self.backgroundWorker.moveToThread(self.backgroundThread)
+ self.backgroundThread.started.connect(self.backgroundWorker.run)
+ self.backgroundWorker.finished.connect(self.handleFinished)
+ self.backgroundWorker.finished.connect(self.backgroundThread.quit)
+ self.backgroundWorker.finished.connect(self.backgroundWorker.deleteLater)
+ self.backgroundThread.finished.connect(self.backgroundThread.deleteLater)
- self.extractWorker.progress.connect(self.progressBarSlot)
- self.extractThread.start()
+ self.backgroundWorker.progress.connect(self.progressBarSlot)
+ self.backgroundThread.start()
# page 2 - browsing
def resetTreeWidget(self):
@@ -174,7 +182,7 @@ class AnimeWwise(QMainWindow):
self.addItems(folder_item, folder_content)
for file in sorted(element.get("files", [])):
- file_item = QTreeWidgetItem([str(file[0]), str(file[1]), str(file[2])])
+ file_item = QTreeWidgetItem([str(file[0]), str(hex(file[1])), str(file[2])])
file_item.setFlags(file_item.flags() | Qt.ItemIsUserCheckable)
file_item.setCheckState(0, Qt.Unchecked)
if parent is None:
@@ -197,7 +205,26 @@ class AnimeWwise(QMainWindow):
print(checked_items)
+ # yet another block of threading bs
+ self.backgroundThread = QThread()
+ self.backgroundWorker = BackgroundWorker("extract", self.extract, {"input": self.folders["input"], "files": checked_items})
+ self.backgroundWorker.moveToThread(self.backgroundThread)
+ self.backgroundThread.started.connect(self.backgroundWorker.run)
+ self.backgroundWorker.finished.connect(self.handleFinished)
+ self.backgroundWorker.finished.connect(self.backgroundThread.quit)
+ self.backgroundWorker.finished.connect(self.backgroundWorker.deleteLater)
+ self.backgroundThread.finished.connect(self.backgroundThread.deleteLater)
+
+ self.backgroundWorker.progress.connect(self.progressBarSlot)
+ self.backgroundThread.start()
+
# misc
+ def resetApp(self):
+ self.resetTreeWidget()
+ self.tabs.setTabEnabled(0, True)
+ self.tabs.setTabEnabled(1, False)
+ self.tabs.setTabEnabled(2, False)
+
def _appendText(self, text):
cursor = self.console.textCursor()
cursor.movePosition(cursor.End)
diff --git a/extract.py b/extract.py
index 34ff96d..d8b72b1 100644
--- a/extract.py
+++ b/extract.py
@@ -9,10 +9,12 @@ import tempfile
import wavescan
import subprocess
from filereader import FileReader
+from allocator import Allocator
cwd = os.getcwd()
path = lambda path: os.path.join(cwd, path)
call = lambda args: subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+allocator = Allocator()
skips = "000000000" # used for debugging
@@ -27,18 +29,18 @@ skips = "000000000" # used for debugging
# 9 - temp clean up
class WwiseExtract:
- def __init__(self, _map, _format, input_folder, output_folder, diff_folder, progress):
- self.map = _map
- self.format = _format
+ def __init__(self):
+ # self.map = _map
+ # self.format = _format
self.paths = {
- "input": input_folder,
- "output": output_folder,
- "diff": diff_folder,
+ "input": "",
+ "output": "",
+ "diff": "",
"temp": tempfile.TemporaryDirectory()
}
- self.progress = progress
+ # self.progress = progress
def path(self, base, path):
base_path = self.paths[base]
@@ -46,7 +48,7 @@ class WwiseExtract:
base_path = base_path.name
return os.path.join(base_path, path)
- def extract(self):
+ def extract2(self):
audio_format = self.format
mapper = self.map
_p = self.path # lazy
@@ -305,18 +307,18 @@ class WwiseExtract:
print("Done extracting everything !")
- ### new content ###
+ ### loading files ###
- def load_folder(self):
+ def load_folder(self, _map, path):
self.mapper = None
- if self.map is not None:
- self.mapper = Mapper(os.path.join(os.getcwd(), f"maps/{self.map}"))
+ if _map is not None:
+ self.mapper = Mapper(os.path.join(os.getcwd(), f"maps/{_map}"))
self.file_structure = {"folders": {}, "files": []}
- files = [f for f in os.listdir(self.paths["input"]) if f.endswith(".pck")]
+ files = [f for f in os.listdir(path) if f.endswith(".pck")]
for file in files:
- self.load_file(os.path.join(self.paths["input"], file))
+ self.load_file(os.path.join(path, file))
return self.file_structure
@@ -364,3 +366,11 @@ class WwiseExtract:
if "files" not in current_level:
current_level["files"] = []
current_level["files"].append([parts[-1], meta[0], meta[1]])
+
+ ### extracting files ###
+
+ def extract_files(self, _input):
+ # load all pck, then extract each file and apply conversion if required
+ # allocator.load_file(os.path.join(_input, "2050.pck"))
+ # allocator.read_at("2050.pck", 0, 0)
+ pass
diff --git a/gui.ui b/gui.ui
index 844a1ac..e08efea 100644
--- a/gui.ui
+++ b/gui.ui
@@ -39,7 +39,7 @@
- 0
+ 2
true
@@ -75,15 +75,11 @@
-
-
-
-
-
- Output folder
+
-
+
+
+ false
-
-
- -
-
Select
@@ -102,17 +98,13 @@
- -
-
-
- Select
+
-
+
+
+ false
-
-
- -
-
-
- Input folder
+
+ true
@@ -123,6 +115,13 @@
+ -
+
+
+ Input folder
+
+
+
-
@@ -136,20 +135,6 @@
- -
-
-
- true
-
-
-
- -
-
-
- true
-
-
-
-
@@ -161,13 +146,6 @@
-
-
-
-
-
- Output format
-
-
-
-
@@ -178,13 +156,6 @@
- -
-
-
-
-
-
-
-
@@ -231,52 +202,117 @@
Extract
-
+
- 153
- 110
- 521
- 201
+ 9
+ 9
+ 1061
+ 601
-
+
-
-
+
-
-
+
- Progress
+ Output folder
-
-
-
- 0
+
+
+ true
+
+
+
+ -
+
+
+ Select
-
-
+
-
-
+
- Task Progress
+ Output format
-
-
-
- 0
+
+
+
+ -
+
+
-
+
+
-
+
+
+ Total progress
+
+
+
+ -
+
+
+ 0
+
+
+
+
+
+ -
+
+
-
+
+
+ Per file progress
+
+
+
+ -
+
+
+ 0
+
+
+
+
+
+ -
+
+
-
+
+
+ Extract All
+
+
+
+ -
+
+
+ Extract Selected
+
+
+
+
+
+
+
@@ -320,19 +356,19 @@
File
-
+
@@ -342,19 +378,19 @@
not working here yet
-
+
All files
-
+
Selected files
-
+
- Clear Table
+ Reset
@@ -368,8 +404,6 @@
changeInput
altInputPath
changeAltInput
- outputPath
- changeOutput
loadFilesButton
tabs