lot of fixes and progress

2026-06-05 07:50:23 +08:00 · 2024-07-23 13:01:57 +02:00
parent 990ffb8901
commit 6472ce2b01
4 changed files with 210 additions and 114 deletions
--- a/allocator.py
+++ b/allocator.py
@@ -0,0 +1,25 @@
+# memory manager to prevent redundant calls to files and save up disk usage
+# may cause massive ram usage if we input too many pck at once, fix required
+import os
+import mmap
+
+class Allocator:
+	def __init__(self):
+		self.files = {}
+
+	def load_file(self, path):
+		filename = os.path.basename(path)
+		with open(path, "r+b") as f:
+			mmap_object = mmap.mmap(f.fileno(), 0)
+
+		self.files[filename] = mmap_object,
+
+
+	def read_at(self, file, offset, size):
+		data = self.files[file][offset:offset+size]
+		return data
+
+	def free_mem(self):
+		for file in list(self.files.keys()):
+			self.files[file].close()
+		self.files.clear()
--- a/app.py
+++ b/app.py
@@ -9,7 +9,7 @@ from PyQt5.QtGui import QTextCursor, QStandardItemModel, QStandardItem
 from PyQt5.QtWidgets import QMessageBox, QMainWindow, QApplication, QFileDialog, QHeaderView, QAbstractItemView, QTreeWidgetItem
 from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QThread, QMetaType, Qt

-QMetaType.type('QTextCursor')
+QMetaType.type("QTextCursor")

 class TextEditStream(QObject):
 	append_text = pyqtSignal(str)
@@ -34,20 +34,25 @@ class BackgroundWorker(QObject):
 	finished = pyqtSignal(dict)
 	progress = pyqtSignal(list)

-	def __init__(self, action, folders, _map, _format):
+	def __init__(self, action, extract, data):
 		super().__init__()
 		self.action = action
-		# TODO: dynamic args depending on action
-		self.folders = folders
-		self.map = _map
-		self.format = _format
+		self.extract = extract
+
+		if action == "load":
+			self.input = data["input"]
+			self.map = data["map"]
+		if action == "extract":
+			self.input = data["input"]

 	def run(self):
 		if self.action == "load":
 			print("Loading files and mapping if necessary...")
-			fileStructure = extract.WwiseExtract(self.map, "mp3", *self.folders.values(), progress=self.progress.emit).load_folder()
+			fileStructure = self.extract.load_folder(self.map, self.input)
 			print("Done !")
 			self.finished.emit({"action": "load", "content": fileStructure})
+		if self.action == "extract":
+			self.extract.extract_files(self.input)

 class AnimeWwise(QMainWindow):
 	def __init__(self):
@@ -61,6 +66,7 @@ class AnimeWwise(QMainWindow):
 		}
 		self.setupActions()
 		sys.stdout = TextEditStream(self.console)
+		self.extract = extract.WwiseExtract()

 		# utils
 		self.selectFolder = lambda: QFileDialog.getExistingDirectory(self, "Select Folder")
@@ -82,15 +88,15 @@ class AnimeWwise(QMainWindow):
 		self.changeAltInput.clicked.connect(lambda: self.setFolder(self.altInputPath, "diff"))
 		self.changeOutput.clicked.connect(lambda: self.setFolder(self.outputPath, "output"))

-		self.outputFormat.addItems(["mp3", "ogg"])
+		self.outputFormat.addItems(["wem (fastest)", "wav (fast)", "mp3 (slow)", "ogg (slow)"])
 		self.assetMap.addItems(["No map", *[f'{e["game"]} - v{e["version"]}' for e in self.maps["maps"]]])

 		self.tabs.setTabEnabled(1, False)
 		self.tabs.setTabEnabled(2, False)

-		self.loadFilesButton.clicked.connect(lambda: self.start())
+		self.loadFilesButton.clicked.connect(lambda: self.loadFiles())

-		self.actionClearTreeView.triggered.connect(lambda: self.resetTreeWidget())
+		self.actionReset.triggered.connect(lambda: self.resetApp())
 		self.actionExit.triggered.connect(lambda: self.close())

 		self.actionExtractSelected.triggered.connect(lambda: self.extractItems(False))
@@ -109,14 +115,15 @@ class AnimeWwise(QMainWindow):
 		if data["action"] == "load":
 			self.fileStructure = data["content"]
 			self.updateTreeWidget()
-			self.tabs.setTabEnabled(0, True)
+			self.tabs.setTabEnabled(0, False)
 			self.tabs.setTabEnabled(1, True)
+			self.tabs.setTabEnabled(2, True)
 			self.tabs.setCurrentIndex(1)

 	# page 1 - config
-	def start(self):
-		if "" in [self.folders["input"], self.folders["output"]]:
-			QMessageBox.warning(None, "Warning", "Missing input/output folder !", QMessageBox.Ok)
+	def loadFiles(self):
+		if self.folders["input"] == "":
+			QMessageBox.warning(None, "Warning", "Missing input folder !", QMessageBox.Ok)
 			return

 		_map = self.assetMap.currentIndex()
@@ -128,17 +135,18 @@ class AnimeWwise(QMainWindow):
 		self.tabs.setTabEnabled(0, False)
 		self.resetTreeWidget()

-		self.extractThread = QThread()
-		self.extractWorker = BackgroundWorker("load", self.folders, _map, self.outputFormat.currentText())
-		self.extractWorker.moveToThread(self.extractThread)
-		self.extractThread.started.connect(self.extractWorker.run)
-		self.extractWorker.finished.connect(self.handleFinished)
-		self.extractWorker.finished.connect(self.extractThread.quit)
-		self.extractWorker.finished.connect(self.extractWorker.deleteLater)
-		self.extractThread.finished.connect(self.extractThread.deleteLater)
+		# why is all this required for threading damnit
+		self.backgroundThread = QThread()
+		self.backgroundWorker = BackgroundWorker("load", self.extract, {"input": self.folders["input"], "map": _map})
+		self.backgroundWorker.moveToThread(self.backgroundThread)
+		self.backgroundThread.started.connect(self.backgroundWorker.run)
+		self.backgroundWorker.finished.connect(self.handleFinished)
+		self.backgroundWorker.finished.connect(self.backgroundThread.quit)
+		self.backgroundWorker.finished.connect(self.backgroundWorker.deleteLater)
+		self.backgroundThread.finished.connect(self.backgroundThread.deleteLater)

-		self.extractWorker.progress.connect(self.progressBarSlot)
-		self.extractThread.start()
+		self.backgroundWorker.progress.connect(self.progressBarSlot)
+		self.backgroundThread.start()

 	# page 2 - browsing
 	def resetTreeWidget(self):
@@ -174,7 +182,7 @@ class AnimeWwise(QMainWindow):
 			self.addItems(folder_item, folder_content)

 		for file in sorted(element.get("files", [])):
-			file_item = QTreeWidgetItem([str(file[0]), str(file[1]), str(file[2])])
+			file_item = QTreeWidgetItem([str(file[0]), str(hex(file[1])), str(file[2])])
 			file_item.setFlags(file_item.flags() | Qt.ItemIsUserCheckable)
 			file_item.setCheckState(0, Qt.Unchecked)
 			if parent is None:
@@ -197,7 +205,26 @@ class AnimeWwise(QMainWindow):
 		
 		print(checked_items)

+		# yet another block of threading bs
+		self.backgroundThread = QThread()
+		self.backgroundWorker = BackgroundWorker("extract", self.extract, {"input": self.folders["input"], "files": checked_items})
+		self.backgroundWorker.moveToThread(self.backgroundThread)
+		self.backgroundThread.started.connect(self.backgroundWorker.run)
+		self.backgroundWorker.finished.connect(self.handleFinished)
+		self.backgroundWorker.finished.connect(self.backgroundThread.quit)
+		self.backgroundWorker.finished.connect(self.backgroundWorker.deleteLater)
+		self.backgroundThread.finished.connect(self.backgroundThread.deleteLater)
+
+		self.backgroundWorker.progress.connect(self.progressBarSlot)
+		self.backgroundThread.start()
+
 	# misc
+	def resetApp(self):
+		self.resetTreeWidget()
+		self.tabs.setTabEnabled(0, True)
+		self.tabs.setTabEnabled(1, False)
+		self.tabs.setTabEnabled(2, False)
+
 	def _appendText(self, text):
 		cursor = self.console.textCursor()
 		cursor.movePosition(cursor.End)
--- a/extract.py
+++ b/extract.py
@@ -9,10 +9,12 @@ import tempfile
 import wavescan
 import subprocess
 from filereader import FileReader
+from allocator import Allocator

 cwd = os.getcwd()
 path = lambda path: os.path.join(cwd, path)
 call = lambda args: subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+allocator = Allocator()

 skips = "000000000" # used for debugging

@@ -27,18 +29,18 @@ skips = "000000000" # used for debugging
 # 9 - temp clean up

 class WwiseExtract:
-	def __init__(self, _map, _format, input_folder, output_folder, diff_folder, progress):
-		self.map = _map
-		self.format = _format
+	def __init__(self):
+		# self.map = _map
+		# self.format = _format

 		self.paths = {
-			"input": input_folder,
-			"output": output_folder,
-			"diff": diff_folder,
+			"input": "",
+			"output": "",
+			"diff": "",
 			"temp": tempfile.TemporaryDirectory()
 		}

-		self.progress = progress
+		# self.progress = progress

 	def path(self, base, path):
 		base_path = self.paths[base]
@@ -46,7 +48,7 @@ class WwiseExtract:
 			base_path = base_path.name
 		return os.path.join(base_path, path)

-	def extract(self):
+	def extract2(self):
 		audio_format = self.format
 		mapper = self.map
 		_p = self.path # lazy
@@ -305,18 +307,18 @@ class WwiseExtract:
 		print("Done extracting everything !")


-	### new content ###
+	### loading files ###

-	def load_folder(self):
+	def load_folder(self, _map, path):
 		self.mapper = None
-		if self.map is not None:
-			self.mapper = Mapper(os.path.join(os.getcwd(), f"maps/{self.map}"))
+		if _map is not None:
+			self.mapper = Mapper(os.path.join(os.getcwd(), f"maps/{_map}"))
 		self.file_structure = {"folders": {}, "files": []}

-		files = [f for f in os.listdir(self.paths["input"]) if f.endswith(".pck")]
+		files = [f for f in os.listdir(path) if f.endswith(".pck")]

 		for file in files:
-			self.load_file(os.path.join(self.paths["input"], file))
+			self.load_file(os.path.join(path, file))

 		return self.file_structure

@@ -364,3 +366,11 @@ class WwiseExtract:
 		if "files" not in current_level:
 			current_level["files"] = []
 		current_level["files"].append([parts[-1], meta[0], meta[1]])
+
+	### extracting files ###
+
+	def extract_files(self, _input):
+		# load all pck, then extract each file and apply conversion if required
+		# allocator.load_file(os.path.join(_input, "2050.pck"))
+		# allocator.read_at("2050.pck", 0, 0)
+		pass
--- a/gui.ui
+++ b/gui.ui
@@ -39,7 +39,7 @@
     </rect>
    </property>
    <property name="currentIndex">
-     <number>0</number>
+     <number>2</number>
    </property>
    <property name="usesScrollButtons">
     <bool>true</bool>
@@ -75,15 +75,11 @@
      <layout class="QVBoxLayout" name="mainVLayout">
       <item>
        <layout class="QGridLayout" name="ioGrid">
-         <item row="2" column="0">
-          <widget class="QLabel" name="outputLabel">
-           <property name="text">
-            <string>Output folder</string>
+         <item row="1" column="2">
+          <widget class="QPushButton" name="changeAltInput">
+           <property name="enabled">
+            <bool>false</bool>
           </property>
-          </widget>
-         </item>
-         <item row="2" column="2">
-          <widget class="QPushButton" name="changeOutput">
           <property name="text">
            <string>Select</string>
           </property>
@@ -102,17 +98,13 @@
           </property>
          </widget>
         </item>
-         <item row="1" column="2">
-          <widget class="QPushButton" name="changeAltInput">
-           <property name="text">
-            <string>Select</string>
+         <item row="1" column="1">
+          <widget class="QLineEdit" name="altInputPath">
+           <property name="enabled">
+            <bool>false</bool>
           </property>
-          </widget>
-         </item>
-         <item row="0" column="0">
-          <widget class="QLabel" name="inputLabel">
-           <property name="text">
-            <string>Input folder</string>
+           <property name="readOnly">
+            <bool>true</bool>
           </property>
          </widget>
         </item>
@@ -123,6 +115,13 @@
           </property>
          </widget>
         </item>
+         <item row="0" column="0">
+          <widget class="QLabel" name="inputLabel">
+           <property name="text">
+            <string>Input folder</string>
+           </property>
+          </widget>
+         </item>
         <item row="0" column="1">
          <widget class="QLineEdit" name="inputPath">
           <property name="enabled">
@@ -136,20 +135,6 @@
           </property>
          </widget>
         </item>
-         <item row="1" column="1">
-          <widget class="QLineEdit" name="altInputPath">
-           <property name="readOnly">
-            <bool>true</bool>
-           </property>
-          </widget>
-         </item>
-         <item row="2" column="1">
-          <widget class="QLineEdit" name="outputPath">
-           <property name="readOnly">
-            <bool>true</bool>
-           </property>
-          </widget>
-         </item>
        </layout>
       </item>
       <item>
@@ -161,13 +146,6 @@
       </item>
       <item>
        <layout class="QGridLayout" name="settingsGrid">
-         <item row="2" column="0">
-          <widget class="QLabel" name="outputFormatLabel">
-           <property name="text">
-            <string>Output format</string>
-           </property>
-          </widget>
-         </item>
         <item row="0" column="1">
          <widget class="QComboBox" name="assetMap"/>
         </item>
@@ -178,13 +156,6 @@
           </property>
          </widget>
         </item>
-         <item row="2" column="1">
-          <widget class="QComboBox" name="outputFormat">
-           <property name="currentText">
-            <string/>
-           </property>
-          </widget>
-         </item>
        </layout>
       </item>
       <item>
@@ -231,52 +202,117 @@
     <attribute name="title">
      <string>Extract</string>
     </attribute>
-     <widget class="QWidget" name="layoutWidget">
+     <widget class="QWidget" name="verticalLayoutWidget_2">
      <property name="geometry">
       <rect>
-        <x>153</x>
-        <y>110</y>
-        <width>521</width>
-        <height>201</height>
+        <x>9</x>
+        <y>9</y>
+        <width>1061</width>
+        <height>601</height>
       </rect>
      </property>
-      <layout class="QVBoxLayout" name="progressWrapperLayout_2">
+      <layout class="QVBoxLayout" name="mainVLayout2">
       <item>
-        <layout class="QHBoxLayout" name="progressLayout_2">
+        <layout class="QHBoxLayout" name="outputLayout">
         <item>
-          <widget class="QLabel" name="progressLabel_2">
+          <widget class="QLabel" name="outputLabel">
           <property name="text">
-            <string>Progress</string>
+            <string>Output folder</string>
           </property>
          </widget>
         </item>
         <item>
-          <widget class="QProgressBar" name="progress_2">
-           <property name="value">
-            <number>0</number>
+          <widget class="QLineEdit" name="outputPath">
+           <property name="readOnly">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="changeOutput">
+           <property name="text">
+            <string>Select</string>
           </property>
          </widget>
         </item>
        </layout>
       </item>
       <item>
-        <layout class="QHBoxLayout" name="taskProgressLayout_2">
+        <layout class="QHBoxLayout" name="outputFormatLayout">
         <item>
-          <widget class="QLabel" name="taskProgressLabel_2">
+          <widget class="QLabel" name="outputFormatLabel">
           <property name="text">
-            <string>Task Progress</string>
+            <string>Output format</string>
           </property>
          </widget>
         </item>
         <item>
-          <widget class="QProgressBar" name="taskProgress_2">
-           <property name="value">
-            <number>0</number>
+          <widget class="QComboBox" name="outputFormat">
+           <property name="currentText">
+            <string/>
           </property>
          </widget>
         </item>
        </layout>
       </item>
+       <item>
+        <layout class="QVBoxLayout" name="progressWrapperLayout">
+         <item>
+          <layout class="QHBoxLayout" name="totalProgressLayout">
+           <item>
+            <widget class="QLabel" name="totalProgressLabel">
+             <property name="text">
+              <string>Total progress</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QProgressBar" name="totalProgress">
+             <property name="value">
+              <number>0</number>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+         <item>
+          <layout class="QHBoxLayout" name="fileProgressLayout">
+           <item>
+            <widget class="QLabel" name="fileProgressLabel">
+             <property name="text">
+              <string>Per file progress</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QProgressBar" name="fileProgress">
+             <property name="value">
+              <number>0</number>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+         <item>
+          <layout class="QHBoxLayout" name="extractLayout">
+           <item>
+            <widget class="QPushButton" name="extractAll">
+             <property name="text">
+              <string>Extract All</string>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QPushButton" name="extractSelected">
+             <property name="text">
+              <string>Extract Selected</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+        </layout>
+       </item>
      </layout>
     </widget>
    </widget>
@@ -320,19 +356,19 @@
    <property name="title">
     <string>File</string>
    </property>
-    <addaction name="actionClearTreeView"/>
+    <addaction name="actionReset"/>
    <addaction name="separator"/>
    <addaction name="actionExit"/>
   </widget>
   <widget class="QMenu" name="menuExtract">
    <property name="enabled">
-     <bool>false</bool>
+     <bool>true</bool>
    </property>
    <property name="title">
     <string>Extract</string>
    </property>
-    <addaction name="actionall"/>
-    <addaction name="actionselected"/>
+    <addaction name="actionExtractAll"/>
+    <addaction name="actionExtractSelected"/>
   </widget>
   <addaction name="menuFile"/>
   <addaction name="menuExtract"/>
@@ -342,19 +378,19 @@
    <string>not working here yet</string>
   </property>
  </action>
-  <action name="actionall">
+  <action name="actionExtractAll">
   <property name="text">
    <string>All files</string>
   </property>
  </action>
-  <action name="actionselected">
+  <action name="actionExtractSelected">
   <property name="text">
    <string>Selected files</string>
   </property>
  </action>
-  <action name="actionClearTreeView">
+  <action name="actionReset">
   <property name="text">
-    <string>Clear Table</string>
+    <string>Reset</string>
   </property>
  </action>
  <action name="actionExit">
@@ -368,8 +404,6 @@
  <tabstop>changeInput</tabstop>
  <tabstop>altInputPath</tabstop>
  <tabstop>changeAltInput</tabstop>
-  <tabstop>outputPath</tabstop>
-  <tabstop>changeOutput</tabstop>
  <tabstop>loadFilesButton</tabstop>
  <tabstop>tabs</tabstop>
 </tabstops>