updated mapping format

extract.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import mapper
 import shutil
 import filecmp
 import wavescan
@@ -9,9 +10,11 @@ from halo import Halo
 from progress.bar import PixelBar
 
 
+print("Setting up...")
 cwd = os.getcwd()
 path = lambda path: os.path.join(cwd, path)
 call = lambda args: subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+mapper.load_mapping(path("mapping/latest.map"))
 spinner = Halo(text="spinner", spinner={'interval': 100, 'frames': ['◜', '◠', '◝', '◞', '◡', '◟']}, placement="right")
 skips = "000000000" # used for debugging
 
@@ -250,15 +253,6 @@ def main():
 				spinner.text = f"[{curr}/{steps}] Mapping names"
 				spinner.start()
 
-				languages = ["english", "japanese", "chinese", "korean"]
-				mapFiles = [f"{path('mapping/mapping')}{f.capitalize()}.json" for f in languages]
-				namesTable = []
-
-				for language in mapFiles:
-					with open(language, "r") as f:
-						namesTable.append(json.loads(f.read()))
-						f.close()
-
 				if alone:
 					os.makedirs(path(f"temp/map/unmapped"), exist_ok=True)
 				else:
@@ -267,26 +261,9 @@ def main():
 					if len(changed_files) > 0:
 						os.makedirs(path(f"temp/map/changed_files/unmapped"), exist_ok=True)
 
-				# guess lang
 				lang = None
-				for file in all_files:
-					if lang is None or namesTable.index(language) == lang:
-						for language in namesTable:
-							if lang is None or namesTable.index(language) == lang:
-								file_name = file.split(".")[0]
-
-								if file_name in language:
-									# lang detected, stick to it
-									lang = namesTable.index(language)
-
-				print(f"\n: {languages[lang] if lang is not None else 'no'} language detected {'(not an error)' if lang is None else ''}")
 
 				for file in all_files:
-					if lang is None:
-						language = []
-					else:
-						language = namesTable[lang]
-
 					file_name = file.split(".")[0]
 					base_path = "temp/map"
 					if not alone:
@@ -295,11 +272,14 @@ def main():
 						elif file in changed_files:
 							base_path = "temp/map/changed_files"
 
-					if file_name in language:
+					key_data = mapper.get_key(file_name, lang is None)
-						# lang detected, stick to it
-						lang = namesTable.index(language)
 
-						dir_path = path(f"{base_path}/{language[file_name]['path']}/{language[file_name]['name']}.mp3")
+					if key_data is not None:
+						if lang is None:
+							lang = key_data[1]
+							print(f"\n: {lang} detected")
+
+						dir_path = path(f"{base_path}/{key_data[0]}.mp3")
 						os.makedirs(os.path.dirname(dir_path), exist_ok=True)
 						shutil.copy(path(f"temp/mp3/{file}"), dir_path)
 					else:

filereader.py

@@ -21,6 +21,12 @@ class FileReader:
 		return struct.unpack(f"{endianness}{mode}", bytearray(self.stream.read(bufferLength)))[0]
 
 	# read methods
+	def ReadInt8(self, endianness:str=None) -> int:
+		return self._read("b", 1, endianness)
+
+	def ReadUInt8(self, endianness:str=None) -> int:
+		return self._read("B", 1, endianness)
+
 	def ReadInt16(self, endianness:str=None) -> int:
 		return self._read("h", 2, endianness)
 	

mapper.py

@@ -0,0 +1,67 @@
+# reader for the .map format i've made to improve reading speed and mapping size
+from filereader import FileReader
+import os
+
+
+langs_offsets = {}
+keys_data = {}
+reader = None
+
+
+def load_mapping(mapping_file, langs=False):
+	global reader
+
+	file = open(mapping_file, "rb")
+	reader = FileReader(file, "little") # encoded as little
+
+	# check file
+	if reader.ReadBytes(4) != b"ESFM":
+		file.close()
+		raise Exception("mapping was invalid")
+
+	reader.ReadBytes(2)
+
+	if reader.ReadBytes(2) != b"\x56\x31":
+		file.close()
+		raise Exception("invalid mapping version")
+
+	reader.ReadBytes(2)
+
+	# read languages
+	n_langs = reader.ReadInt8()
+	for i in range(n_langs):
+		offset = reader.GetBufferPos()
+		lang_length = reader.ReadInt8()
+		langs_offsets[str(offset)] = reader.ReadBytes(lang_length).decode("utf-8")
+
+	# read keys
+	n_keys = reader.ReadInt32()
+	for i in range(n_keys):
+		key_length = reader.ReadInt8()
+		key = reader.ReadBytes(key_length).decode("utf-8")
+		lang_offset = reader.ReadInt8()
+		data_offset = reader.ReadInt32()
+		linked_lang = langs_offsets[str(lang_offset)]
+		keys_data[str(key)] = [linked_lang, data_offset]
+
+	if langs:
+		return list(langs_offsets.values())
+	return True
+
+
+def get_key(key, lang=False):
+	if key not in keys_data.keys():
+		return None
+
+	# search data
+	data = keys_data[key]
+	lang = data[0]
+
+	reader.SetBufferPos(data[1])
+	data_length = reader.ReadInt8()
+	data = [reader.ReadBytes(data_length).decode("utf-8")]
+
+	if lang:
+		data.append(lang)
+
+	return data