optimized mapping format

extract.py

@@ -3,18 +3,18 @@ import sys
 import mapper
 import shutil
 import filecmp
+import argparse
 import wavescan
 import subprocess
 from halo import Halo
 from progress.bar import PixelBar
-import argparse
 
 
 print("Setting up...")
 cwd = os.getcwd()
 path = lambda path: os.path.join(cwd, path)
 call = lambda args: subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
-mapper.load_mapping(path("mapping/latest.map"))
+mapper = mapper.Mapper(path("mapping/latest.map"))
 spinner = Halo(text="spinner", spinner={'interval': 100, 'frames': ['◜', '◠', '◝', '◞', '◡', '◟']}, placement="right")
 skips = "000000000" # used for debugging
 
@@ -30,6 +30,7 @@ skips = "000000000" # used for debugging
 
 def main():
 	parser = argparse.ArgumentParser()
+	# TODO: add skip / select mapping option
 	parser.add_argument("--format", nargs="?", type=str, default="mp3", help="Output audio format, can be either mp3 or ogg")
 	args = parser.parse_args()
 
@@ -290,6 +291,7 @@ def main():
 					if key_data is not None:
 						if lang is None:
 							lang = key_data[1]
+							# TODO: use language for output path
 							print(f"\n: {lang} detected")
 
 						dir_path = path(f"{base_path}/{key_data[0]}.{audio_format}")

mapper.py

@@ -1,67 +1,136 @@
 # reader for the .map format i've made to improve reading speed and mapping size
 from filereader import FileReader
 import os
+import json
 
 
-langs_offsets = {}
+class Mapper:
-keys_data = {}
+	def __init__(self, mapping_file):
-reader = None
+		file = open(mapping_file, "rb")
+		reader = FileReader(file, "little") # encoded as little
+
+		# check file
+		if reader.ReadBytes(4) != b"ESFM":
+			file.close()
+			raise Exception("mapping was invalid")
+
+		reader.ReadBytes(2)
+
+		map_version = reader.ReadBytes(2)
+		if map_version == b"\x56\x31":
+			print(f"Warning: you are using an old version of the mapping that is no longer supported, please use a newer one or download an older version of this tool.")
+			raise Exception("outdated mapping")
+		elif map_version  == b"\x56\x32":
+			self.reader = reader
+			self.process_map()
+		else:
+			file.close()
+			raise Exception("invalid mapping version")
+
+	def process_map(self):
+		reader = self.reader
+
+		# utils
+		val = lambda length: int.from_bytes(reader.ReadBytes(length), "little")
+		raw = lambda length: reader.ReadBytes(length).rstrip(b"\x00").decode("utf-8")
+
+		# get map meta
+		reader.ReadBytes(2)
+
+		games = {
+			b"ys": "Genshin"
+			# more later
+		}
+
+		infos = {
+			"game": games[reader.ReadBytes(2)],
+			"version": list(raw(2)),
+			"null": reader.ReadBytes(4)
+			# more later
+		}
+
+		print(f"> Loading mapping for {infos['game']} v{infos['version'][0]}.{infos['version'][1]}, this may take a few seconds...")
+
+		# read prefixes
+		prefixes = {}
+		n_prefixes = reader.ReadUInt8()
+
+		for i in range(n_prefixes):
+			prefix = raw(4)
+			marker = reader.ReadBytes(1)
+			prefixes[marker] = prefix
+
+		# read languages
+		langs_offsets = {}
+		n_langs = reader.ReadUInt8()
+		
+		for i in range(n_langs):
+			offset = reader.GetBufferPos()
+			langs_offsets[offset] = raw(11)
+
+		# read folders
+		folder_offsets = {}
+		n_folders = reader.ReadUInt8()
+
+		for i in range(n_folders):
+			offset = reader.GetBufferPos()
+			length = reader.ReadUInt8()
+			prefix = reader.ReadBytes(1)
+			folder = raw(length)
+			folder = f"{prefixes[prefix]}{folder}"
+			folder_offsets[offset] = folder
+
+		# read files
+		files_offsets = {}
+		n_files = val(3)
+
+		for i in range(n_files):
+			offset = reader.GetBufferPos()
+			path_length = reader.ReadUInt8()
+			path = []
+			for i in range(path_length):
+				path.append(folder_offsets[reader.ReadUInt16()])
+			
+			name_length = reader.ReadUInt8()
+			prefix = prefixes[reader.ReadBytes(1)]
+			name = raw(name_length)
+			
+			name = f"{prefix}{name}"
+			path.append(name)
+			path = "\\".join(path)
+
+			files_offsets[offset] = path
+
+		# read keys
+		keys_data = {}
+		n_keys = val(3)
+
+		for i in range(n_keys):
+			key = raw(16)
+			
+			lang_offset = reader.ReadUInt8()
+			file_offset = val(3)
+
+			keys_data[key] = [files_offsets[file_offset], langs_offsets[lang_offset]]
+
+		self.keys_data = keys_data
+
+		# done
+		print(f"> Finished loading mapping")
+		print(f": {n_langs} supported languages")
+		print(f": {n_files} mapped files")
+		print(f": {n_keys} available keys")
 
 
-def load_mapping(mapping_file, langs=False):
+	def get_key(self, key, lang=False):
-	global reader
+		keys_data = self.keys_data
+		if key not in keys_data.keys():
+			return None
 
-	file = open(mapping_file, "rb")
+		key_data = keys_data[key]
-	reader = FileReader(file, "little") # encoded as little
+		data = [key_data[0]]
 
-	# check file
+		if lang:
-	if reader.ReadBytes(4) != b"ESFM":
+			data.append(key_data[1])
-		file.close()
-		raise Exception("mapping was invalid")
 
-	reader.ReadBytes(2)
+		return data
-
-	if reader.ReadBytes(2) != b"\x56\x31":
-		file.close()
-		raise Exception("invalid mapping version")
-
-	reader.ReadBytes(2)
-
-	# read languages
-	n_langs = reader.ReadInt8()
-	for i in range(n_langs):
-		offset = reader.GetBufferPos()
-		lang_length = reader.ReadInt8()
-		langs_offsets[str(offset)] = reader.ReadBytes(lang_length).decode("utf-8")
-
-	# read keys
-	n_keys = reader.ReadInt32()
-	for i in range(n_keys):
-		key_length = reader.ReadInt8()
-		key = reader.ReadBytes(key_length).decode("utf-8")
-		lang_offset = reader.ReadInt8()
-		data_offset = reader.ReadInt32()
-		linked_lang = langs_offsets[str(lang_offset)]
-		keys_data[str(key)] = [linked_lang, data_offset]
-
-	if langs:
-		return list(langs_offsets.values())
-	return True
-
-
-def get_key(key, lang=False):
-	if key not in keys_data.keys():
-		return None
-
-	# search data
-	data = keys_data[key]
-	lang = data[0]
-
-	reader.SetBufferPos(data[1])
-	data_length = reader.ReadInt8()
-	data = [reader.ReadBytes(data_length).decode("utf-8")]
-
-	if lang:
-		data.append(lang)
-
-	return data