map v3 and update hsr to 3.7

mapper.py

@@ -1,39 +1,33 @@
 # reader for the .map format i've made to improve reading speed and mapping size
+import io
+import json
 from filereader import FileReader
 
 
 class Mapper:
 	def __init__(self, mapping_file):
 		file = open(mapping_file, "rb")
-		reader = FileReader(file, "little") # encoded as little
+		self.data = file.read()
+		file.close()
+
+		reader = FileReader(io.BytesIO(self.data), "little")
 
 		# check file
 		if reader.ReadBytes(4) != b"ESFM":
-			file.close()
 			raise Exception("mapping was invalid")
 
 		reader.ReadBytes(2)
 
 		map_version = reader.ReadBytes(2)
-		if map_version != b"\x32\x31":
+		if map_version != b"\x33\x30":
 			print(f"Warning: you are using an unknown / unsupported version of the mapping that is no longer supported, please use a newer one or download an older version of this tool.")
 			raise Exception("incompatible mapping")
 
-		self.reader = reader
-		self.process_map()
-
-	def process_map(self):
-		reader = self.reader
-
-		# utils
-		val = lambda length: vl2(reader.ReadBytes(length))
-		vl2 = lambda data: int.from_bytes(data, "little")
-		raw = lambda length: rw2(reader.ReadBytes(length))
-		rw2 = lambda data: data.rstrip(b"\x00").decode("utf-8")
-
-		# get map meta
 		reader.ReadBytes(2)
 
+		self.process_map(reader)
+
+	def process_map(self, reader):
 		games = {
 			"hk4e": "Genshin",
 			"hkrpg": "Star Rail",
@@ -41,156 +35,72 @@ class Mapper:
 			# more later
 		}
 
-		coverages = [
-			"english voicelines",
-			"chinese voicelines",
-			"japanese voicelines",
-			"korean voicelines",
-			"music",
-			"sfx"
-		]
-
-		# read sectors
-		sectors_signature = reader.ReadBytes(9)
-		if sectors_signature != b"\xFF\x53\x45\x43\x54\x4F\x52\x53\xFF": # ff sectors ff
-			raise Exception("invalid mapping sectors signature")
-
-		n_sectors = val(1)
-		sectors = {}
-
-		for i in range(n_sectors):
-			name_length = val(1)
-			name = raw(name_length)
-			offset = val(4)
-			size = val(4)
-
-			sectors[name] = {
-				"offset": offset,
-				"size": size
-			}
-
 		# read config
-		reader.SetBufferPos(sectors["HEADER"]["offset"])
+		game_size = reader.ReadInt8()
-
+		game = reader.ReadBytes(game_size).decode("utf-8")
-		header_sig = reader.ReadBytes(8) # hardcoded but lazy, this value is for this sector only
-
-		n_configs = val(1)
-		config = {}
-		for i in range(n_configs):
-			name = raw(4)
-			value = raw(5)
-			config[name] = value
 
 		infos = {
-			"game": games[config["game"]],
+			"game": games[game],
-			"version": config["verS"],
+			"version": ".".join(list(str(reader.ReadInt8())))
-			# "coverage": config["covR"],
-			"useBanksSector": config["bnkS"],
-			# "bankSectorCoverage": config["bCov"]
 		}
 
 		print(f"> Loading mapping for {infos['game']} v{infos['version']}, this may take a few seconds...")
 
-		# read prefixes
+		# sectors
-		prefixes = {}
+		int24 = lambda: int.from_bytes(reader.ReadBytes(3))
-		n_prefixes = reader.ReadUInt8()
-		l_prefixes = reader.ReadUInt8()
 
-		for i in range(n_prefixes):
+		sectors = {
-			prefix = raw(l_prefixes)
+			# offset | size
-			marker = reader.ReadBytes(1)
+			"languages": [int24(), int24()],
-			prefixes[marker] = prefix
+			"strings": [int24(), int24()],
+			"words": [int24(), int24()],
+			"files": [int24(), int24()],
+			"keys": [int24(), int24()],
+			"music": [int24(), int24()]
+		}
 
-		# sector jump here
+		# languages
-		reader.SetBufferPos(sectors["ITEMS"]["offset"])
+		reader.SetBufferPos(sectors["languages"][0])
+		self.languages = []
 
-		items_sec_sig = reader.ReadBytes(7) # hardcoded too
+		n_langs = reader.ReadInt8()
-
-		# read languages
-		langs_offsets = {}
-		n_langs = reader.ReadUInt8()
-		l_langs = reader.ReadUInt8()
-		
 		for i in range(n_langs):
-			offset = reader.GetBufferPos()
+			size = reader.ReadInt8()
-			langs_offsets[offset] = raw(l_langs)
+			name = reader.ReadBytes(size).decode("utf-8")
+			self.languages.append(name)
 
-		self.langs_offsets = langs_offsets
+		# alloc sectors
-
+		reader.SetBufferPos(sectors["strings"][0])
-		# read folders
+		self.strings = bytearray(reader.ReadBytes(sectors["strings"][1]))
-		folder_offsets = {}
+		reader.SetBufferPos(sectors["words"][0])
-		n_folders = reader.ReadUInt16()
+		self.words = bytearray(reader.ReadBytes(sectors["words"][1]))
-
+		reader.SetBufferPos(sectors["files"][0])
-		for i in range(n_folders):
+		self.files = bytearray(reader.ReadBytes(sectors["files"][1]))
-			offset = reader.GetBufferPos()
-			length = reader.ReadUInt8()
-			prefix = reader.ReadBytes(1)
-			folder = raw(length)
-			folder = f"{prefixes[prefix]}{folder}"
-			folder_offsets[offset] = folder
-
-		# read files
-		files_offsets = {}
-		n_files = val(3)
-
-		for i in range(n_files):
-			offset = reader.GetBufferPos()
-			path_length = reader.ReadUInt8()
-			path = []
-			for i in range(path_length):
-				path.append(folder_offsets[reader.ReadUInt16()])
-			
-			name_length = reader.ReadUInt8()
-			prefix = reader.ReadBytes(1)
-			if prefix != b"\x00":
-				prefix = prefixes[prefix]
-			else:
-				prefix = ""
-			name = raw(name_length)
-			
-			name = f"{prefix}{name}"
-			path.append(name)
-			path = "\\".join(path)
-
-			files_offsets[offset] = path
-
-		self.files_offsets = files_offsets
 
 		# read keys
-		# GI 3649050 (outdated value, use items sector size instead)
+		reader.SetBufferPos(sectors["keys"][0])
-		keys_data = {}
+		key_size = reader.ReadInt8()
-		n_keys = val(3)
+		n_keys = (sectors["keys"][1]-1) // key_size
+		n_files = n_keys // n_langs
 
-		left = reader.GetRemainingLength()
+		keys_data = bytearray(reader.ReadBytes(sectors["keys"][1]-1))
-		if infos["useBanksSector"] == "TRUE":
+		self.keys = {keys_data[i+3:i+key_size].hex(): int.from_bytes(keys_data[i:i+3]) for i in range(0, len(keys_data), key_size)}
-			left -= sectors["BANKS"]["size"]
 
-		data = bytearray(reader.ReadBytes(left))
+		# music
-		keys_data = {rw2(data[i:i+16]): bytes(data[i+16:i+21]) for i in range(0, len(data), 21)}
+		self.music_keys = {}
+		hasMusic = sectors["music"][1] > 0
 
-		self.keys_data = keys_data
+		if hasMusic:
+			reader.SetBufferPos(sectors["music"][0])
+			root_size = reader.ReadInt8()
+			root = reader.ReadBytes(root_size).decode("utf-8")
+			n_music = int.from_bytes(reader.ReadBytes(2))
 
-		# read banks sector
+			for i in range(n_music):
-		bank_keys = {}
+				key = int.from_bytes(reader.ReadBytes(4))
-		if infos["useBanksSector"] == "TRUE":
+				name_size = reader.ReadInt8()
-			reader.SetBufferPos(sectors["BANKS"]["offset"])
+				name = reader.ReadBytes(name_size).decode("utf-8")
-
+				self.music_keys[str(key)] = f"{root}\\{name}"
-			banks_sec_sig = reader.ReadBytes(7) # hardcoded
-
-			global_path_size = val(1)
-			global_path = raw(global_path_size)
-
-			n_bank_keys = val(2)
-
-			for i in range(n_bank_keys):
-				key_length = val(1)
-				key = raw(key_length)
-				value_length = val(1)
-				value = raw(value_length)
-
-				bank_keys[key] = f"{global_path}\\{value}"
-
-		self.bank_keys = bank_keys
 
 		# done
 		print(f"> Finished loading mapping")
@@ -198,30 +108,49 @@ class Mapper:
 		print(f": {n_langs} languages")
 		print(f": {n_files} mapped files")
 		print(f": {n_keys} keys")
-		if infos["useBanksSector"] == "TRUE":
+		if hasMusic:
-			print(f"=-=-= Music sector =-=-=")
+			print(f": {n_music} musics")
-			print(f": {n_bank_keys} keys")
 
 	def get_key(self, key, lang=False):
-		keys_data = self.keys_data
+		if (not key in self.keys) and (not key in self.music_keys):
-		banks_data = self.bank_keys
+			return None
 
-		if key in keys_data.keys():
+		if key in self.music_keys:
-			key_data = keys_data[key]
+			return [self.music_keys[key], ""]
-			data = [self.files_offsets[int.from_bytes(key_data[2:], "little")]]
 
-			if lang:
+		lang, offset = (self.keys[key] >> 22) & 0x03, self.keys[key] & 0x3FFFFF
-				data.append(self.langs_offsets[int.from_bytes(key_data[:1], "little")])
 
-			return data
+		parts = int.from_bytes(self.files[offset:offset+1])
+		name = []
 
-		if key in banks_data.keys():
+		for i in range(parts):
-			return [banks_data[str(key)], ""]
+			word_offset = int.from_bytes(self.files[offset+1+(3*i):offset+4+(3*i)])
+			word_parts = int.from_bytes(self.words[word_offset:word_offset+1])
+			word = []
 
-		return None
+			for j in range(word_parts):
+				string_offset = int.from_bytes(self.words[word_offset+1+(2*j):word_offset+3+(2*j)])
+				string_size = int.from_bytes(self.strings[string_offset:string_offset+1])
+				if string_size > 128:
+					string = str(int.from_bytes(self.strings[string_offset+1:string_offset+1+(string_size-128)]))
+				else:
+					string = self.strings[string_offset+1:string_offset+1+string_size].decode("utf-8")
+				word.append(string)
+
+			word = "_".join(word)
+			name.append(word)
+
+		name = ["\\".join(name)]
+		
+		if lang:
+			name.append(self.languages[lang])
+
+		return name
 
 	def reset(self):
 		self.reader = None
-		self.langs_offsets.clear()
+		self.languages.clear()
-		self.files_offsets.clear()
+		self.strings.clear()
-		self.keys_data.clear()
+		self.words.clear()
+		self.files.clear()
+		self.music_keys.clear()

version.json

@@ -1,6 +1,6 @@
 {
     "version": 221,
-    "mapsVersion": 119,
+    "mapsVersion": 120,
     "maps": [
         {
             "name": "hk4e.map",
@@ -10,7 +10,7 @@
         {
             "name": "hkrpg.map",
             "game": "Honkai: Star Rail",
-            "version": "3.6"
+            "version": "3.7"
         },
         {
             "name": "nap.map",