diff --git a/mapper.py b/mapper.py index 0cdace8..e1c408c 100644 --- a/mapper.py +++ b/mapper.py @@ -1,39 +1,33 @@ # reader for the .map format i've made to improve reading speed and mapping size +import io +import json from filereader import FileReader class Mapper: def __init__(self, mapping_file): file = open(mapping_file, "rb") - reader = FileReader(file, "little") # encoded as little + self.data = file.read() + file.close() + + reader = FileReader(io.BytesIO(self.data), "little") # check file if reader.ReadBytes(4) != b"ESFM": - file.close() raise Exception("mapping was invalid") reader.ReadBytes(2) map_version = reader.ReadBytes(2) - if map_version != b"\x32\x31": + if map_version != b"\x33\x30": print(f"Warning: you are using an unknown / unsupported version of the mapping that is no longer supported, please use a newer one or download an older version of this tool.") raise Exception("incompatible mapping") - self.reader = reader - self.process_map() - - def process_map(self): - reader = self.reader - - # utils - val = lambda length: vl2(reader.ReadBytes(length)) - vl2 = lambda data: int.from_bytes(data, "little") - raw = lambda length: rw2(reader.ReadBytes(length)) - rw2 = lambda data: data.rstrip(b"\x00").decode("utf-8") - - # get map meta reader.ReadBytes(2) + self.process_map(reader) + + def process_map(self, reader): games = { "hk4e": "Genshin", "hkrpg": "Star Rail", @@ -41,156 +35,72 @@ class Mapper: # more later } - coverages = [ - "english voicelines", - "chinese voicelines", - "japanese voicelines", - "korean voicelines", - "music", - "sfx" - ] - - # read sectors - sectors_signature = reader.ReadBytes(9) - if sectors_signature != b"\xFF\x53\x45\x43\x54\x4F\x52\x53\xFF": # ff sectors ff - raise Exception("invalid mapping sectors signature") - - n_sectors = val(1) - sectors = {} - - for i in range(n_sectors): - name_length = val(1) - name = raw(name_length) - offset = val(4) - size = val(4) - - sectors[name] = { - "offset": offset, - "size": size - } - # read config - reader.SetBufferPos(sectors["HEADER"]["offset"]) - - header_sig = reader.ReadBytes(8) # hardcoded but lazy, this value is for this sector only - - n_configs = val(1) - config = {} - for i in range(n_configs): - name = raw(4) - value = raw(5) - config[name] = value + game_size = reader.ReadInt8() + game = reader.ReadBytes(game_size).decode("utf-8") infos = { - "game": games[config["game"]], - "version": config["verS"], - # "coverage": config["covR"], - "useBanksSector": config["bnkS"], - # "bankSectorCoverage": config["bCov"] + "game": games[game], + "version": ".".join(list(str(reader.ReadInt8()))) } print(f"> Loading mapping for {infos['game']} v{infos['version']}, this may take a few seconds...") - # read prefixes - prefixes = {} - n_prefixes = reader.ReadUInt8() - l_prefixes = reader.ReadUInt8() + # sectors + int24 = lambda: int.from_bytes(reader.ReadBytes(3)) - for i in range(n_prefixes): - prefix = raw(l_prefixes) - marker = reader.ReadBytes(1) - prefixes[marker] = prefix + sectors = { + # offset | size + "languages": [int24(), int24()], + "strings": [int24(), int24()], + "words": [int24(), int24()], + "files": [int24(), int24()], + "keys": [int24(), int24()], + "music": [int24(), int24()] + } - # sector jump here - reader.SetBufferPos(sectors["ITEMS"]["offset"]) + # languages + reader.SetBufferPos(sectors["languages"][0]) + self.languages = [] - items_sec_sig = reader.ReadBytes(7) # hardcoded too - - # read languages - langs_offsets = {} - n_langs = reader.ReadUInt8() - l_langs = reader.ReadUInt8() - + n_langs = reader.ReadInt8() for i in range(n_langs): - offset = reader.GetBufferPos() - langs_offsets[offset] = raw(l_langs) + size = reader.ReadInt8() + name = reader.ReadBytes(size).decode("utf-8") + self.languages.append(name) - self.langs_offsets = langs_offsets - - # read folders - folder_offsets = {} - n_folders = reader.ReadUInt16() - - for i in range(n_folders): - offset = reader.GetBufferPos() - length = reader.ReadUInt8() - prefix = reader.ReadBytes(1) - folder = raw(length) - folder = f"{prefixes[prefix]}{folder}" - folder_offsets[offset] = folder - - # read files - files_offsets = {} - n_files = val(3) - - for i in range(n_files): - offset = reader.GetBufferPos() - path_length = reader.ReadUInt8() - path = [] - for i in range(path_length): - path.append(folder_offsets[reader.ReadUInt16()]) - - name_length = reader.ReadUInt8() - prefix = reader.ReadBytes(1) - if prefix != b"\x00": - prefix = prefixes[prefix] - else: - prefix = "" - name = raw(name_length) - - name = f"{prefix}{name}" - path.append(name) - path = "\\".join(path) - - files_offsets[offset] = path - - self.files_offsets = files_offsets + # alloc sectors + reader.SetBufferPos(sectors["strings"][0]) + self.strings = bytearray(reader.ReadBytes(sectors["strings"][1])) + reader.SetBufferPos(sectors["words"][0]) + self.words = bytearray(reader.ReadBytes(sectors["words"][1])) + reader.SetBufferPos(sectors["files"][0]) + self.files = bytearray(reader.ReadBytes(sectors["files"][1])) # read keys - # GI 3649050 (outdated value, use items sector size instead) - keys_data = {} - n_keys = val(3) + reader.SetBufferPos(sectors["keys"][0]) + key_size = reader.ReadInt8() + n_keys = (sectors["keys"][1]-1) // key_size + n_files = n_keys // n_langs - left = reader.GetRemainingLength() - if infos["useBanksSector"] == "TRUE": - left -= sectors["BANKS"]["size"] + keys_data = bytearray(reader.ReadBytes(sectors["keys"][1]-1)) + self.keys = {keys_data[i+3:i+key_size].hex(): int.from_bytes(keys_data[i:i+3]) for i in range(0, len(keys_data), key_size)} - data = bytearray(reader.ReadBytes(left)) - keys_data = {rw2(data[i:i+16]): bytes(data[i+16:i+21]) for i in range(0, len(data), 21)} + # music + self.music_keys = {} + hasMusic = sectors["music"][1] > 0 - self.keys_data = keys_data + if hasMusic: + reader.SetBufferPos(sectors["music"][0]) + root_size = reader.ReadInt8() + root = reader.ReadBytes(root_size).decode("utf-8") + n_music = int.from_bytes(reader.ReadBytes(2)) - # read banks sector - bank_keys = {} - if infos["useBanksSector"] == "TRUE": - reader.SetBufferPos(sectors["BANKS"]["offset"]) - - banks_sec_sig = reader.ReadBytes(7) # hardcoded - - global_path_size = val(1) - global_path = raw(global_path_size) - - n_bank_keys = val(2) - - for i in range(n_bank_keys): - key_length = val(1) - key = raw(key_length) - value_length = val(1) - value = raw(value_length) - - bank_keys[key] = f"{global_path}\\{value}" - - self.bank_keys = bank_keys + for i in range(n_music): + key = int.from_bytes(reader.ReadBytes(4)) + name_size = reader.ReadInt8() + name = reader.ReadBytes(name_size).decode("utf-8") + self.music_keys[str(key)] = f"{root}\\{name}" # done print(f"> Finished loading mapping") @@ -198,30 +108,49 @@ class Mapper: print(f": {n_langs} languages") print(f": {n_files} mapped files") print(f": {n_keys} keys") - if infos["useBanksSector"] == "TRUE": - print(f"=-=-= Music sector =-=-=") - print(f": {n_bank_keys} keys") + if hasMusic: + print(f": {n_music} musics") def get_key(self, key, lang=False): - keys_data = self.keys_data - banks_data = self.bank_keys + if (not key in self.keys) and (not key in self.music_keys): + return None - if key in keys_data.keys(): - key_data = keys_data[key] - data = [self.files_offsets[int.from_bytes(key_data[2:], "little")]] + if key in self.music_keys: + return [self.music_keys[key], ""] - if lang: - data.append(self.langs_offsets[int.from_bytes(key_data[:1], "little")]) + lang, offset = (self.keys[key] >> 22) & 0x03, self.keys[key] & 0x3FFFFF - return data + parts = int.from_bytes(self.files[offset:offset+1]) + name = [] - if key in banks_data.keys(): - return [banks_data[str(key)], ""] + for i in range(parts): + word_offset = int.from_bytes(self.files[offset+1+(3*i):offset+4+(3*i)]) + word_parts = int.from_bytes(self.words[word_offset:word_offset+1]) + word = [] - return None + for j in range(word_parts): + string_offset = int.from_bytes(self.words[word_offset+1+(2*j):word_offset+3+(2*j)]) + string_size = int.from_bytes(self.strings[string_offset:string_offset+1]) + if string_size > 128: + string = str(int.from_bytes(self.strings[string_offset+1:string_offset+1+(string_size-128)])) + else: + string = self.strings[string_offset+1:string_offset+1+string_size].decode("utf-8") + word.append(string) + + word = "_".join(word) + name.append(word) + + name = ["\\".join(name)] + + if lang: + name.append(self.languages[lang]) + + return name def reset(self): self.reader = None - self.langs_offsets.clear() - self.files_offsets.clear() - self.keys_data.clear() + self.languages.clear() + self.strings.clear() + self.words.clear() + self.files.clear() + self.music_keys.clear() diff --git a/maps/hk4e.map b/maps/hk4e.map index 3695174..50b61b6 100644 Binary files a/maps/hk4e.map and b/maps/hk4e.map differ diff --git a/maps/hkrpg.map b/maps/hkrpg.map index d03bf6b..4a9687c 100644 Binary files a/maps/hkrpg.map and b/maps/hkrpg.map differ diff --git a/maps/nap.map b/maps/nap.map index dbfc5c4..992192d 100644 Binary files a/maps/nap.map and b/maps/nap.map differ diff --git a/version.json b/version.json index 066530f..b3c17a1 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "version": 221, - "mapsVersion": 119, + "mapsVersion": 120, "maps": [ { "name": "hk4e.map", @@ -10,7 +10,7 @@ { "name": "hkrpg.map", "game": "Honkai: Star Rail", - "version": "3.6" + "version": "3.7" }, { "name": "nap.map",