Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
print("Compiling YAML documents...")
pool.map(_do_yml, yml_files)
hashes = util.get_hash_table(util.get_settings("wiiu"))
print("Packing SARCs...")
_pack_sarcs(tmp_dir, hashes, pool)
for folder in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
_pack_sarcs(folder, hashes, pool)
for option_dir in tmp_dir.glob("options/*"):
for file in {
f
for f in option_dir.rglob("**/*")
if (f.is_file() and (tmp_dir / f.relative_to(option_dir)).exists())
}:
xh1 = xxhash.xxh64_intdigest(
(tmp_dir / file.relative_to(option_dir)).read_bytes()
)
xh2 = xxhash.xxh64_intdigest(file.read_bytes())
if xh1 == xh2:
util.vprint(
f"Removing {file} from option {option_dir.name}, identical to base mod"
)
file.unlink()
if not options:
options = {"disable": [], "options": {}}
options["options"]["texts"] = {"user_only": False}
try:
_make_bnp_logs(tmp_dir, options)
for option_dir in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
try:
old_sarc = oead.Sarc(util.unyaz_if_needed(stock_file.read_bytes()))
except (RuntimeError, ValueError, oead.InvalidDataError):
raise ValueError("Cannot open file from game dump")
old_files = {f.name for f in old_sarc.get_files()}
except (FileNotFoundError, ValueError):
for file in {f for f in folder.rglob("**/*") if f.is_file()}:
packed.files[file.relative_to(folder).as_posix()] = file.read_bytes()
else:
for file in {
f
for f in folder.rglob("**/*")
if f.is_file() and not f.suffix in EXCLUDE_EXTS
}:
file_data = file.read_bytes()
xhash = xxhash.xxh64_intdigest(util.unyaz_if_needed(file_data))
file_name = file.relative_to(folder).as_posix()
if file_name in old_files:
old_hash = xxhash.xxh64_intdigest(
util.unyaz_if_needed(old_sarc.get_file(file_name).data)
)
if file_name not in old_files or (
xhash != old_hash and file.suffix not in util.AAMP_EXTS
):
packed.files[file_name] = file_data
finally:
shutil.rmtree(folder)
if not packed.files:
return # pylint: disable=lost-exception
sarc_bytes = packed.write()[1]
folder.write_bytes(
util.compress(sarc_bytes)
def key_to_hash(key):
if isinstance(key, str):
return xxh64_intdigest(key.lower()) & 0xffffffffff
else:
return key
def murmur(x):
return np.uint64(xxhash.xxh64_intdigest(x))
def check_iter(self, paths):
"""Check paths from an iterable"""
# failsafe for common dumb error
if isinstance(paths, str):
raise TypeError("expected iterable of strings, got a string")
unknown = self.unknown
for p in paths:
h = xxh64_intdigest(p)
if h in unknown:
self._add_known(h, p)
hashes = util.get_hash_table(util.get_settings("wiiu"))
print("Packing SARCs...")
_pack_sarcs(tmp_dir, hashes, pool)
for folder in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
_pack_sarcs(folder, hashes, pool)
for option_dir in tmp_dir.glob("options/*"):
for file in {
f
for f in option_dir.rglob("**/*")
if (f.is_file() and (tmp_dir / f.relative_to(option_dir)).exists())
}:
xh1 = xxhash.xxh64_intdigest(
(tmp_dir / file.relative_to(option_dir)).read_bytes()
)
xh2 = xxhash.xxh64_intdigest(file.read_bytes())
if xh1 == xh2:
util.vprint(
f"Removing {file} from option {option_dir.name}, identical to base mod"
)
file.unlink()
if not options:
options = {"disable": [], "options": {}}
options["options"]["texts"] = {"user_only": False}
try:
_make_bnp_logs(tmp_dir, options)
for option_dir in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
_make_bnp_logs(option_dir, options)
except Exception as err: # pylint: disable=broad-except
pool.terminate()
def xxhash_u64_v1(): # 100 loops, best of 100: 700 usec per loop
for m1, m2 in prepare():
i = xxhash.xxh64_intdigest(TEXT[m1.end():m2.start()])
def check(self, p):
"""Check a single hash, print and add to known on match"""
h = xxh64_intdigest(p)
if h in self.unknown:
self._add_known(h, p)