From 1694304b95522959f135c1479b3b0e924e066a78 Mon Sep 17 00:00:00 2001 From: Ilia Petrov-Komotskii Date: Fri, 23 Oct 2026 13:48:52 +0200 Subject: [PATCH 1/6] feat: Added new extraction options: -x f and -x p, standard -x behaviour also changed --- CHANGELOG.md | 8 +++ README.md | 8 +++ src/archives.py | 85 +++++++++++++++++++++++++++++ src/operations.py | 22 ++++++-- src/romlm.py | 33 ++++++++--- tests/test_archives.py | 115 +++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 44 +++++++++++++++ tests/test_operations.py | 41 ++++++++++++++ tests/test_romlm.py | 12 +++- 9 files changed, 354 insertions(+), 14 deletions(-) create mode 100644 tests/test_archives.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 976441a..aac5e9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added +- Added extraction modes to `-x`/`--extract`: `-x f` extracts every archive into its own folder named after it, and + `-x p` extracts everything next to the archives. + +### Changed +- `-x` without a mode now gives an archive its own folder when it holds more than one file, or a file with a generic + name like `game.iso`, `Disc 1.cue` or `track01.bin`, instead of always extracting next to the archive. Use `-x p` + for the old behavior. `--dry-run` reads the archives to show where they would be extracted. ## [1.2.0] - 2026-10-16 ### Added diff --git a/README.md b/README.md index 8c1af05..bb67b22 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,14 @@ feel free to open an issue on the [GitHub issues](https://github.com/ManeFunctio - **Extract (`-x, --extract`)** Automatically unpacks all `.7z` or `.zip` files in the current directory (or a specified folder) into subfolders. Supports nested subfolders, keeping all the structure. + - `-x` extracts the files right next to the archive, but gives an archive its own folder (named after the + archive) when the files would make a mess otherwise: when there is more than one file inside (like `.cue` and + `.bin` tracks), or when the file has a generic name, telling nothing about the game, which is common for + disc-based systems: `game.iso`, `Disc 1.cue`, `CD2.bin`, `track01.bin`, `disk1.adf`, `Disk 1 Side A.st`, + `EBOOT.BIN`, `default.xbe`, `main.dol` and so on. An archive that already keeps everything in one properly + named folder is extracted as it is. + - `f` extracts every archive into its own folder. + - `p` is a plain extraction: all the files go right next to the archives. ![](https://raw.githubusercontent.com/wiki/ManeFunction/romlm/extract.png) diff --git a/src/archives.py b/src/archives.py index fbab097..7601f46 100644 --- a/src/archives.py +++ b/src/archives.py @@ -1,7 +1,9 @@ import os +import re import shutil import subprocess import zipfile +from enum import Enum from functools import lru_cache # Candidate binary names for a 7-Zip CLI, in preference order: "7zz" is what @@ -11,6 +13,27 @@ ARCHIVE_EXTENSIONS = (".7z", ".zip") +# System junk that comes along with files: it is cleaned after processing, and doesn't count as archive contents +META_FILES = ("desktop.ini", "thumbs.db", ".ds_store") +META_FOLDERS = ("__macosx",) + +# Names that tell nothing about the game, as disc-based dumps often have: "game.iso", "Disc 1.cue", +# "CD2.bin", "track01.bin" (Dreamcast GDI, cue/bin), "disk1.adf", "Disk 1 Side A.st" (floppies), +# "EBOOT.BIN" (PS3), "default.xbe" (Xbox), "boot.dol"/"main.dol" (GameCube, Wii). +# Such a name is made of these words only, each one can be numbered ("disc2", "Disk 1A", "Side B", "Disc 1 of 2"). +# A bare number counts too, if it's short ("01.bin"), so "1942.nes" is still a game. +GENERIC_WORDS = ("game", "rom", "image", "img", "iso", "disc", "disk", "cd", "cdrom", "dvd", "gd", "gdrom", "umd", + "floppy", "tape", "side", "track", "data", "default", "boot", "eboot", "main", "of") +GENERIC_NUMBER_LENGTH = 2 + +# A short extension, without spaces, so "Vol. 1 (USA)" is not taken for one +_EXTENSION = re.compile(r"\.[a-z0-9]{1,5}", re.IGNORECASE) + +class ExtractMode(Enum): + PLAIN = 1 # the contents go next to the archive + FOLDERS = 2 # every archive gets its own folder + AUTO = 3 # own folder only when the plain extraction would be a mess (see needs_own_folder) + @lru_cache(maxsize=None) def find_seven_zip_binary(): """Locate a usable 7-Zip CLI binary on PATH, or None if none is installed.""" @@ -23,6 +46,68 @@ def find_seven_zip_binary(): def is_archive(file_name) -> bool: return file_name.lower().endswith(ARCHIVE_EXTENSIONS) +def strip_extensions(name) -> str: + """The name without its extensions: "game.nkit.iso" -> "game".""" + stem, extension = os.path.splitext(name) + while stem and _EXTENSION.fullmatch(extension): + name = stem + stem, extension = os.path.splitext(name) + return name + +def is_generic_name(name) -> bool: + """Is the file (or folder) name telling nothing about the game, like "Disc 1.iso" or "track01.bin".""" + stem = strip_extensions(name).lower() + if not re.fullmatch(r"[a-z0-9\s._-]+", stem): + return False + previous = None + for token in re.findall(r"[a-z]+|\d+", stem): + is_number = token.isdigit() and (previous is not None or len(token) <= GENERIC_NUMBER_LENGTH) + is_letter = len(token) == 1 and previous is not None + if not (is_number or is_letter or token in GENERIC_WORDS): + return False + previous = token + return previous is not None + +def list_entries(file_name) -> list[str]: + """Paths of everything inside the archive, with '/' separators.""" + if file_name.lower().endswith(".zip"): + with zipfile.ZipFile(file_name, 'r') as archive: + return archive.namelist() + # The technical listing has one "Path = ..." line per entry, after the archive's own block ending with dashes + result = subprocess.run([find_seven_zip_binary(), "l", "-slt", os.path.abspath(file_name)], + check=True, capture_output=True) + _, _, entries = result.stdout.decode(errors="replace").partition("\n----------") + return [line[len("Path = "):].replace("\\", "/") for line in entries.splitlines() if line.startswith("Path = ")] + +def top_level_entries(entries) -> set[str]: + """Names of the files and folders that land right in the target folder, without the system junk.""" + names = set() + for entry in entries: + parts = [part for part in entry.split("/") if part not in ("", ".")] + if parts and parts[0].lower() not in META_FILES + META_FOLDERS: + names.add(parts[0]) + return names + +def needs_own_folder(top_level) -> bool: + """An archive with a few things inside, or one with a generic name, would clutter the folder it's extracted to.""" + return len(top_level) > 1 or any(is_generic_name(name) for name in top_level) + +def own_folder_name(file_name, top_level) -> str: + """The archive name without its extension, and without the inner file's extension too, if romlm packed it.""" + name = os.path.splitext(os.path.basename(file_name))[0] + if name in top_level: + name = os.path.splitext(name)[0] or name + return name + +def extraction_folder(file_name, target_folder, mode) -> str: + """Where the archive's contents should go, according to the mode. AUTO mode reads the archive to decide.""" + if mode == ExtractMode.PLAIN: + return target_folder + top_level = top_level_entries(list_entries(file_name)) + if mode == ExtractMode.AUTO and not needs_own_folder(top_level): + return target_folder + return os.path.join(target_folder, own_folder_name(file_name, top_level)) + def unpack(file_name, target_folder): """Extracts the archive into the folder, and removes the archive.""" if file_name.lower().endswith(".7z"): diff --git a/src/operations.py b/src/operations.py index 50b2615..8fece2f 100644 --- a/src/operations.py +++ b/src/operations.py @@ -21,11 +21,13 @@ class Operation: source: str target_folder: str packing_format: str = "7z" + extract_mode: archives.ExtractMode = archives.ExtractMode.PLAIN -def plan_operations(files_list, target_folder_for, is_unpacking_enabled, is_packing_enabled, packing_format) -> list: +def plan_operations(files_list, target_folder_for, is_unpacking_enabled, is_packing_enabled, packing_format, + extract_mode=archives.ExtractMode.PLAIN) -> list: """ Decides what to do with every file, without touching the disk: - - archives are unpacked with `is_unpacking_enabled`, + - archives are unpacked with `is_unpacking_enabled`, the `extract_mode` decides on their own folders later, - other files (but [BIOS]) are packed with `is_packing_enabled`, - the rest is moved into its target folder, unless it is already there. `target_folder_for` tells where a file belongs. @@ -34,7 +36,7 @@ def plan_operations(files_list, target_folder_for, is_unpacking_enabled, is_pack for file_name in files_list: target_folder = target_folder_for(file_name) if is_unpacking_enabled and archives.is_archive(file_name): - operations.append(Operation(Kind.UNPACK, file_name, target_folder)) + operations.append(Operation(Kind.UNPACK, file_name, target_folder, extract_mode=extract_mode)) elif is_packing_enabled and not archives.is_archive(file_name) and not Release.parse(file_name).is_bios: operations.append(Operation(Kind.PACK, file_name, target_folder, packing_format)) elif os.path.normpath(os.path.dirname(file_name) or ".") != os.path.normpath(target_folder): @@ -48,8 +50,9 @@ def execute(operation) -> tuple[str, str, str]: os.makedirs(operation.target_folder, exist_ok=True) target = f"{Fore.BLUE}{operation.target_folder}{Style.RESET_ALL}" if operation.kind == Kind.UNPACK: - archives.unpack(operation.source, operation.target_folder) - result_log = f" >> Unpacked to: {target}" + folder = archives.extraction_folder(operation.source, operation.target_folder, operation.extract_mode) + archives.unpack(operation.source, folder) + result_log = f" >> Unpacked to: {Fore.BLUE}{folder or '.'}{Style.RESET_ALL}" elif operation.kind == Kind.PACK: archive_file = archives.pack(operation.source, operation.target_folder, operation.packing_format) result_log = f" >> Packed to: {Fore.BLUE}{archive_file}{Style.RESET_ALL}" @@ -64,7 +67,14 @@ def describe(operation) -> str: """What the operation does, for the dry run.""" target = operation.target_folder or "." if operation.kind == Kind.UNPACK: - return f" >> Would unpack: {Fore.GREEN}{operation.source}{Style.RESET_ALL} to {Fore.BLUE}{target}{Style.RESET_ALL}" + # Reading the archive to find its folder changes nothing, so the dry run shows the real destination + try: + target = archives.extraction_folder(operation.source, operation.target_folder, operation.extract_mode) or "." + note = "" + except Exception as error: + note = f" ({Fore.RED}can't read the archive{Style.RESET_ALL}: {archives.describe_error(error)})" + return (f" >> Would unpack: {Fore.GREEN}{operation.source}{Style.RESET_ALL} " + f"to {Fore.BLUE}{target}{Style.RESET_ALL}{note}") if operation.kind == Kind.PACK: return (f" >> Would pack: {Fore.GREEN}{operation.source}{Style.RESET_ALL} to {operation.packing_format} " f"in {Fore.BLUE}{target}{Style.RESET_ALL}") diff --git a/src/romlm.py b/src/romlm.py index 15f58ad..b76e11e 100644 --- a/src/romlm.py +++ b/src/romlm.py @@ -63,7 +63,7 @@ def get_new_folder(filename, separation_options, sorting_options, subfolders, ex def remove_meta_files(path, is_log_enabled): for dirpath, dirnames, filenames in os.walk(path): for f in filenames: - if f.lower() in ("desktop.ini", "thumbs.db", ".ds_store"): + if f.lower() in archives.META_FILES: file_path = os.path.join(dirpath, f) os.remove(file_path) if is_log_enabled: @@ -94,6 +94,7 @@ class Options: subfolders: list exclude_tags: list is_unpacking_enabled: bool + extract_mode: archives.ExtractMode is_packing_enabled: bool packing_format: str is_remove_duplicates: bool @@ -124,6 +125,16 @@ def sort_argument(value): def unlicensed_argument(value) -> CategoryOption: return CategoryOption(0) if value == "none" else parse_category_flags(value, "hp") +EXTRACT_MODES = { + "f": archives.ExtractMode.FOLDERS, + "p": archives.ExtractMode.PLAIN, +} + +def extract_argument(value) -> archives.ExtractMode: + if value not in EXTRACT_MODES: + raise argparse.ArgumentTypeError(f"unknown option '{value}', use 'f' or 'p'") + return EXTRACT_MODES[value] + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="romlm", usage="%(prog)s [parameters]", add_help=False, allow_abbrev=False, @@ -144,10 +155,17 @@ def build_parser() -> argparse.ArgumentParser: "'a' - all\n" "or use any combinations, like 'hp' or 'ps'.\n" "'reverse' - un-sort ROMs, placing all in the root.\n") - archives = parser.add_mutually_exclusive_group() - archives.add_argument("-x", "--extract", action="store_true", - help="Extract all 7z/zip files in the folder.\n") - archives.add_argument("-p", "--pack", metavar="format", nargs="?", const="7z", choices=("7z", "zip"), + archive_options = parser.add_mutually_exclusive_group() + archive_options.add_argument("-x", "--extract", metavar="mode", nargs="?", const=archives.ExtractMode.AUTO, + type=extract_argument, + help="Extract all 7z/zip files in the folder.\n" + "[mode] can be:\n" + "'f' - extract every archive into its own folder\n" + "'p' - extract all files next to the archives\n" + "Default is to use an own folder only for archives\n" + "with more than one file, or with a generic\n" + "file name inside, like 'game.iso' or 'Disc 1.cue'.\n") + archive_options.add_argument("-p", "--pack", metavar="format", nargs="?", const="7z", choices=("7z", "zip"), help="Pack all files in the folder to 7z/zip format.\n" "[format] can be '7z' or 'zip'. Default is '7z'.\n") parser.add_argument("-u", "--unlicensed", metavar="options", nargs="?", const=CategoryOption(0), type=unlicensed_argument, @@ -195,7 +213,8 @@ def parse_options(argv) -> Options: separation_options=CategoryOption.HOMEBREW | CategoryOption.PIRATES if args.unlicensed is None else args.unlicensed, subfolders=args.folders, exclude_tags=args.exclude, - is_unpacking_enabled=args.extract, + is_unpacking_enabled=args.extract is not None, + extract_mode=args.extract or archives.ExtractMode.PLAIN, is_packing_enabled=args.pack is not None, packing_format=args.pack or "7z", is_remove_duplicates=args.remove_duplicates is not None, @@ -276,7 +295,7 @@ def run(options): print(">> Preparing processing...") planned = operations.plan_operations(files_list, lambda file_name: target_folder_for(file_name, options), options.is_unpacking_enabled, options.is_packing_enabled, - options.packing_format) + options.packing_format, options.extract_mode) print(">> Processing files...") failures = operations.run_operations(planned, options.is_log_enabled, options.is_dry_run) diff --git a/tests/test_archives.py b/tests/test_archives.py new file mode 100644 index 0000000..4bbbb64 --- /dev/null +++ b/tests/test_archives.py @@ -0,0 +1,115 @@ +import os +import subprocess +import sys +import tempfile +import unittest +import zipfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +import archives +from archives import ExtractMode + + +def make_zip(path, *names): + with zipfile.ZipFile(path, "w") as z: + for name in names: + z.writestr(name, "data") + return path + + +class GenericNameTest(unittest.TestCase): + def test_generic_names_of_disc_and_disk_dumps(self): + for name in ("game.iso", "GAME.CUE", "Disc 1.cue", "disc2.chd", "disk1.adf", "Disk 1A.adf", "CD2.bin", + "cd_1.img", "track01.bin", "Track 03.raw", "Disk 1 Side A.st", "Side B.st", "Disc 1 of 2.chd", + "EBOOT.BIN", "default.xbe", "main.dol", "boot.bin", "image.img", "rom.bin", "01.bin", + "game.nkit.iso", "dvd1.iso", "gdrom.gdi", "umd.iso"): + with self.subTest(name=name): + self.assertTrue(archives.is_generic_name(name)) + + def test_game_names_are_not_generic(self): + for name in ("Zelda (USA).nes", "Final Fantasy VII (USA) (Disc 1).cue", "Disco Elysium.iso", "games.iso", + "diska.adf", "Vol. 1 (USA).bin", "Super Mario Bros. 3 (USA).nes", "1942.nes", "a.iso", + "Game (USA)", "Gamé.iso", ""): + with self.subTest(name=name): + self.assertFalse(archives.is_generic_name(name)) + + def test_generic_folder_names(self): + self.assertTrue(archives.is_generic_name("Disc 1")) + self.assertFalse(archives.is_generic_name("Game (USA)")) + + def test_strip_extensions(self): + self.assertEqual(archives.strip_extensions("game.nkit.iso"), "game") + self.assertEqual(archives.strip_extensions("Game Vol. 1 (USA).bin"), "Game Vol. 1 (USA)") + self.assertEqual(archives.strip_extensions("Disc 1"), "Disc 1") + + +class ContentsTest(unittest.TestCase): + def test_top_level_entries_skip_system_junk(self): + entries = ["Game/", "Game/track01.bin", "./Game/track02.raw", "__MACOSX/._Game", ".DS_Store", "Thumbs.db"] + self.assertEqual(archives.top_level_entries(entries), {"Game"}) + + def test_own_folder_is_needed_for_many_entries_or_a_generic_one(self): + self.assertFalse(archives.needs_own_folder(set())) + self.assertFalse(archives.needs_own_folder({"Zelda (USA).nes"})) + self.assertFalse(archives.needs_own_folder({"Zelda (USA)"})) + self.assertTrue(archives.needs_own_folder({"Game (USA).cue", "Game (USA).bin"})) + self.assertTrue(archives.needs_own_folder({"game.iso"})) + self.assertTrue(archives.needs_own_folder({"Disc 1"})) + + def test_own_folder_name(self): + self.assertEqual(archives.own_folder_name("A/Game (USA).zip", {"game.iso"}), "Game (USA)") + self.assertEqual(archives.own_folder_name("Game Vol. 1 (USA).7z", {"a", "b"}), "Game Vol. 1 (USA)") + # romlm packs "Game.nes" into "Game.nes.zip" + self.assertEqual(archives.own_folder_name("Game (USA).nes.zip", {"Game (USA).nes"}), "Game (USA)") + + +class ExtractionFolderTest(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory() + self.addCleanup(self.folder.cleanup) + self.single = make_zip(os.path.join(self.folder.name, "Zelda (USA).zip"), "Zelda (USA).nes") + self.generic = make_zip(os.path.join(self.folder.name, "Saturn Game (USA).zip"), "game.iso") + self.many = make_zip(os.path.join(self.folder.name, "PS Game (USA).zip"), "PS Game (USA).cue", + "PS Game (USA).bin") + + def test_plain_mode_does_not_read_the_archive(self): + missing = os.path.join(self.folder.name, "Missing.zip") + self.assertEqual(archives.extraction_folder(missing, "out", ExtractMode.PLAIN), "out") + + def test_folders_mode_always_makes_a_folder(self): + self.assertEqual(archives.extraction_folder(self.single, "out", ExtractMode.FOLDERS), + os.path.join("out", "Zelda (USA)")) + self.assertEqual(archives.extraction_folder(self.many, "", ExtractMode.FOLDERS), "PS Game (USA)") + + def test_auto_mode_makes_a_folder_only_when_needed(self): + self.assertEqual(archives.extraction_folder(self.single, "out", ExtractMode.AUTO), "out") + self.assertEqual(archives.extraction_folder(self.generic, "out", ExtractMode.AUTO), + os.path.join("out", "Saturn Game (USA)")) + self.assertEqual(archives.extraction_folder(self.many, "out", ExtractMode.AUTO), + os.path.join("out", "PS Game (USA)")) + + def test_auto_mode_keeps_a_named_folder_inside_the_archive(self): + archive = make_zip(os.path.join(self.folder.name, "DC Game (USA).zip"), + "DC Game (USA)/disc.gdi", "DC Game (USA)/track01.bin") + self.assertEqual(archives.extraction_folder(archive, "", ExtractMode.AUTO), "") + + +@unittest.skipUnless(archives.find_seven_zip_binary(), "7-Zip is not installed") +class SevenZipContentsTest(unittest.TestCase): + def test_lists_7z_entries(self): + with tempfile.TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "src", "Game")) + for name in ("Game/track01.bin", "Game/track02.raw", "disc.gdi"): + with open(os.path.join(root, "src", name), "w") as f: + f.write("data") + archive = os.path.join(root, "DC Game (USA).7z") + subprocess.run([archives.find_seven_zip_binary(), "a", archive, "Game", "disc.gdi"], + cwd=os.path.join(root, "src"), check=True, capture_output=True) + self.assertEqual(sorted(archives.list_entries(archive)), + ["Game", "Game/track01.bin", "Game/track02.raw", "disc.gdi"]) + self.assertEqual(archives.extraction_folder(archive, "", ExtractMode.AUTO), "DC Game (USA)") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py index 181e39f..1a9c503 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,10 @@ import os +import re import subprocess import sys import tempfile import unittest +import zipfile ROMLM = os.path.join(os.path.dirname(__file__), "..", "src", "romlm.py") @@ -53,5 +55,47 @@ def test_nothing_to_do(self): self.assertIn("Nothing to do", result.stdout) +class ExtractTest(unittest.TestCase): + ARCHIVES = { + "Single (USA).zip": ["Single (USA).nes"], + "Generic (USA).zip": ["game.iso"], + "Many (USA).zip": ["Many (USA).cue", "Many (USA).bin"], + } + + def extract(self, *args): + with tempfile.TemporaryDirectory() as root: + for name, entries in self.ARCHIVES.items(): + with zipfile.ZipFile(os.path.join(root, name), "w") as z: + for entry in entries: + z.writestr(entry, "data") + result = subprocess.run([sys.executable, ROMLM, "-i", root, "-x", *args], + capture_output=True, text=True, stdin=subprocess.DEVNULL) + self.assertEqual(result.returncode, 0, result.stderr) + self.output = result.stdout + return tree(root) + + def test_default_makes_folders_only_for_many_files_or_generic_names(self): + self.assertEqual(self.extract(), sorted([ + "Single (USA).nes", os.path.join("Generic (USA)", "game.iso"), + os.path.join("Many (USA)", "Many (USA).bin"), os.path.join("Many (USA)", "Many (USA).cue"), + ])) + + def test_folders_for_every_archive(self): + self.assertEqual(self.extract("f"), sorted([ + os.path.join("Single (USA)", "Single (USA).nes"), os.path.join("Generic (USA)", "game.iso"), + os.path.join("Many (USA)", "Many (USA).bin"), os.path.join("Many (USA)", "Many (USA).cue"), + ])) + + def test_plain_extraction(self): + self.assertEqual(self.extract("p"), sorted(["Single (USA).nes", "game.iso", "Many (USA).bin", "Many (USA).cue"])) + + def test_dry_run_shows_the_folders_and_changes_nothing(self): + self.assertEqual(self.extract("-n"), sorted(self.ARCHIVES)) + output = re.sub(r"\x1b\[\d+m", "", self.output) + self.assertEqual(set(re.findall(r"Would unpack: (.*)", output)), { + "Single (USA).zip to .", "Generic (USA).zip to Generic (USA)", "Many (USA).zip to Many (USA)", + }) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_operations.py b/tests/test_operations.py index d2a165a..cd779da 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -10,6 +10,7 @@ import archives import operations +from archives import ExtractMode from operations import Kind, Operation @@ -57,6 +58,11 @@ def test_files_are_packed_but_archives_and_bios_are_not(self): def test_packing_format_is_kept(self): self.assertEqual(plan(["A.nes"], pack=True, fmt="7z")[0].packing_format, "7z") + def test_extract_mode_is_kept(self): + self.assertEqual(plan(["A.zip"], unpack=True)[0].extract_mode, ExtractMode.PLAIN) + operations_list = operations.plan_operations(["A.zip"], lambda f: "", True, False, "7z", ExtractMode.AUTO) + self.assertEqual(operations_list, [Operation(Kind.UNPACK, "A.zip", "", extract_mode=ExtractMode.AUTO)]) + class ExecuteTest(unittest.TestCase): def setUp(self): @@ -103,6 +109,17 @@ def test_unpacks_an_archive_with_an_upper_case_extension(self): self.assertIsNone(error) self.assertEqual(read(self.path("out", "Up (USA).nes")), "rom") + def test_unpacks_into_own_folder_by_the_mode(self): + archive = self.path("Game (USA).zip") + with zipfile.ZipFile(archive, "w") as z: + z.writestr("Game (USA).cue", "cue") + z.writestr("Game (USA).bin", "bin") + _, log, error = operations.execute(Operation(Kind.UNPACK, archive, self.path("out"), + extract_mode=ExtractMode.AUTO)) + self.assertIsNone(error) + self.assertIn(os.path.join("out", "Game (USA)"), log) + self.assertEqual(sorted(os.listdir(self.path("out", "Game (USA)"))), ["Game (USA).bin", "Game (USA).cue"]) + def test_bad_archive_is_reported_and_kept(self): bad = self.touch("Bad (USA).zip", content="not a zip") name, log, error = operations.execute(Operation(Kind.UNPACK, bad, self.path("out"))) @@ -111,6 +128,12 @@ def test_bad_archive_is_reported_and_kept(self): self.assertIn("BadZipFile", error) self.assertTrue(os.path.exists(bad)) + def test_bad_archive_is_reported_and_kept_in_auto_mode(self): + bad = self.touch("Bad (USA).zip", content="not a zip") + _, _, error = operations.execute(Operation(Kind.UNPACK, bad, self.path("out"), extract_mode=ExtractMode.AUTO)) + self.assertIn("BadZipFile", error) + self.assertTrue(os.path.exists(bad)) + def test_run_continues_after_a_failure(self): bad = self.touch("Bad (USA).zip", content="not a zip") rom = self.touch("Game (USA).nes") @@ -139,6 +162,24 @@ def test_nothing_is_touched_and_every_operation_is_described(self): for line, word in zip(lines, ("move", "pack", "unpack")): self.assertIn(f"Would {word}", line) + def test_shows_the_own_folder_of_an_archive(self): + with tempfile.TemporaryDirectory() as root: + archive = os.path.join(root, "Game (USA).zip") + with zipfile.ZipFile(archive, "w") as z: + z.writestr("game.iso", "iso") + line = operations.describe(Operation(Kind.UNPACK, archive, root, extract_mode=ExtractMode.AUTO)) + self.assertIn(os.path.join(root, "Game (USA)"), line) + self.assertTrue(os.path.exists(archive)) + + def test_unreadable_archive_is_noted(self): + with tempfile.TemporaryDirectory() as root: + bad = os.path.join(root, "Bad (USA).zip") + with open(bad, "w") as f: + f.write("not a zip") + line = operations.describe(Operation(Kind.UNPACK, bad, root, extract_mode=ExtractMode.FOLDERS)) + self.assertIn("Would unpack", line) + self.assertIn("can't read the archive", line) + @unittest.skipUnless(archives.find_seven_zip_binary(), "7-Zip is not installed") class SevenZipTest(unittest.TestCase): diff --git a/tests/test_romlm.py b/tests/test_romlm.py index 4690698..d89a01c 100644 --- a/tests/test_romlm.py +++ b/tests/test_romlm.py @@ -9,6 +9,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) import romlm +from archives import ExtractMode from duplicates import Action from romlm import CategoryOption as C @@ -130,6 +131,14 @@ def test_pack_format(self): self.assertEqual(self.parse("-p").packing_format, "7z") self.assertEqual(self.parse("-p", "zip").packing_format, "zip") + def test_extract_mode(self): + self.assertFalse(self.parse().is_unpacking_enabled) + self.assertTrue(self.parse("-x").is_unpacking_enabled) + self.assertEqual(self.parse("-x").extract_mode, ExtractMode.AUTO) + self.assertEqual(self.parse("-x", "-l").extract_mode, ExtractMode.AUTO) + self.assertEqual(self.parse("-x", "f").extract_mode, ExtractMode.FOLDERS) + self.assertEqual(self.parse("--extract", "p").extract_mode, ExtractMode.PLAIN) + def test_duplicates_action_depends_on_log(self): self.assertEqual(self.parse("-r").remove_duplicates_action, Action.KEEP_ALL) self.assertEqual(self.parse("-r", "-l").remove_duplicates_action, Action.ASK) @@ -148,7 +157,8 @@ def test_lists(self): self.assertEqual(options.exclude_tags, ["USA"]) def test_mistakes_are_usage_errors(self): - for args in (["-p", "rar"], ["-r", "bogus"], ["-u", "pirates"], ["-s", "x"], ["-x", "-p"], ["--nope"]): + for args in (["-p", "rar"], ["-r", "bogus"], ["-u", "pirates"], ["-s", "x"], ["-x", "-p"], ["-x", "a"], + ["--nope"]): with self.subTest(args=args), mock.patch("sys.stderr"), self.assertRaises(SystemExit) as raised: self.parse(*args) self.assertEqual(raised.exception.code, 2) From f31bc9561abc4fe49784c74c22ed00b80ad60a66 Mon Sep 17 00:00:00 2001 From: Ilia Petrov-Komotskii Date: Fri, 23 Oct 2026 13:58:47 +0200 Subject: [PATCH 2/6] docs(readme): Tuned description of new feature a bit --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bb67b22..fcca100 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ feel free to open an issue on the [GitHub issues](https://github.com/ManeFunctio - **Extract (`-x, --extract`)** Automatically unpacks all `.7z` or `.zip` files in the current directory (or a specified folder) into subfolders. Supports nested subfolders, keeping all the structure. - - `-x` extracts the files right next to the archive, but gives an archive its own folder (named after the - archive) when the files would make a mess otherwise: when there is more than one file inside (like `.cue` and - `.bin` tracks), or when the file has a generic name, telling nothing about the game, which is common for - disc-based systems: `game.iso`, `Disc 1.cue`, `CD2.bin`, `track01.bin`, `disk1.adf`, `Disk 1 Side A.st`, + - parameterless call is a "smart" mode, it extracts the files right next to the archive, but gives an archive + its own folder (named after the archive) when the files would make a mess otherwise: when there is more than + one file inside (like `.cue` and `.bin` tracks), or when the file has a generic name, telling nothing about the game, + which is common for disc-based systems: `game.iso`, `Disc 1.cue`, `CD2.bin`, `track01.bin`, `disk1.adf`, `Disk 1 Side A.st`, `EBOOT.BIN`, `default.xbe`, `main.dol` and so on. An archive that already keeps everything in one properly named folder is extracted as it is. - `f` extracts every archive into its own folder. From 8dcdefa868355958e1f7853df33bcd56226fff0f Mon Sep 17 00:00:00 2001 From: Ilia Petrov-Komotskii Date: Fri, 23 Oct 2026 14:03:26 +0200 Subject: [PATCH 3/6] feat: Removed __MACOS leftover folders --- CHANGELOG.md | 1 + README.md | 2 +- src/romlm.py | 8 ++++++++ tests/test_romlm.py | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aac5e9f..c1225e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - `-x` without a mode now gives an archive its own folder when it holds more than one file, or a file with a generic name like `game.iso`, `Disc 1.cue` or `track01.bin`, instead of always extracting next to the archive. Use `-x p` for the old behavior. `--dry-run` reads the archives to show where they would be extracted. +- The cleanup after processing now also removes `__MACOSX` folders, left by zips made on a Mac. ## [1.2.0] - 2026-10-16 ### Added diff --git a/README.md b/README.md index fcca100..de1857f 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ feel free to open an issue on the [GitHub issues](https://github.com/ManeFunctio - **Other Utilities** - - Cleans out unwanted system meta-files (e.g., `desktop.ini`, `.DS_Store`). + - Cleans out unwanted system meta-files (e.g., `desktop.ini`, `.DS_Store`, `__MACOSX` folders). - Removes empty subdirectories after sorting. - Provides an in-script help guide (`-h, --help`). diff --git a/src/romlm.py b/src/romlm.py index b76e11e..2b7a9e0 100644 --- a/src/romlm.py +++ b/src/romlm.py @@ -1,6 +1,7 @@ import sys import os import glob +import shutil import argparse from dataclasses import dataclass from enum import Flag, auto @@ -62,6 +63,13 @@ def get_new_folder(filename, separation_options, sorting_options, subfolders, ex def remove_meta_files(path, is_log_enabled): for dirpath, dirnames, filenames in os.walk(path): + # Whole folders of junk, like "__MACOSX" from the zips made on a Mac, are removed without walking into them + for d in [d for d in dirnames if d.lower() in archives.META_FOLDERS]: + folder_path = os.path.join(dirpath, d) + shutil.rmtree(folder_path) + dirnames.remove(d) + if is_log_enabled: + print(f"Removed meta folder: {Fore.RED}{folder_path}{Style.RESET_ALL}") for f in filenames: if f.lower() in archives.META_FILES: file_path = os.path.join(dirpath, f) diff --git a/tests/test_romlm.py b/tests/test_romlm.py index d89a01c..b0f2b9d 100644 --- a/tests/test_romlm.py +++ b/tests/test_romlm.py @@ -99,6 +99,22 @@ def test_logs_each_removed_folder_children_first(self): self.assertEqual(self.clean(False), "") +class RemoveMetaFilesTest(unittest.TestCase): + def test_removes_meta_files_and_macos_folders(self): + with tempfile.TemporaryDirectory() as root: + for path in ("G/Game (USA)/__MACOSX/._game.iso", "G/Game (USA)/game.iso", "__MACOSX/._a", + "G/.DS_Store", "Thumbs.db", "Keep (USA).nes"): + os.makedirs(os.path.dirname(os.path.join(root, path)), exist_ok=True) + open(os.path.join(root, path), "w").close() + output = io.StringIO() + with contextlib.redirect_stdout(output): + romlm.remove_meta_files(root, True) + files = sorted(os.path.relpath(os.path.join(d, f), root) for d, _, names in os.walk(root) for f in names) + self.assertEqual(files, sorted(["Keep (USA).nes", os.path.join("G", "Game (USA)", "game.iso")])) + self.assertFalse(os.path.exists(os.path.join(root, "G", "Game (USA)", "__MACOSX"))) + self.assertEqual(output.getvalue().count("Removed meta folder"), 2) + + class ParseOptionsTest(unittest.TestCase): def parse(self, *args): return romlm.parse_options(list(args)) From 905a165b510ef14c70546b1a22ddea96b1947cfe Mon Sep 17 00:00:00 2001 From: Ilia Petrov-Komotskii Date: Fri, 23 Oct 2026 19:20:12 +0200 Subject: [PATCH 4/6] feat(-x): Create folders if game files collids with a name --- CHANGELOG.md | 9 ++-- README.md | 8 ++- src/archives.py | 88 ++++++++++++++++++++++-------- src/operations.py | 56 ++++++++++++++++++- src/romlm.py | 6 ++- tests/test_archives.py | 24 +++++++-- tests/test_cli.py | 18 ++++++- tests/test_operations.py | 113 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 284 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1225e9..baa89e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). `-x p` extracts everything next to the archives. ### Changed -- `-x` without a mode now gives an archive its own folder when it holds more than one file, or a file with a generic - name like `game.iso`, `Disc 1.cue` or `track01.bin`, instead of always extracting next to the archive. Use `-x p` - for the old behavior. `--dry-run` reads the archives to show where they would be extracted. +- `-x` without a mode now gives an archive its own folder when it holds more than one file, a file with a generic + name like `game.iso`, `Disc 1.cue` or `track01.bin`, or a file that would overwrite a file already in the folder or + one from another archive, instead of always extracting next to the archive. Use `-x p` for the old behavior. + `--dry-run` reads the archives to show where they would be extracted. +- With `-x` and `-x f`, archives of the same name in different formats with the same files inside get folders with + the format in the name, like `Game (USA).zip/` and `Game (USA).7z/`. - The cleanup after processing now also removes `__MACOSX` folders, left by zips made on a Mac. ## [1.2.0] - 2026-10-16 diff --git a/README.md b/README.md index de1857f..8f1a17d 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,13 @@ feel free to open an issue on the [GitHub issues](https://github.com/ManeFunctio its own folder (named after the archive) when the files would make a mess otherwise: when there is more than one file inside (like `.cue` and `.bin` tracks), or when the file has a generic name, telling nothing about the game, which is common for disc-based systems: `game.iso`, `Disc 1.cue`, `CD2.bin`, `track01.bin`, `disk1.adf`, `Disk 1 Side A.st`, - `EBOOT.BIN`, `default.xbe`, `main.dol` and so on. An archive that already keeps everything in one properly - named folder is extracted as it is. + `EBOOT.BIN`, `default.xbe`, `main.dol` and so on, or when its files would overwrite the files already + in the folder, or the files of another archive (all such archives get own folders then). An archive that + already keeps everything in one properly named folder is extracted as it is. - `f` extracts every archive into its own folder. + - With `-x` and `-x f`, archives of the same name but different formats (like `Game (USA).zip` and + `Game (USA).7z`) that have the same files inside get folders with the format in the name: `Game (USA).zip/` + and `Game (USA).7z/`. The same happens when the own folder already has these files. - `p` is a plain extraction: all the files go right next to the archives. ![](https://raw.githubusercontent.com/wiki/ManeFunction/romlm/extract.png) diff --git a/src/archives.py b/src/archives.py index 7601f46..03d0b58 100644 --- a/src/archives.py +++ b/src/archives.py @@ -2,6 +2,7 @@ import re import shutil import subprocess +import tempfile import zipfile from enum import Enum from functools import lru_cache @@ -32,7 +33,8 @@ class ExtractMode(Enum): PLAIN = 1 # the contents go next to the archive FOLDERS = 2 # every archive gets its own folder - AUTO = 3 # own folder only when the plain extraction would be a mess (see needs_own_folder) + AUTO = 3 # own folder only when the plain extraction would be a mess (see needs_own_folder), + # or would overwrite some files @lru_cache(maxsize=None) def find_seven_zip_binary(): @@ -68,25 +70,35 @@ def is_generic_name(name) -> bool: previous = token return previous is not None -def list_entries(file_name) -> list[str]: - """Paths of everything inside the archive, with '/' separators.""" +def _list_paths(file_name): + """Paths of the files (not folders) inside the archive, as the archive keeps them.""" if file_name.lower().endswith(".zip"): with zipfile.ZipFile(file_name, 'r') as archive: - return archive.namelist() - # The technical listing has one "Path = ..." line per entry, after the archive's own block ending with dashes + return [info.filename for info in archive.infolist() if not info.is_dir()] + # The technical listing has a "Key = Value" block per entry, after the archive's own block ending with dashes result = subprocess.run([find_seven_zip_binary(), "l", "-slt", os.path.abspath(file_name)], check=True, capture_output=True) - _, _, entries = result.stdout.decode(errors="replace").partition("\n----------") - return [line[len("Path = "):].replace("\\", "/") for line in entries.splitlines() if line.startswith("Path = ")] - -def top_level_entries(entries) -> set[str]: - """Names of the files and folders that land right in the target folder, without the system junk.""" - names = set() - for entry in entries: - parts = [part for part in entry.split("/") if part not in ("", ".")] - if parts and parts[0].lower() not in META_FILES + META_FOLDERS: - names.add(parts[0]) - return names + _, _, listing = result.stdout.decode(errors="replace").partition("\n----------") + paths = [] + for block in re.split(r"\n\s*\n", listing): + fields = dict(line.split(" = ", 1) for line in block.splitlines() if " = " in line) + is_folder = fields.get("Folder") == "+" or fields.get("Attributes", "").startswith("D") + if "Path" in fields and not is_folder: + paths.append(fields["Path"].replace("\\", "/")) + return paths + +def list_files(file_name) -> list[str]: + """Paths of the files inside the archive, with '/' separators, without the system junk.""" + files = [] + for path in _list_paths(file_name): + parts = [part for part in path.split("/") if part not in ("", ".")] + if parts and not any(part.lower() in META_FILES + META_FOLDERS for part in parts): + files.append("/".join(parts)) + return files + +def top_level_entries(files) -> set[str]: + """Names of the files and folders that land right in the target folder.""" + return {path.split("/")[0] for path in files} def needs_own_folder(top_level) -> bool: """An archive with a few things inside, or one with a generic name, would clutter the folder it's extracted to.""" @@ -99,17 +111,29 @@ def own_folder_name(file_name, top_level) -> str: name = os.path.splitext(name)[0] or name return name -def extraction_folder(file_name, target_folder, mode) -> str: - """Where the archive's contents should go, according to the mode. AUTO mode reads the archive to decide.""" +def extraction_folders(file_name, target_folder, mode, files) -> list[str]: + """ + Where the archive's `files` can go, from the most to the least preferred, according to the mode: the target + folder itself (if AUTO mode finds it fine), an own folder ("Game (USA)"), and an own folder with the archive format + ("Game (USA).zip"), for the case another archive of the same name has the same files. + """ if mode == ExtractMode.PLAIN: - return target_folder - top_level = top_level_entries(list_entries(file_name)) + return [target_folder] + top_level = top_level_entries(files) + own_folder = own_folder_name(file_name, top_level) + folders = [os.path.join(target_folder, own_folder), + os.path.join(target_folder, own_folder + os.path.splitext(file_name)[1])] if mode == ExtractMode.AUTO and not needs_own_folder(top_level): + folders.insert(0, target_folder) + return folders + +def extraction_folder(file_name, target_folder, mode) -> str: + """Where the archive's contents should go, if nothing is in the way. Reads the archive, unless in PLAIN mode.""" + if mode == ExtractMode.PLAIN: return target_folder - return os.path.join(target_folder, own_folder_name(file_name, top_level)) + return extraction_folders(file_name, target_folder, mode, list_files(file_name))[0] -def unpack(file_name, target_folder): - """Extracts the archive into the folder, and removes the archive.""" +def _extract(file_name, target_folder): if file_name.lower().endswith(".7z"): subprocess.run( [find_seven_zip_binary(), "x", os.path.abspath(file_name), f"-o{os.path.abspath(target_folder)}", "-y"], @@ -118,7 +142,25 @@ def unpack(file_name, target_folder): elif file_name.lower().endswith(".zip"): with zipfile.ZipFile(file_name, 'r') as archive: archive.extractall(target_folder) + +def unpack(file_name, target_folder): + """Extracts the archive into the folder, and removes the archive. The folder can be named as the archive itself.""" + # A folder can't take the archive's place before the archive is gone, so it is extracted next to it first + is_in_place = os.path.isfile(target_folder) and os.path.samefile(target_folder, file_name) + if is_in_place: + folder = tempfile.mkdtemp(prefix=".romlm-", dir=os.path.dirname(os.path.abspath(file_name))) + else: + folder = target_folder + os.makedirs(folder or ".", exist_ok=True) + try: + _extract(file_name, folder) + except Exception: + if is_in_place: + shutil.rmtree(folder, ignore_errors=True) + raise os.remove(file_name) + if is_in_place: + os.rename(folder, target_folder) def pack(file_name, target_folder, packing_format) -> str: """Packs the file into an archive in the folder, and removes the file. Returns the archive path.""" diff --git a/src/operations.py b/src/operations.py index 8fece2f..04eec26 100644 --- a/src/operations.py +++ b/src/operations.py @@ -1,6 +1,7 @@ import os import shutil -from dataclasses import dataclass +from collections import defaultdict +from dataclasses import dataclass, replace from enum import Enum from multiprocessing import Pool from colorama import Fore, Style @@ -43,10 +44,61 @@ def plan_operations(files_list, target_folder_for, is_unpacking_enabled, is_pack operations.append(Operation(Kind.MOVE, file_name, target_folder)) return operations +def read_files(operation): + """The files inside the operation's archive, or None if it can't be read.""" + try: + return archives.list_files(operation.source) + except Exception: + return None + +def _path_key(path) -> str: + """The same key for the paths of one file (case-insensitive, as file systems often are).""" + return os.path.normpath(path).lower() + +def resolve_extraction_folders(operations) -> list: + """ + Reads the archives that may need their own folders, and decides where each one goes (see archives.ExtractMode). + An archive whose files would overwrite another archive's ones, or the files on disk (or moved there by this run), + goes to its next folder of archives.extraction_folders, until nothing is overwritten or no folders are left. + Colliding archives move on together. The archives that can't be read are left as they are, to fail on unpacking. + """ + to_read = [op for op in operations if op.kind == Kind.UNPACK and op.extract_mode != archives.ExtractMode.PLAIN] + if not to_read: + return operations + with Pool(processes=os.cpu_count()) as pool: + listings = {op: files for op, files in zip(to_read, pool.map(read_files, to_read)) if files is not None} + + choices = {op: archives.extraction_folders(op.source, op.target_folder, op.extract_mode, files) + for op, files in listings.items()} + choice = dict.fromkeys(listings, 0) + moved_in = {_path_key(os.path.join(op.target_folder, os.path.basename(op.source))) + for op in operations if op.kind == Kind.MOVE} + + def is_taken(path): + return _path_key(path) in moved_in or os.path.exists(path) + + while True: + owners = defaultdict(set) + for op, files in listings.items(): + for path in files: + owners[_path_key(os.path.join(choices[op][choice[op]], path))].add(op) + colliding = {op for ops in owners.values() if len(ops) > 1 for op in ops} + colliding |= {op for op, files in listings.items() if op not in colliding + and any(is_taken(os.path.join(choices[op][choice[op]], path)) for path in files)} + moving_on = [op for op in colliding if choice[op] + 1 < len(choices[op])] + if not moving_on: + break + for op in moving_on: + choice[op] += 1 + + return [replace(op, target_folder=choices[op][choice[op]], extract_mode=archives.ExtractMode.PLAIN) + if op in listings else op for op in operations] + def execute(operation) -> tuple[str, str, str]: """Carries out the operation. Returns the file name, a log line and an error message.""" try: - if operation.target_folder: + # Unpacking makes its folder itself, as the folder may take the archive's place + if operation.target_folder and operation.kind != Kind.UNPACK: os.makedirs(operation.target_folder, exist_ok=True) target = f"{Fore.BLUE}{operation.target_folder}{Style.RESET_ALL}" if operation.kind == Kind.UNPACK: diff --git a/src/romlm.py b/src/romlm.py index 2b7a9e0..6be02e5 100644 --- a/src/romlm.py +++ b/src/romlm.py @@ -171,8 +171,9 @@ def build_parser() -> argparse.ArgumentParser: "'f' - extract every archive into its own folder\n" "'p' - extract all files next to the archives\n" "Default is to use an own folder only for archives\n" - "with more than one file, or with a generic\n" - "file name inside, like 'game.iso' or 'Disc 1.cue'.\n") + "with more than one file, with a generic file name\n" + "inside, like 'game.iso' or 'Disc 1.cue', or with\n" + "a file that is already there or in another archive.\n") archive_options.add_argument("-p", "--pack", metavar="format", nargs="?", const="7z", choices=("7z", "zip"), help="Pack all files in the folder to 7z/zip format.\n" "[format] can be '7z' or 'zip'. Default is '7z'.\n") @@ -304,6 +305,7 @@ def run(options): planned = operations.plan_operations(files_list, lambda file_name: target_folder_for(file_name, options), options.is_unpacking_enabled, options.is_packing_enabled, options.packing_format, options.extract_mode) + planned = operations.resolve_extraction_folders(planned) print(">> Processing files...") failures = operations.run_operations(planned, options.is_log_enabled, options.is_dry_run) diff --git a/tests/test_archives.py b/tests/test_archives.py index 4bbbb64..fbff13f 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -45,9 +45,15 @@ def test_strip_extensions(self): class ContentsTest(unittest.TestCase): - def test_top_level_entries_skip_system_junk(self): - entries = ["Game/", "Game/track01.bin", "./Game/track02.raw", "__MACOSX/._Game", ".DS_Store", "Thumbs.db"] - self.assertEqual(archives.top_level_entries(entries), {"Game"}) + def test_files_are_listed_without_folders_and_system_junk(self): + with tempfile.TemporaryDirectory() as root: + archive = make_zip(os.path.join(root, "Game.zip"), "Game/", "Game/track01.bin", "./Game/track02.raw", + "__MACOSX/._Game", ".DS_Store", "Game/Thumbs.db") + self.assertEqual(archives.list_files(archive), ["Game/track01.bin", "Game/track02.raw"]) + + def test_top_level_entries(self): + self.assertEqual(archives.top_level_entries(["Game/track01.bin", "Game/track02.raw"]), {"Game"}) + self.assertEqual(archives.top_level_entries(["a.cue", "a.bin"]), {"a.cue", "a.bin"}) def test_own_folder_is_needed_for_many_entries_or_a_generic_one(self): self.assertFalse(archives.needs_own_folder(set())) @@ -89,6 +95,15 @@ def test_auto_mode_makes_a_folder_only_when_needed(self): self.assertEqual(archives.extraction_folder(self.many, "out", ExtractMode.AUTO), os.path.join("out", "PS Game (USA)")) + def test_folder_choices_by_the_mode(self): + self.assertEqual(archives.extraction_folders("A/Game (USA).zip", "A", ExtractMode.AUTO, ["Game (USA).nes"]), + ["A", os.path.join("A", "Game (USA)"), os.path.join("A", "Game (USA).zip")]) + self.assertEqual(archives.extraction_folders("Game (USA).7z", "", ExtractMode.AUTO, ["game.iso"]), + ["Game (USA)", "Game (USA).7z"]) + self.assertEqual(archives.extraction_folders("Game (USA).nes.zip", "", ExtractMode.FOLDERS, ["Game (USA).nes"]), + ["Game (USA)", "Game (USA).zip"]) + self.assertEqual(archives.extraction_folders("Game (USA).zip", "out", ExtractMode.PLAIN, ["game.iso"]), ["out"]) + def test_auto_mode_keeps_a_named_folder_inside_the_archive(self): archive = make_zip(os.path.join(self.folder.name, "DC Game (USA).zip"), "DC Game (USA)/disc.gdi", "DC Game (USA)/track01.bin") @@ -106,8 +121,7 @@ def test_lists_7z_entries(self): archive = os.path.join(root, "DC Game (USA).7z") subprocess.run([archives.find_seven_zip_binary(), "a", archive, "Game", "disc.gdi"], cwd=os.path.join(root, "src"), check=True, capture_output=True) - self.assertEqual(sorted(archives.list_entries(archive)), - ["Game", "Game/track01.bin", "Game/track02.raw", "disc.gdi"]) + self.assertEqual(sorted(archives.list_files(archive)), ["Game/track01.bin", "Game/track02.raw", "disc.gdi"]) self.assertEqual(archives.extraction_folder(archive, "", ExtractMode.AUTO), "DC Game (USA)") diff --git a/tests/test_cli.py b/tests/test_cli.py index 1a9c503..1f9b2b9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -62,8 +62,10 @@ class ExtractTest(unittest.TestCase): "Many (USA).zip": ["Many (USA).cue", "Many (USA).bin"], } - def extract(self, *args): + def extract(self, *args, existing=()): with tempfile.TemporaryDirectory() as root: + for name in existing: + open(os.path.join(root, name), "w").close() for name, entries in self.ARCHIVES.items(): with zipfile.ZipFile(os.path.join(root, name), "w") as z: for entry in entries: @@ -80,6 +82,20 @@ def test_default_makes_folders_only_for_many_files_or_generic_names(self): os.path.join("Many (USA)", "Many (USA).bin"), os.path.join("Many (USA)", "Many (USA).cue"), ])) + def test_default_makes_folders_for_archives_with_the_same_files(self): + self.ARCHIVES = {"Game (USA).zip": ["Game.nes"], "Game (USA) (Rev 1).zip": ["Game.nes"], + "Other (USA).zip": ["Other.nes"]} + self.assertEqual(self.extract(), sorted([ + os.path.join("Game (USA)", "Game.nes"), os.path.join("Game (USA) (Rev 1)", "Game.nes"), "Other.nes", + ])) + + def test_default_does_not_overwrite_the_files_on_disk(self): + self.assertEqual(self.extract(existing=["Single (USA).nes"]), sorted([ + "Single (USA).nes", os.path.join("Single (USA)", "Single (USA).nes"), + os.path.join("Generic (USA)", "game.iso"), + os.path.join("Many (USA)", "Many (USA).bin"), os.path.join("Many (USA)", "Many (USA).cue"), + ])) + def test_folders_for_every_archive(self): self.assertEqual(self.extract("f"), sorted([ os.path.join("Single (USA)", "Single (USA).nes"), os.path.join("Generic (USA)", "game.iso"), diff --git a/tests/test_operations.py b/tests/test_operations.py index cd779da..5ff46c7 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -64,6 +64,103 @@ def test_extract_mode_is_kept(self): self.assertEqual(operations_list, [Operation(Kind.UNPACK, "A.zip", "", extract_mode=ExtractMode.AUTO)]) +class ResolveExtractionFoldersTest(unittest.TestCase): + def setUp(self): + self.folder = tempfile.TemporaryDirectory() + self.addCleanup(self.folder.cleanup) + + def archive(self, name, *entries): + path = os.path.join(self.folder.name, name) + os.makedirs(os.path.dirname(path), exist_ok=True) + with zipfile.ZipFile(path, "w") as z: + for entry in entries: + z.writestr(entry, "data") + return path + + def resolve(self, *archives_list, mode=ExtractMode.AUTO): + ops = [Operation(Kind.UNPACK, path, os.path.dirname(path), extract_mode=mode) for path in archives_list] + return {os.path.relpath(op.source, self.folder.name): os.path.relpath(op.target_folder, self.folder.name) + for op in operations.resolve_extraction_folders(ops)} + + def test_archives_with_the_same_file_get_own_folders(self): + self.assertEqual(self.resolve(self.archive("Game (USA).zip", "game (usa).nes"), + self.archive("Game (USA) (Rev 1).zip", "Game (USA).nes"), + self.archive("Other (USA).zip", "Other (USA).nes")), { + "Game (USA).zip": "Game (USA)", "Game (USA) (Rev 1).zip": "Game (USA) (Rev 1)", "Other (USA).zip": ".", + }) + + def test_same_files_in_different_folders_do_not_collide(self): + self.assertEqual(self.resolve(self.archive(os.path.join("A", "Game (USA).zip"), "Game (USA).nes"), + self.archive(os.path.join("B", "Game (USA).zip"), "Game (USA).nes")), + {os.path.join("A", "Game (USA).zip"): "A", os.path.join("B", "Game (USA).zip"): "B"}) + + def test_archive_does_not_overwrite_a_file_on_disk(self): + with open(os.path.join(self.folder.name, "Game (USA).nes"), "w") as f: + f.write("old") + self.assertEqual(self.resolve(self.archive("Game (USA) (Rev 1).zip", "Game (USA).nes"), + self.archive("Other (USA).zip", "Other (USA).nes")), + {"Game (USA) (Rev 1).zip": "Game (USA) (Rev 1)", "Other (USA).zip": "."}) + + def test_archive_does_not_overwrite_a_file_moved_in_by_the_run(self): + archive = self.archive("Game (USA) (Rev 1).zip", "Game (USA).nes") + ops = [Operation(Kind.MOVE, os.path.join("Sub", "Game (USA).nes"), self.folder.name), + Operation(Kind.UNPACK, archive, self.folder.name, extract_mode=ExtractMode.AUTO)] + self.assertEqual(operations.resolve_extraction_folders(ops)[1].target_folder, + os.path.join(self.folder.name, "Game (USA) (Rev 1)")) + + def test_archives_of_one_name_and_the_same_files_get_folders_with_the_format(self): + for mode in (ExtractMode.AUTO, ExtractMode.FOLDERS): + with self.subTest(mode=mode): + self.assertEqual(self.resolve(self.archive("Game (USA).zip", "Game (USA).nes"), + self.archive("Game (USA).7z", "Game (USA).nes"), mode=mode), + {"Game (USA).zip": "Game (USA).zip", "Game (USA).7z": "Game (USA).7z"}) + + def test_archives_of_one_name_with_different_files_share_the_folder(self): + self.assertEqual(self.resolve(self.archive("Game (USA).zip", "game.iso"), + self.archive("Game (USA).7z", "Game (USA).cue", "Game (USA).bin")), + {"Game (USA).zip": "Game (USA)", "Game (USA).7z": "Game (USA)"}) + + def test_own_folder_already_holding_the_files_is_not_overwritten(self): + self.archive(os.path.join("Game (USA)", "keep.zip")) + os.rename(os.path.join(self.folder.name, "Game (USA)", "keep.zip"), + os.path.join(self.folder.name, "Game (USA)", "game.iso")) + self.assertEqual(self.resolve(self.archive("Game (USA).zip", "game.iso")), {"Game (USA).zip": "Game (USA).zip"}) + + def test_files_in_the_same_inner_folder_collide(self): + self.assertEqual(self.resolve(self.archive("A (USA).zip", "Game/track01.bin"), + self.archive("B (USA).zip", "Game/track01.bin"), + self.archive("C (USA).zip", "Other/track01.bin")), + {"A (USA).zip": "A (USA)", "B (USA).zip": "B (USA)", "C (USA).zip": "."}) + + def test_system_junk_does_not_collide(self): + self.assertEqual(self.resolve(self.archive("A (USA).zip", "A (USA).nes", ".DS_Store"), + self.archive("B (USA).zip", "B (USA).nes", ".DS_Store")), + {"A (USA).zip": ".", "B (USA).zip": "."}) + + def test_resolved_operations_do_not_read_the_archive_again(self): + op = operations.resolve_extraction_folders([Operation( + Kind.UNPACK, self.archive("G (USA).zip", "game.iso"), self.folder.name, extract_mode=ExtractMode.AUTO)])[0] + self.assertEqual(op.extract_mode, ExtractMode.PLAIN) + self.assertEqual(op.target_folder, os.path.join(self.folder.name, "G (USA)")) + + def test_only_auto_mode_looks_for_collisions(self): + a = self.archive("A (USA).zip", "Game.nes") + b = self.archive("B (USA).zip", "Game.nes") + self.assertEqual(self.resolve(a, b, mode=ExtractMode.FOLDERS), {"A (USA).zip": "A (USA)", "B (USA).zip": "B (USA)"}) + self.assertEqual(self.resolve(a, b, mode=ExtractMode.PLAIN), {"A (USA).zip": ".", "B (USA).zip": "."}) + + def test_unreadable_archive_is_left_to_fail_on_unpacking(self): + bad = os.path.join(self.folder.name, "Bad (USA).zip") + with open(bad, "w") as f: + f.write("not a zip") + op = Operation(Kind.UNPACK, bad, self.folder.name, extract_mode=ExtractMode.AUTO) + self.assertEqual(operations.resolve_extraction_folders([op]), [op]) + + def test_other_operations_are_kept(self): + ops = [Operation(Kind.MOVE, "A.nes", "A"), Operation(Kind.PACK, "B.nes", "B")] + self.assertEqual(operations.resolve_extraction_folders(ops), ops) + + class ExecuteTest(unittest.TestCase): def setUp(self): self.folder = tempfile.TemporaryDirectory() @@ -120,6 +217,22 @@ def test_unpacks_into_own_folder_by_the_mode(self): self.assertIn(os.path.join("out", "Game (USA)"), log) self.assertEqual(sorted(os.listdir(self.path("out", "Game (USA)"))), ["Game (USA).bin", "Game (USA).cue"]) + def test_unpacks_into_a_folder_named_as_the_archive(self): + archive = self.path("Game (USA).zip") + with zipfile.ZipFile(archive, "w") as z: + z.writestr("game.iso", "iso") + _, _, error = operations.execute(Operation(Kind.UNPACK, archive, archive)) + self.assertIsNone(error) + self.assertEqual(os.listdir(self.root), ["Game (USA).zip"]) + self.assertEqual(read(self.path("Game (USA).zip", "game.iso")), "iso") + + def test_bad_archive_is_kept_when_unpacking_into_a_folder_named_as_it(self): + bad = self.touch("Bad (USA).zip", content="not a zip") + _, _, error = operations.execute(Operation(Kind.UNPACK, bad, bad)) + self.assertIn("BadZipFile", error) + self.assertEqual(os.listdir(self.root), ["Bad (USA).zip"]) + self.assertEqual(read(bad), "not a zip") + def test_bad_archive_is_reported_and_kept(self): bad = self.touch("Bad (USA).zip", content="not a zip") name, log, error = operations.execute(Operation(Kind.UNPACK, bad, self.path("out"))) From 0129ecab07acf61c1c9bef6e84611c7b3f1be257 Mon Sep 17 00:00:00 2001 From: Ilia Petrov-Komotskii Date: Fri, 23 Oct 2026 23:41:32 +0200 Subject: [PATCH 5/6] feat(-p): --pack option can work with folders of disk-based systems, also made all searches and matches case-insensitive --- CHANGELOG.md | 8 ++++ README.md | 13 +++++++ pyproject.toml | 2 +- src/archives.py | 81 ++++++++++++++++++++++++++++++++++----- src/discs.py | 71 ++++++++++++++++++++++++++++++++++ src/duplicates.py | 2 +- src/operations.py | 24 ++++++++++-- src/tags.py | 18 +++++---- tests/test_archives.py | 10 +++++ tests/test_cli.py | 17 +++++++++ tests/test_discs.py | 82 ++++++++++++++++++++++++++++++++++++++++ tests/test_duplicates.py | 4 ++ tests/test_operations.py | 61 ++++++++++++++++++++++++++++++ tests/test_tags.py | 7 +++- 14 files changed, 376 insertions(+), 24 deletions(-) create mode 100644 src/discs.py create mode 100644 tests/test_discs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index baa89e3..0cd7f81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added - Added extraction modes to `-x`/`--extract`: `-x f` extracts every archive into its own folder named after it, and `-x p` extracts everything next to the archives. +- `-p`/`--pack` packs the folders of disc-based games as a whole, into a single archive named after the folder, with + just the files of the folder inside: the opposite of `-x`. A folder is taken for a game when it has a disc sheet + (`.cue`, `.gdi`, `.ccd`, `.mds`, `.toc` or `.m3u`) or a file with a generic name, and only the files of one game, or + when it's an extracted disc (PS1/PS2, PS3, PSP, Xbox, Xbox 360). ### Changed - `-x` without a mode now gives an archive its own folder when it holds more than one file, a file with a generic @@ -19,6 +23,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). the format in the name, like `Game (USA).zip/` and `Game (USA).7z/`. - The cleanup after processing now also removes `__MACOSX` folders, left by zips made on a Mac. +### Fixed +- Fixed `-r` treating the names that differ only in case (like `Pac-Man (USA)` and `PAC-MAN (Japan)`) as different + games, including the discs of one multi-disc set. + ## [1.2.0] - 2026-10-16 ### Added - Added `-n`/`--dry-run` to preview what would be done without changing anything: duplicates are decided but not diff --git a/README.md b/README.md index 8f1a17d..3d8dbcd 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,19 @@ feel free to open an issue on the [GitHub issues](https://github.com/ManeFunctio Compresses all uncompressed files in the current directory (or a specified folder) into `.7z` or `.zip` archives. Supports nested subfolders, keeping all the structure. + Folders of disc-based games are packed as a whole: all the files of such a folder go into a single archive named + after it (`Sonic CD (USA)/` becomes `Sonic CD (USA).7z`), without the folder itself, so it's the opposite of `-x`. + A folder is taken for a game when it holds: + - a disc sheet with its tracks: `.cue` (with `.bin`, `.iso` or `.img`), `.gdi`, `.ccd` (with `.img` and + `.sub`), `.mds` (with `.mdf`), `.toc`, or an `.m3u` playlist of a multi-disc game, + - or a file with a generic name, like `game.iso` or `track01.bin` (see `-x` above), + - and nothing but the files of one game, besides the files with generic names: either the game it is named + after (`Sonic CD (USA)/Sonic CD (USA).cue`), or any single game (`FF7/Final Fantasy VII (USA).cue`) if the + folder has no subfolders and isn't a `--sort` one (`A`-`Z`, `1-9`, `!...`). So the folders with many games + are never packed together. + - An extracted disc is a game folder whatever the names are: it has `SYSTEM.CNF` (PS1, PS2), `PS3_DISC.SFB` or + `PS3_GAME/` (PS3), `UMD_DATA.BIN` or `PSP_GAME/` (PSP), `default.xbe` (Xbox) or `default.xex` (Xbox 360). + ![](https://raw.githubusercontent.com/wiki/ManeFunction/romlm/pack.png) diff --git a/pyproject.toml b/pyproject.toml index a2925b0..952ae83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ romlm = "romlm:mane" version = { attr = "romlm.version" } [tool.setuptools] -py-modules = ["romlm", "tags", "duplicates", "archives", "operations"] +py-modules = ["romlm", "tags", "duplicates", "archives", "operations", "discs"] [tool.setuptools.package-dir] "" = "src" \ No newline at end of file diff --git a/src/archives.py b/src/archives.py index 03d0b58..0605bd8 100644 --- a/src/archives.py +++ b/src/archives.py @@ -45,6 +45,23 @@ def find_seven_zip_binary(): return path return None +def exists_ignoring_case(path, listdir=os.listdir) -> bool: + """Is there a file or folder at the path, with its names in any case, as case-insensitive file systems see it.""" + return _actual_path(os.path.normpath(path), listdir) is not None + +def _actual_path(path, listdir): + """The path as it is on disk, with the names in any case, or None if there is nothing.""" + if os.path.exists(path): + return path + parent, name = os.path.split(path) + if not name: + return None + actual_parent = _actual_path(parent or ".", listdir) + if actual_parent is None or not os.path.isdir(actual_parent): + return None + match = next((entry for entry in listdir(actual_parent) if entry.lower() == name.lower()), None) + return os.path.join(actual_parent, match) if match is not None else None + def is_archive(file_name) -> bool: return file_name.lower().endswith(ARCHIVE_EXTENSIONS) @@ -107,7 +124,7 @@ def needs_own_folder(top_level) -> bool: def own_folder_name(file_name, top_level) -> str: """The archive name without its extension, and without the inner file's extension too, if romlm packed it.""" name = os.path.splitext(os.path.basename(file_name))[0] - if name in top_level: + if name.lower() in {entry.lower() for entry in top_level}: name = os.path.splitext(name)[0] or name return name @@ -162,28 +179,72 @@ def unpack(file_name, target_folder): if is_in_place: os.rename(folder, target_folder) +def _clean_and_list(folder) -> list[str]: + """Removes the system junk from the folder, and lists the files left, relative to it.""" + files = [] + for dirpath, dirnames, filenames in os.walk(folder): + for d in [d for d in dirnames if d.lower() in META_FOLDERS]: + shutil.rmtree(os.path.join(dirpath, d)) + dirnames.remove(d) + for f in filenames: + if f.lower() in META_FILES: + os.remove(os.path.join(dirpath, f)) + else: + files.append(os.path.relpath(os.path.join(dirpath, f), folder)) + return sorted(files) + +def _remove_packed(folder, files): + """Removes the packed files, and the folders left empty after them, up to the packed folder itself.""" + for name in files: + os.remove(os.path.join(folder, name)) + for dirpath, _, _ in os.walk(folder, topdown=False): + if not os.listdir(dirpath): + os.rmdir(dirpath) + def pack(file_name, target_folder, packing_format) -> str: - """Packs the file into an archive in the folder, and removes the file. Returns the archive path.""" + """ + Packs the file into an archive in the folder, and removes the file. Returns the archive path. + A folder is packed the same way, as a single archive with the files of the folder, but not the folder itself. + """ archive_file = f"{os.path.join(target_folder, os.path.basename(file_name))}.{packing_format}" + if os.path.isdir(file_name): + # Adding to an existing archive would mix two games in one + if exists_ignoring_case(archive_file): + raise FileExistsError(f"'{archive_file}' already exists") + source_folder = os.path.abspath(file_name) + files = _clean_and_list(file_name) + else: + source_folder = os.path.dirname(os.path.abspath(file_name)) + files = [os.path.basename(file_name)] + list_file = None try: if packing_format == "7z": - # Run with cwd set to the source file's folder, passing only its basename, - # so the archive stores just that name (matching the old arcname behavior) - # instead of the full source path. + # Run with cwd set to the source folder, passing the names relative to it, so the archive stores + # just them, instead of the full source paths. A list file keeps a big folder within the + # command line length limit. + with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".txt", delete=False) as listing: + listing.write("\n".join(files)) + list_file = listing.name subprocess.run( - [find_seven_zip_binary(), "a", os.path.abspath(archive_file), os.path.basename(file_name)], - cwd=os.path.dirname(os.path.abspath(file_name)), - check=True, capture_output=True, + [find_seven_zip_binary(), "a", os.path.abspath(archive_file), "-scsUTF-8", f"@{list_file}"], + cwd=source_folder, check=True, capture_output=True, ) elif packing_format == "zip": with zipfile.ZipFile(archive_file, 'w', zipfile.ZIP_DEFLATED) as archive: - archive.write(file_name, arcname=os.path.basename(file_name)) + for name in files: + archive.write(os.path.join(source_folder, name), arcname=name) except Exception: # Don't leave a half-written archive next to the source file if os.path.exists(archive_file): os.remove(archive_file) raise - os.remove(file_name) + finally: + if list_file is not None: + os.remove(list_file) + if os.path.isdir(file_name): + _remove_packed(file_name, files) + else: + os.remove(file_name) return archive_file def describe_error(error) -> str: diff --git a/src/discs.py b/src/discs.py new file mode 100644 index 0000000..51a369b --- /dev/null +++ b/src/discs.py @@ -0,0 +1,71 @@ +import os +import re + +import archives +from tags import get_base_name, strip_tags + +# Sheets that describe a disc image and come along with its data files: ".cue" (with ".bin", ".iso", ".img" or +# ".wav" tracks), ".gdi" (Dreamcast, with ".bin" and ".raw" tracks), ".ccd" (CloneCD, with ".img" and ".sub"), +# ".mds" (Alcohol 120%, with ".mdf") and ".toc" (cdrdao, with ".bin"). Also ".m3u" playlists of multi-disc games. +SHEET_EXTENSIONS = (".cue", ".gdi", ".ccd", ".mds", ".toc", ".m3u") + +# Playlists are often named on their own ("FF7.m3u"), so they don't tell which game the folder holds +PLAYLIST_EXTENSIONS = (".m3u",) + +# Files and folders at the root of an extracted disc: PS3 ("PS3_DISC.SFB", "PS3_GAME"), PSP ("UMD_DATA.BIN", +# "PSP_GAME"), PS1 and PS2 ("SYSTEM.CNF"), Xbox ("default.xbe") and Xbox 360 ("default.xex"). Such a folder +# is a game of its own, whatever the names inside are. +DISC_ROOT_FILES = ("ps3_disc.sfb", "umd_data.bin", "system.cnf", "default.xbe", "default.xex") +DISC_ROOT_FOLDERS = ("ps3_game", "psp_game") + +# The folders of --sort: lettered ones ("A", "1-9"), and the special ones ("!!BIOS", "!Homebrew", "!Japan") +SORT_FOLDER = re.compile(r"[a-z]|1-9|!.*", re.IGNORECASE) + +def _is_of_one_game(folder_name, file_names, folder_names) -> bool: + """ + Are all the files (but the generic named ones, like "track01.bin") of the game the folder is named after, + or of one game at all ("FF7/Final Fantasy VII (USA).cue"), if the folder has nothing else inside, and it's not + a --sort folder, that can hold a single game just by chance ("Q/Quake (USA).cue"). + """ + games = {get_base_name(name).lower() for name in file_names + if not name.lower().endswith(PLAYLIST_EXTENSIONS) and not archives.is_generic_name(name)} + if games <= {strip_tags(folder_name).lower()}: + return True + return len(games) == 1 and not folder_names and not SORT_FOLDER.fullmatch(folder_name) + +def is_game_folder(folder_name, file_names, folder_names=()) -> bool: + """ + Does the folder hold a single disc-based game, by the names of the files (and folders) right in it: + - an extracted disc, with the files like "SYSTEM.CNF" or "PS3_DISC.SFB", or a "PS3_GAME" folder, + - a disc sheet, like ".cue" for ".bin" or ".iso" tracks, or a file with a generic name, like "game.iso", + when the other files are of one game (see _is_of_one_game). + The second check keeps away the folders of many games, like the lettered ones of --sort. + """ + if any(name.lower() in DISC_ROOT_FILES for name in file_names) \ + or any(name.lower() in DISC_ROOT_FOLDERS for name in folder_names): + return True + has_hint = any(name.lower().endswith(SHEET_EXTENSIONS) or archives.is_generic_name(name) for name in file_names) + return has_hint and _is_of_one_game(folder_name, file_names, folder_names) + +def find_game_folders(files_list) -> list[str]: + """ + The folders of disc-based games (see is_game_folder) among the folders of the files, outer ones first. + A game folder inside another one is a part of it, and the root folder is never a game folder. + """ + files_in = {} + folders_in = {} + for path in files_list: + folder, name = os.path.split(os.path.normpath(path)) + files_in.setdefault(folder, []).append(name) + while folder: + parent, child = os.path.split(folder) + folders_in.setdefault(parent, set()).add(child) + folder = parent + + game_folders = [] + for folder in sorted(set(files_in) | set(folders_in), key=lambda f: f.count(os.sep)): + if not folder or any(folder.startswith(game + os.sep) for game in game_folders): + continue + if is_game_folder(os.path.basename(folder), files_in.get(folder, []), folders_in.get(folder, ())): + game_folders.append(folder) + return game_folders diff --git a/src/duplicates.py b/src/duplicates.py index ce28076..80bc631 100644 --- a/src/duplicates.py +++ b/src/duplicates.py @@ -203,7 +203,7 @@ def pick_editions(by_game) -> dict: discs_of_base = {} for key, releases in by_game.items(): if releases[0].disc > -1: - discs_of_base.setdefault(releases[0].base_name, {})[releases[0].disc] = key + discs_of_base.setdefault(releases[0].base_name.lower(), {})[releases[0].disc] = key losers = {} for discs in discs_of_base.values(): diff --git a/src/operations.py b/src/operations.py index 04eec26..be8c0e4 100644 --- a/src/operations.py +++ b/src/operations.py @@ -2,12 +2,14 @@ import shutil from collections import defaultdict from dataclasses import dataclass, replace +from functools import lru_cache from enum import Enum from multiprocessing import Pool from colorama import Fore, Style from tqdm import tqdm import archives +import discs from tags import Release class Kind(Enum): @@ -29,12 +31,22 @@ def plan_operations(files_list, target_folder_for, is_unpacking_enabled, is_pack """ Decides what to do with every file, without touching the disk: - archives are unpacked with `is_unpacking_enabled`, the `extract_mode` decides on their own folders later, - - other files (but [BIOS]) are packed with `is_packing_enabled`, + - other files (but [BIOS]) are packed with `is_packing_enabled`, and the folders of disc-based games + (see discs.find_game_folders) are packed as a whole, each one into a single archive named after it, - the rest is moved into its target folder, unless it is already there. - `target_folder_for` tells where a file belongs. + `target_folder_for` tells where a file (or a game folder) belongs. """ + game_folders = discs.find_game_folders(files_list) if is_packing_enabled else [] + packed_folders = set() operations = [] for file_name in files_list: + game_folder = next((folder for folder in game_folders + if os.path.normpath(file_name).startswith(folder + os.sep)), None) + if game_folder is not None: + if game_folder not in packed_folders: + packed_folders.add(game_folder) + operations.append(Operation(Kind.PACK, game_folder, target_folder_for(game_folder), packing_format)) + continue target_folder = target_folder_for(file_name) if is_unpacking_enabled and archives.is_archive(file_name): operations.append(Operation(Kind.UNPACK, file_name, target_folder, extract_mode=extract_mode)) @@ -74,8 +86,11 @@ def resolve_extraction_folders(operations) -> list: moved_in = {_path_key(os.path.join(op.target_folder, os.path.basename(op.source))) for op in operations if op.kind == Kind.MOVE} + # Nothing changes on disk while deciding, so the folder listings can be reused + listdir = lru_cache(maxsize=None)(os.listdir) + def is_taken(path): - return _path_key(path) in moved_in or os.path.exists(path) + return _path_key(path) in moved_in or archives.exists_ignoring_case(path, listdir) while True: owners = defaultdict(set) @@ -128,7 +143,8 @@ def describe(operation) -> str: return (f" >> Would unpack: {Fore.GREEN}{operation.source}{Style.RESET_ALL} " f"to {Fore.BLUE}{target}{Style.RESET_ALL}{note}") if operation.kind == Kind.PACK: - return (f" >> Would pack: {Fore.GREEN}{operation.source}{Style.RESET_ALL} to {operation.packing_format} " + source = operation.source + (os.sep if os.path.isdir(operation.source) else "") + return (f" >> Would pack: {Fore.GREEN}{source}{Style.RESET_ALL} to {operation.packing_format} " f"in {Fore.BLUE}{target}{Style.RESET_ALL}") return f" >> Would move: {Fore.GREEN}{operation.source}{Style.RESET_ALL} to {Fore.BLUE}{target}{Style.RESET_ALL}" diff --git a/src/tags.py b/src/tags.py index 08cf853..3f1baec 100644 --- a/src/tags.py +++ b/src/tags.py @@ -25,14 +25,17 @@ def get_from_filename(filename: str) -> list[str]: return [tag.lower() for tag in all_tags] def get_base_name(filename: str) -> str: - name_no_ext = os.path.splitext(filename)[0] - idx_1 = name_no_ext.find('(') - idx_2 = name_no_ext.find('[') + return strip_tags(os.path.splitext(filename)[0]) + +def strip_tags(name: str) -> str: + """The name before the first (...) or [...] tag, like a folder name without them.""" + idx_1 = name.find('(') + idx_2 = name.find('[') idx = min(idx_1, idx_2) if idx_1 != -1 and idx_2 != -1 else max(idx_1, idx_2) if idx != -1: - base = name_no_ext[:idx].strip() + base = name[:idx].strip() else: - base = name_no_ext.strip() + base = name.strip() return base # Disc tag, like "disc 2" or "track 03" @@ -79,5 +82,6 @@ def parse(cls, path: str) -> "Release": @property def game_key(self) -> str: - """The same for all the files of one game (each disc is a game of its own).""" - return f"{self.base_name} (Disc {self.disc})" if self.disc > -1 else self.base_name + """The same for all the files of one game (each disc is a game of its own), whatever the case of the name.""" + base_name = self.base_name.lower() + return f"{base_name} (disc {self.disc})" if self.disc > -1 else base_name diff --git a/tests/test_archives.py b/tests/test_archives.py index fbff13f..461b850 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -68,6 +68,16 @@ def test_own_folder_name(self): self.assertEqual(archives.own_folder_name("Game Vol. 1 (USA).7z", {"a", "b"}), "Game Vol. 1 (USA)") # romlm packs "Game.nes" into "Game.nes.zip" self.assertEqual(archives.own_folder_name("Game (USA).nes.zip", {"Game (USA).nes"}), "Game (USA)") + self.assertEqual(archives.own_folder_name("Game (USA).nes.zip", {"GAME (USA).NES"}), "Game (USA)") + + def test_exists_ignoring_case(self): + with tempfile.TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "Game (USA)")) + open(os.path.join(root, "Game (USA)", "game.iso"), "w").close() + self.assertTrue(archives.exists_ignoring_case(os.path.join(root, "game (usa)", "GAME.ISO"))) + self.assertTrue(archives.exists_ignoring_case(os.path.join(root, "Game (USA)"))) + self.assertFalse(archives.exists_ignoring_case(os.path.join(root, "Game (USA)", "game.cue"))) + self.assertFalse(archives.exists_ignoring_case(os.path.join(root, "Other", "game.iso"))) class ExtractionFolderTest(unittest.TestCase): diff --git a/tests/test_cli.py b/tests/test_cli.py index 1f9b2b9..873add9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -55,6 +55,23 @@ def test_nothing_to_do(self): self.assertIn("Nothing to do", result.stdout) +class PackGameFoldersTest(unittest.TestCase): + def test_game_folders_are_packed_and_sorted_as_a_whole(self): + with tempfile.TemporaryDirectory() as root: + for name in ("Sonic CD (USA)/Sonic CD (USA).cue", "Sonic CD (USA)/Sonic CD (USA) (Track 01).bin", + "Lunar (USA)/game.iso", "Zelda (USA).nes"): + os.makedirs(os.path.dirname(os.path.join(root, name)), exist_ok=True) + open(os.path.join(root, name), "w").close() + result = subprocess.run([sys.executable, ROMLM, "-i", root, "-p", "zip", "-s"], + capture_output=True, text=True, stdin=subprocess.DEVNULL) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(tree(root), sorted([os.path.join("S", "Sonic CD (USA).zip"), + os.path.join("L", "Lunar (USA).zip"), + os.path.join("Z", "Zelda (USA).nes.zip")])) + with zipfile.ZipFile(os.path.join(root, "S", "Sonic CD (USA).zip")) as z: + self.assertEqual(sorted(z.namelist()), ["Sonic CD (USA) (Track 01).bin", "Sonic CD (USA).cue"]) + + class ExtractTest(unittest.TestCase): ARCHIVES = { "Single (USA).zip": ["Single (USA).nes"], diff --git a/tests/test_discs.py b/tests/test_discs.py new file mode 100644 index 0000000..fc6ef9e --- /dev/null +++ b/tests/test_discs.py @@ -0,0 +1,82 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +import discs + + +def game_folders(*files): + return discs.find_game_folders([os.path.join(*f.split("/")) for f in files]) + + +class IsGameFolderTest(unittest.TestCase): + def test_disc_sheets_with_their_tracks(self): + for files in (["Game (USA).cue", "Game (USA).bin"], ["Game (USA).cue", "Game (USA).iso"], + ["Game (USA).cue", "Game (USA) (Track 01).bin", "Game (USA) (Track 02).bin"], + ["Game (USA).ccd", "Game (USA).img", "Game (USA).sub"], ["Game (USA).mds", "Game (USA).mdf"], + ["Game (USA).toc", "Game (USA).bin"], ["Game (Europe).cue", "Game (Europe).bin", "Game (Europe).sbi"], + ["Game.m3u", "Game (USA) (Disc 1).chd", "Game (USA) (Disc 2).chd"]): + with self.subTest(files=files): + self.assertTrue(discs.is_game_folder("Game (USA)", files)) + + def test_generic_names(self): + self.assertTrue(discs.is_game_folder("Saturn Game (USA)", ["game.iso"])) + self.assertTrue(discs.is_game_folder("DC Game (USA)", ["disc.gdi", "track01.bin", "track02.raw"])) + self.assertTrue(discs.is_game_folder("PC Game (USA)", ["Disc 1.cue", "Disc 1.bin", "PC Game (USA).txt"])) + + def test_extracted_discs_whatever_the_names(self): + self.assertTrue(discs.is_game_folder("Demon's Souls (USA)", ["PS3_DISC.SFB"], ["PS3_GAME"])) + self.assertTrue(discs.is_game_folder("BLUS30443", [], ["PS3_GAME"])) + self.assertTrue(discs.is_game_folder("PS2 Game", ["SYSTEM.CNF", "SLUS_200.62"], ["DATA"])) + self.assertTrue(discs.is_game_folder("PSP Game", ["UMD_DATA.BIN"], ["PSP_GAME"])) + self.assertTrue(discs.is_game_folder("Halo", ["default.xbe", "maps.dat"])) + + def test_folders_of_many_games_are_not_game_folders(self): + self.assertFalse(discs.is_game_folder("S", ["Snatcher (USA).cue", "Snatcher (USA).bin", + "Shining Force CD (USA).cue", "Shining Force CD (USA).bin"])) + self.assertFalse(discs.is_game_folder("Q", ["Quake (USA).cue", "Quake (USA).bin"])) + self.assertFalse(discs.is_game_folder("!Japan", ["Game (Japan).cue", "Game (Japan).bin"])) + + def test_files_of_one_game_in_a_folder_named_otherwise(self): + self.assertTrue(discs.is_game_folder("Final Fantasy VII (USA)", ["FF7.CUE", "FF7.BIN"])) + self.assertTrue(discs.is_game_folder("Sonic_CD", ["Sonic CD (USA).cue", "Sonic CD (USA) (Track 01).bin"])) + + def test_sort_folders_with_a_single_game_are_not_game_folders(self): + for folder in ("Q", "q", "1-9", "!Homebrew", "!Japan", "!!BIOS"): + with self.subTest(folder=folder): + self.assertFalse(discs.is_game_folder(folder, ["Quake (USA).cue", "Quake (USA).bin"])) + + def test_folder_named_otherwise_with_subfolders_is_not_a_game_folder(self): + self.assertFalse(discs.is_game_folder("PS1", ["Quake (USA).cue", "Quake (USA).bin"], ["Other Game (USA)"])) + self.assertTrue(discs.is_game_folder("Quake (USA)", ["Quake (USA).cue", "Quake (USA).bin"], ["Extras"])) + + def test_folders_without_disc_hints_are_not_game_folders(self): + self.assertFalse(discs.is_game_folder("Zelda (USA)", ["Zelda (USA).nes"])) + self.assertFalse(discs.is_game_folder("Game (USA)", ["Game (USA).bin", "Game (USA).txt"])) + self.assertFalse(discs.is_game_folder("Empty", [])) + + def test_names_with_dots(self): + self.assertTrue(discs.is_game_folder("Game Vol. 1 (USA)", ["Game Vol. 1 (USA).cue", "Game Vol. 1 (USA).bin"])) + + +class FindGameFoldersTest(unittest.TestCase): + def test_finds_game_folders_at_any_depth(self): + self.assertEqual(game_folders("Sonic CD (USA)/Sonic CD (USA).cue", "Sonic CD (USA)/Sonic CD (USA).bin", + "PS3/Demon's Souls (USA)/PS3_DISC.SFB", + "PS3/Demon's Souls (USA)/PS3_GAME/USRDIR/EBOOT.BIN", + "Zelda (USA).nes", "N/NES Game (USA).nes"), + ["Sonic CD (USA)", os.path.join("PS3", "Demon's Souls (USA)")]) + + def test_inner_game_folders_are_a_part_of_the_outer_one(self): + self.assertEqual(game_folders("MGS (USA)/MGS (USA).m3u", "MGS (USA)/Disc 1/MGS (USA) (Disc 1).cue", + "MGS (USA)/Disc 1/MGS (USA) (Disc 1).bin", "MGS (USA)/Disc 2/disc.gdi"), + ["MGS (USA)"]) + + def test_root_is_never_a_game_folder(self): + self.assertEqual(game_folders("Game (USA).cue", "Game (USA).bin", "SYSTEM.CNF"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py index ccc9619..5ca365c 100644 --- a/tests/test_duplicates.py +++ b/tests/test_duplicates.py @@ -128,6 +128,10 @@ def test_removes_the_worse_files_and_returns_the_rest(self): self.assertEqual(self.left(), ["Game (USA).nes", "Other (USA).nes"]) self.assertEqual([os.path.basename(p) for p in kept], ["Game (USA).nes", "Other (USA).nes"]) + def test_names_in_another_case_are_duplicates(self): + self.clean(self.make("Pac-Man (USA).nes", "PAC-MAN (Japan).nes")) + self.assertEqual(self.left(), ["Pac-Man (USA).nes"]) + def test_keep_all_keeps_the_tied_files(self): self.make("Game (USA) (Foo).nes", "Game (USA) (Bar).nes") self.clean([os.path.join(self.folder.name, n) for n in self.left()]) diff --git a/tests/test_operations.py b/tests/test_operations.py index 5ff46c7..a3c6951 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -55,6 +55,24 @@ def test_files_are_packed_but_archives_and_bios_are_not(self): Operation(Kind.MOVE, "!!BIOS/[BIOS] Sega CD (USA).bin", "out"), ]) + def test_game_folders_are_packed_as_a_whole(self): + files = [os.path.join("Sonic CD (USA)", "Sonic CD (USA).cue"), os.path.join("Sonic CD (USA)", "Sonic CD (USA).bin"), + "Zelda (USA).nes", os.path.join("S", "Snatcher (USA).cue")] + self.assertEqual(plan(files, lambda f: "out", pack=True), [ + Operation(Kind.PACK, "Sonic CD (USA)", "out", "zip"), + Operation(Kind.PACK, "Zelda (USA).nes", "out", "zip"), + Operation(Kind.PACK, os.path.join("S", "Snatcher (USA).cue"), "out", "zip"), + ]) + + def test_game_folder_goes_next_to_itself_without_sorting(self): + files = [os.path.join("CD", "Game (USA)", "Game (USA).cue"), os.path.join("CD", "Game (USA)", "Game (USA).bin")] + self.assertEqual(plan(files, pack=True), [Operation(Kind.PACK, os.path.join("CD", "Game (USA)"), "CD", "zip")]) + + def test_game_folders_are_left_alone_without_packing(self): + files = [os.path.join("Game (USA)", "Game (USA).cue"), os.path.join("Game (USA)", "Game (USA).bin")] + self.assertEqual(plan(files), []) + self.assertEqual([op.source for op in plan(files, lambda f: "G")], files) + def test_packing_format_is_kept(self): self.assertEqual(plan(["A.nes"], pack=True, fmt="7z")[0].packing_format, "7z") @@ -101,6 +119,11 @@ def test_archive_does_not_overwrite_a_file_on_disk(self): self.archive("Other (USA).zip", "Other (USA).nes")), {"Game (USA) (Rev 1).zip": "Game (USA) (Rev 1)", "Other (USA).zip": "."}) + def test_file_on_disk_in_another_case_is_not_overwritten(self): + open(os.path.join(self.folder.name, "GAME (USA).NES"), "w").close() + self.assertEqual(self.resolve(self.archive("Game (USA) (Rev 1).zip", "Game (USA).nes")), + {"Game (USA) (Rev 1).zip": "Game (USA) (Rev 1)"}) + def test_archive_does_not_overwrite_a_file_moved_in_by_the_run(self): archive = self.archive("Game (USA) (Rev 1).zip", "Game (USA).nes") ops = [Operation(Kind.MOVE, os.path.join("Sub", "Game (USA).nes"), self.folder.name), @@ -198,6 +221,30 @@ def test_pack_and_unpack_zip(self): self.assertFalse(os.path.exists(archive)) self.assertEqual(read(self.path("back", "Game (USA).nes")), "rom") + def test_packs_a_folder_without_the_folder_itself(self): + self.touch("Game (USA)", "Game (USA).cue", content="cue") + self.touch("Game (USA)", "Extras", "Game (USA).bin", content="bin") + self.touch("Game (USA)", ".DS_Store") + self.touch("Game (USA)", "__MACOSX", "._Game (USA).cue") + _, log, error = operations.execute(Operation(Kind.PACK, self.path("Game (USA)"), self.path("out"), "zip")) + self.assertIsNone(error) + self.assertFalse(os.path.exists(self.path("Game (USA)"))) + with zipfile.ZipFile(self.path("out", "Game (USA).zip")) as z: + self.assertEqual(sorted(z.namelist()), ["Extras/Game (USA).bin", "Game (USA).cue"]) + self.assertEqual(z.read("Game (USA).cue"), b"cue") + + def test_folder_is_not_packed_into_an_existing_archive(self): + self.touch("Game (USA)", "Game (USA).cue", content="cue") + self.touch("Game (USA).zip", content="other") + self.touch("Other (USA)", "Other (USA).cue", content="cue") + self.touch("OTHER (USA).ZIP", content="other") + _, _, error = operations.execute(Operation(Kind.PACK, self.path("Other (USA)"), self.root, "zip")) + self.assertIn("already exists", error) + _, _, error = operations.execute(Operation(Kind.PACK, self.path("Game (USA)"), self.root, "zip")) + self.assertIn("already exists", error) + self.assertEqual(read(self.path("Game (USA).zip")), "other") + self.assertEqual(read(self.path("Game (USA)", "Game (USA).cue")), "cue") + def test_unpacks_an_archive_with_an_upper_case_extension(self): archive = self.path("Up (USA).ZIP") with zipfile.ZipFile(archive, "w") as z: @@ -307,6 +354,20 @@ def test_pack_and_unpack_7z(self): archives.unpack(archive, os.path.join(root, "back")) self.assertEqual(read(os.path.join(root, "back", "Game (USA).nes")), "rom") + def test_pack_a_folder_to_7z_and_back(self): + with tempfile.TemporaryDirectory() as root: + folder = os.path.join(root, "Gamé (USA)") + os.makedirs(os.path.join(folder, "Sub")) + for name in ("Gamé (USA).cue", os.path.join("Sub", "track 01.bin")): + with open(os.path.join(folder, name), "w") as f: + f.write(name) + archive = archives.pack(folder, root, "7z") + self.assertEqual(archive, os.path.join(root, "Gamé (USA).7z")) + self.assertFalse(os.path.exists(folder)) + self.assertEqual(sorted(archives.list_files(archive)), ["Gamé (USA).cue", "Sub/track 01.bin"]) + archives.unpack(archive, folder) + self.assertEqual(read(os.path.join(folder, "Sub", "track 01.bin")), os.path.join("Sub", "track 01.bin")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_tags.py b/tests/test_tags.py index a171912..174dc6c 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -25,12 +25,17 @@ def test_empty_tag_is_kept_as_empty_string(self): def test_each_disc_is_a_game_of_its_own(self): disc_1 = Release.parse("Game (USA) (Disc 1).bin") disc_2 = Release.parse("Game (Europe) (Disc 2).bin") - self.assertEqual(disc_1.game_key, "Game (Disc 1)") + self.assertEqual(disc_1.game_key, "game (disc 1)") self.assertNotEqual(disc_1.game_key, disc_2.game_key) def test_regions_do_not_split_a_game(self): self.assertEqual(Release.parse("Game (USA).nes").game_key, Release.parse("Game (Japan).nes").game_key) + def test_case_does_not_split_a_game(self): + self.assertEqual(Release.parse("Pac-Man (USA).nes").game_key, Release.parse("PAC-MAN (Japan).nes").game_key) + self.assertEqual(Release.parse("Game (USA) (Disc 1).bin").game_key, + Release.parse("GAME (Japan) (DISC 1).bin").game_key) + def test_bios(self): release = Release.parse("!!BIOS/[BIOS] Sega CD (USA).bin") self.assertTrue(release.is_bios) From 2f1c92d6e351e4fc1e8207fdd377d67254f15de4 Mon Sep 17 00:00:00 2001 From: Ilia Petrov-Komotskii Date: Fri, 23 Oct 2026 23:54:31 +0200 Subject: [PATCH 6/6] fix: Fixed .7z and path issues on Windows --- src/archives.py | 5 +++-- tests/test_romlm.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/archives.py b/src/archives.py index 0605bd8..8cec2ac 100644 --- a/src/archives.py +++ b/src/archives.py @@ -93,9 +93,10 @@ def _list_paths(file_name): with zipfile.ZipFile(file_name, 'r') as archive: return [info.filename for info in archive.infolist() if not info.is_dir()] # The technical listing has a "Key = Value" block per entry, after the archive's own block ending with dashes - result = subprocess.run([find_seven_zip_binary(), "l", "-slt", os.path.abspath(file_name)], + # -sccUTF-8 since 7-Zip prints in the OEM code page on Windows otherwise + result = subprocess.run([find_seven_zip_binary(), "l", "-slt", "-sccUTF-8", os.path.abspath(file_name)], check=True, capture_output=True) - _, _, listing = result.stdout.decode(errors="replace").partition("\n----------") + _, _, listing = result.stdout.decode("utf-8", errors="replace").partition("\n----------") paths = [] for block in re.split(r"\n\s*\n", listing): fields = dict(line.split(" = ", 1) for line in block.splitlines() if " = " in line) diff --git a/tests/test_romlm.py b/tests/test_romlm.py index b0f2b9d..d032318 100644 --- a/tests/test_romlm.py +++ b/tests/test_romlm.py @@ -66,7 +66,7 @@ def make(self, *paths): open(full, "w").close() def folders(self): - return sorted(os.path.relpath(d, self.root) for d, _, _ in os.walk(self.root)) + return sorted(os.path.relpath(d, self.root).replace(os.sep, "/") for d, _, _ in os.walk(self.root)) def clean(self, is_log_enabled=False): output = io.StringIO()