diff --git a/CHANGELOG.md b/CHANGELOG.md index c1edec2..24b8ec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Index ePub documents for full-text search, with a new `get_epub_index_data` helper and automatic indexing of ePub items, like PDFs already are (#333) - Send a Wikimedia-compliant default `User-Agent` in `stream_file` when downloading from `upload.wikimedia.org` (#214) +- Add `add_alias` to `zim/creator` and `add_aliases_to_zim` to `zim/filesystem` to support ZIM aliases, alongside existing redirects (#148) ### Fixed diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index c6dfd5b..42cae44 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -467,6 +467,36 @@ def add_redirect( self.can_finish = False # pragma: no cover raise + def add_alias( # pyright: ignore[reportIncompatibleMethodOverride] + self, + path: str, + target_path: str, + title: str | None = "", + *, + is_front: bool | None = None, + duplicate_ok: bool | None = None, + ): + """Add an alias from path to target_path + + title is optional. when set, the alias itself + can be found on suggestions (indexed) if considered FRONT_ARTICLE""" + hints: dict[libzim.writer.Hint, int] = {} + if is_front is not None: + hints[libzim.writer.Hint.FRONT_ARTICLE] = bool(is_front) + + duplicate_ok = duplicate_ok or self.ignore_duplicates + + try: + try: + super().add_alias(path, title or path, target_path, hints) + except RuntimeError as exc: + if not DUPLICATE_EXC_STR.match(str(exc)) or not duplicate_ok: + raise exc + except Exception: + if self.workaround_nocancel: + self.can_finish = False # pragma: no cover + raise + def finish( self, _: type[BaseException] | None = None, diff --git a/src/zimscraperlib/zim/dedup.py b/src/zimscraperlib/zim/dedup.py index 7f9f2cf..cf1517d 100644 --- a/src/zimscraperlib/zim/dedup.py +++ b/src/zimscraperlib/zim/dedup.py @@ -3,7 +3,6 @@ from typing import Any import xxhash -from libzim.writer import Hint # pyright: ignore[reportMissingModuleSource] from zimscraperlib.zim.creator import Creator @@ -69,9 +68,9 @@ def add_item_for( if existing_item := self.added_items.get(digest): self.creator.add_alias( path, - targetPath=existing_item, - title=title or path, - hints={Hint.FRONT_ARTICLE: True} if kwargs.get("is_front") else {}, + existing_item, + title or path, + is_front=True if kwargs.get("is_front") else None, ) return else: diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 8960f90..31ca59f 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -9,6 +9,7 @@ Also included: - Add redirect from a list of (source, destination, title) strings +- Add alias from a list of (source, destination, title) strings Note: due to the lack of a cancel() method in the libzim itself, it is not possible to stop a zim creation process. Should an error occur in your code, a Zim file @@ -114,6 +115,33 @@ def add_redirects_to_zim( zim_file.add_redirect(path, target_url, title) +def add_aliases_to_zim( + zim_file: Creator, + aliases: Sequence[tuple[str, str, str | None]] | None = None, + aliases_file: pathlib.Path | None = None, +): + """add aliases from list of source/target or aliases file to zim""" + if aliases is None: + aliases = [] + for source_url, target_url, title in aliases: + zim_file.add_alias(source_url, target_url, title) + + if aliases_file: + with open(aliases_file) as fh: + for linenumber, line in enumerate(fh.readlines()): + match = re.match(r"^(.)\t(.+)\t(.*)\t(.+)$", line) + if not match: + logger.warning( + f"Aliases file: line {linenumber} does not match expected " + f"regexp: {line}" + ) + continue + namespace, path, title, target_url = match.groups() + if namespace.strip(): + path = f"{namespace.strip()}/{path}" + zim_file.add_alias(path, target_url, title) + + def make_zim_file( *, build_dir: pathlib.Path, @@ -135,6 +163,8 @@ def make_zim_file( without_fulltext_index: bool = False, # noqa: ARG001 redirects: Sequence[tuple[str, str, str]] | None = None, redirects_file: pathlib.Path | None = None, + aliases: Sequence[tuple[str, str, str]] | None = None, + aliases_file: pathlib.Path | None = None, rewrite_links: bool = True, # noqa: ARG001 workaround_nocancel: bool = True, ignore_duplicates: bool = True, @@ -146,6 +176,7 @@ def make_zim_file( illustration: relative path to illustration file in build_dir tags: list of str tags to add to meta redirects: list of (src, dst, title) tuple to create redirects from + aliases: list of (src, dst, title) tuple to create aliases from rewrite_links controls whether to rewrite HTML/CSS content -> add namespaces to relative links workaround_nocancel: disable workaround to prevent ZIM creation on error""" @@ -208,6 +239,10 @@ def make_zim_file( zim_file, redirects=redirects, redirects_file=redirects_file ) + if aliases or aliases_file: + logger.debug("Creating aliases") + add_aliases_to_zim(zim_file, aliases=aliases, aliases_file=aliases_file) + # prevents .finish() which would create an incomplete .zim file # this would leave a .zim.tmp folder behind. # UPSTREAM: wait until a proper cancel() is provided diff --git a/tests/zim/test_fs.py b/tests/zim/test_fs.py index 37a77dc..288750e 100644 --- a/tests/zim/test_fs.py +++ b/tests/zim/test_fs.py @@ -55,6 +55,60 @@ def test_redirects_file( ) +def test_aliases(png_image: pathlib.Path, build_data: dict[str, Any]): + build_data["build_dir"].mkdir() + shutil.copyfile(png_image, build_data["build_dir"] / png_image.name) + + make_zim_file( + build_dir=build_data["build_dir"], + fpath=build_data["fpath"], + name="test-zim", + main_page="welcome", + illustration=png_image.name, + title="Test ZIM", + description="A test ZIM", + aliases=[ + ("alias", "commons48.png", ""), + ("alias2", "commons48.png", "Alias 2"), + ], + ) + + reader = Archive(build_data["fpath"]) + for path in ("alias", "alias2"): + assert not reader.get_entry_by_path(path).is_redirect + assert bytes(reader.get_item(path).content) == png_image.read_bytes() + assert reader.get_entry_by_path("alias").title == "alias" + assert reader.get_entry_by_path("alias2").title == "Alias 2" + + +def test_aliases_file( + tmp_path: pathlib.Path, png_image: pathlib.Path, build_data: dict[str, Any] +): + build_data["build_dir"].mkdir() + shutil.copyfile(png_image, build_data["build_dir"] / png_image.name) + build_data["aliases_file"] = tmp_path / "aliases.tsv" + with open(build_data["aliases_file"], "w") as fh: + fh.write("A\tAccueil\t\tcommons48.png\n") + fh.write(" \timage\t\tcommons48.png\n") + fh.write("this_is_not_matching") + + make_zim_file( + build_dir=build_data["build_dir"], + fpath=build_data["fpath"], + name="test-zim", + main_page="welcome", + illustration=png_image.name, + title="Test ZIM", + description="A test ZIM", + aliases_file=build_data["aliases_file"], + ) + + reader = Archive(build_data["fpath"]) + for path in ("A/Accueil", "image"): + assert not reader.get_entry_by_path(path).is_redirect + assert bytes(reader.get_item(path).content) == png_image.read_bytes() + + def test_make_zim_file_fail_nobuildir(build_data: dict[str, Any]): # ensure we fail on missing build dir with pytest.raises(IOError): diff --git a/tests/zim/test_zim_creator.py b/tests/zim/test_zim_creator.py index e16e558..b709746 100644 --- a/tests/zim/test_zim_creator.py +++ b/tests/zim/test_zim_creator.py @@ -653,10 +653,13 @@ def test_duplicates(tmp_path: pathlib.Path): creator.add_item_for(path="A", content="A") creator.add_item_for(path="C", content="C") creator.add_redirect(path="B", target_path="A") + creator.add_alias(path="D", target_path="A") with pytest.raises(RuntimeError, match="existing dirent's title"): creator.add_item_for(path="A", content="test2") with pytest.raises(RuntimeError, match="existing dirent's title"): creator.add_redirect(path="B", target_path="C") + with pytest.raises(RuntimeError, match="existing dirent's title"): + creator.add_alias(path="D", target_path="C") def test_ignore_duplicates(tmp_path: pathlib.Path): @@ -667,6 +670,41 @@ def test_ignore_duplicates(tmp_path: pathlib.Path): creator.add_item_for(path="A", content="A2") creator.add_redirect(path="B", target_path="A") creator.add_redirect(path="B", target_path="C") + creator.add_alias(path="D", target_path="A") + creator.add_alias(path="D", target_path="C") + + +def test_alias_duplicate_ok(tmp_path: pathlib.Path): + with Creator(tmp_path / "test.zim", "").config_dev_metadata() as creator: + creator.add_item_for(path="A", content="A") + creator.add_item_for(path="C", content="C") + creator.add_alias(path="D", target_path="A") + creator.add_alias(path="D", target_path="C", duplicate_ok=True) + + +def test_add_alias(tmp_path: pathlib.Path, html_str: str): + fpath = tmp_path / "test.zim" + + with Creator(fpath, "welcome").config_dev_metadata() as creator: + creator.add_item_for("welcome", "wel", content=html_str, is_front=True) + creator.add_alias("welcome-alias", "welcome") + creator.add_alias("welcome-alias2", "welcome", "Home !!") + creator.add_alias("welcome-alias3", "welcome", "Home !!", is_front=True) + creator.add_alias("welcome-alias4", "welcome", "Home !!", is_front=False) + + reader = Archive(fpath) + for path in ( + "welcome-alias", + "welcome-alias2", + "welcome-alias3", + "welcome-alias4", + ): + entry = reader.get_entry_by_path(path) + assert not entry.is_redirect + assert bytes(entry.get_item().content).decode(UTF8) == html_str + assert reader.get_entry_by_path("welcome-alias").title == "welcome-alias" + assert reader.get_entry_by_path("welcome-alias2").title == "Home !!" + assert reader.main_entry.get_item().path == "welcome" def test_without_metadata(tmp_path: pathlib.Path):