Fixed faction image fetching
CI / quality (3.11) (push) Has been cancelled
CI / quality (3.12) (push) Has been cancelled

This commit is contained in:
2026-10-15 18:12:27 +02:00
parent d651455499
commit 3b886acaba
11 changed files with 1301 additions and 6 deletions
+51
View File
@@ -13,9 +13,11 @@ from rich.table import Table
from genshin_tcg import __version__
from genshin_tcg.config import CardSettings, Settings
from genshin_tcg.data.catalog import load_catalog
from genshin_tcg.data.fandom import FandomError
from genshin_tcg.data.paimon import SourceError
from genshin_tcg.logging_setup import configure_logging
from genshin_tcg.pipeline import build as build_pipeline
from genshin_tcg.pipeline import factions as factions_pipeline
from genshin_tcg.pipeline import fetch as fetch_pipeline
from genshin_tcg.pipeline import impose as impose_pipeline
from genshin_tcg.pipeline import render as render_pipeline
@@ -139,6 +141,55 @@ def impose(
)
@app.command()
def factions(
ctx: typer.Context,
overwrite: Annotated[
bool,
typer.Option("--overwrite", help="Discard hand-made entries and regenerate the file."),
] = False,
force: Annotated[
bool, typer.Option(help="Bypass the cached wiki pages and icons.")
] = False,
icons: Annotated[
bool,
typer.Option("--icons/--no-icons", help="Download nation emblems for the factions."),
] = True,
dry_run: Annotated[
bool, typer.Option("--dry-run", help="Resolve and print without writing the file.")
] = False,
) -> None:
"""Resolve card factions/nations from the Genshin Fandom wiki."""
state = _state(ctx)
try:
result = factions_pipeline(
state.settings, overwrite=overwrite, force=force, icons=icons, write=not dry_run
)
except (FandomError, FileNotFoundError) as exc:
_fail(exc)
resolution = result.resolution
console.print(
f"[green]Factions:[/green] {len(resolution.assignments)} card(s) "
f"({len(resolution.playable)} nation, {len(resolution.enemies)} enemy)"
)
if resolution.unmatched:
console.print(f"[yellow]Unmatched:[/yellow] {', '.join(resolution.unmatched)}")
if result.icons is not None:
console.print(
f"[green]Icons:[/green] {len(result.icons.downloaded)} downloaded, "
f"{len(result.icons.skipped)} cached"
)
if result.icons.missing:
console.print(f"[dim]No emblem for: {', '.join(result.icons.missing)}[/dim]")
if result.icons.failed:
console.print(f"[yellow]Icon failures:[/yellow] {', '.join(result.icons.failed)}")
if result.written:
console.print(f"[green]Written:[/green] {result.output} (+{len(result.added)} new)")
else:
for card_id, faction in sorted(resolution.assignments.items()):
console.print(f" {card_id}: {faction}")
@app.command()
def build(
ctx: typer.Context,
+9
View File
@@ -116,10 +116,18 @@ class Settings(BaseSettings):
def tags_json(self) -> Path:
return self.source_dir / f"tags_{self.language}.json"
@property
def wiki_dir(self) -> Path:
return self.source_dir / "wiki"
@property
def tcg_dir(self) -> Path:
return self.assets_dir / "tcg"
@property
def faction_icons_dir(self) -> Path:
return self.tcg_dir / "icons" / "factions"
@property
def factions_override(self) -> Path:
return self.assets_dir / "overrides" / "factions.json"
@@ -144,6 +152,7 @@ class Settings(BaseSettings):
"""Create every directory the pipeline writes to."""
for directory in (
self.source_dir,
self.wiki_dir,
self.tcg_dir,
self.html_dir,
self.png_dir,
+1
View File
@@ -96,6 +96,7 @@ class Faction(StrEnum):
FONTAINE = "fontaine"
NATLAN = "natlan"
SNEZHNAYA = "snezhnaya"
NOD_KRAI = "nod_krai"
KHAENRIAH = "khaenriah"
FATUI = "fatui"
HILICHURL = "hilichurl"
+14
View File
@@ -3,14 +3,28 @@
from __future__ import annotations
from genshin_tcg.data.catalog import Catalog, load_catalog, load_raw_source, save_catalog
from genshin_tcg.data.fandom import (
FactionIconReport,
FactionResolution,
FandomError,
FandomSource,
resolve_factions,
sync_faction_icons,
)
from genshin_tcg.data.paimon import DownloadReport, PaimonSource, SourceError
__all__ = [
"Catalog",
"DownloadReport",
"FactionIconReport",
"FactionResolution",
"FandomError",
"FandomSource",
"PaimonSource",
"SourceError",
"load_catalog",
"load_raw_source",
"resolve_factions",
"save_catalog",
"sync_faction_icons",
]
+467
View File
@@ -0,0 +1,467 @@
"""Faction resolution from the Genshin Impact community wiki (Fandom).
``paimon-moe`` ships no faction/nation metadata, so the override file consumed
by the renderer (``assets/overrides/factions.json``) is derived from the
community wiki through its MediaWiki API:
* **playable characters** inherit the ``Region`` column of ``Character/List``
(itself generated by ``Template:Characters by Category Table``), and
* **enemies / bosses** inherit their ``family`` from ``Enemy/List``.
Every extractor works on the *rendered* HTML returned by ``action=parse`` so it
only depends on stable, user-visible structure (tables and card lists) rather
than on template internals. Card ids are matched against wiki names with
:func:`slug`, which mirrors the ``a-z0-9_`` convention of the dataset.
The wiki is only ever read: results are cached under ``assets/source/wiki`` so
repeated runs (and offline usage) stay deterministic.
"""
from __future__ import annotations
import logging
import re
import unicodedata
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Protocol
import httpx
from bs4 import BeautifulSoup, Tag
from genshin_tcg.constants import Faction
from genshin_tcg.logging_setup import get_logger
from genshin_tcg.models import Card, CharacterCard
__all__ = [
"DEFAULT_WIKI_BASE_URL",
"FactionIconReport",
"FactionIconSource",
"FactionResolution",
"FactionSource",
"FandomError",
"FandomSource",
"faction_from_family",
"faction_from_region",
"parse_character_regions",
"parse_enemy_families",
"parse_region_emblems",
"resolve_factions",
"slug",
"sync_faction_icons",
]
DEFAULT_WIKI_BASE_URL = "https://genshin-impact.fandom.com"
_CHARACTER_LIST_PAGE = "Character/List"
_ENEMY_LIST_PAGE = "Enemy/List"
#: Enemy families that map to a dedicated faction instead of the ``monster`` fallback.
_FAMILY_FACTIONS: dict[str, Faction] = {
"fatui": Faction.FATUI,
"hilichurls": Faction.HILICHURL,
}
_DEFAULT_FAMILY_FACTION = Faction.MONSTER
class FandomError(RuntimeError):
"""Raised when the wiki cannot be reached or a page cannot be parsed."""
class FactionSource(Protocol):
"""Anything able to provide wiki-derived faction lookups (network or fake)."""
def character_regions(self) -> dict[str, str]:
"""Return ``{slug(character name): region title}`` for playable characters."""
def enemy_families(self) -> dict[str, str]:
"""Return ``{slug(enemy name): family}`` for enemies and bosses."""
class FactionIconSource(Protocol):
"""Anything able to provide and download faction emblem images."""
def region_emblems(self) -> dict[str, str]:
"""Return ``{region title: wiki file name}`` for the nation emblems."""
def file_urls(self, names: Sequence[str]) -> dict[str, str]:
"""Resolve wiki ``File:`` names to their original download URLs."""
def download(self, url: str, destination: Path, *, force: bool = False) -> bool:
"""Download *url* into *destination*; return ``True`` when it was written."""
def slug(text: str) -> str:
"""Normalise *text* to the ``a-z0-9_`` form shared by card ids.
Apostrophes (as in ``Khaenri'ah``) are dropped rather than turned into a
separator so that wiki names line up with their dataset ids.
"""
ascii_text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii")
ascii_text = ascii_text.replace("'", "")
return re.sub(r"[^a-z0-9]+", "_", ascii_text.lower()).strip("_")
def faction_from_region(region: str) -> Faction | None:
"""Map a wiki ``Region`` title to a nation faction.
Returns ``None`` for regions that are not part of the :class:`Faction`
vocabulary (for example the wiki's literal ``None`` for region-less
characters such as the Traveller or Aloy).
"""
try:
return Faction(slug(region))
except ValueError:
return None
def faction_from_family(family: str) -> Faction:
"""Map a wiki enemy ``family`` to a faction, defaulting to ``monster``."""
return _FAMILY_FACTIONS.get(slug(family), _DEFAULT_FAMILY_FACTION)
def _character_table(html: str) -> tuple[list[str], list[Tag]]:
"""Return the ``Character/List`` header labels and body rows."""
soup = BeautifulSoup(html, "html.parser")
table = soup.find("table", class_="fandom-table")
if not isinstance(table, Tag):
raise FandomError("Could not find the character table on Character/List")
rows = table.find_all("tr")
if not rows:
raise FandomError("The character table on Character/List is empty")
header = [cell.get_text(strip=True) for cell in rows[0].find_all(["th", "td"])]
if "Name" not in header or "Region" not in header:
raise FandomError(f"Unexpected character table header: {header!r}")
return header, rows[1:]
def _region_of(cell: Tag) -> str:
"""Read a character's region title from its table cell (link target or text)."""
link = cell.find("a", href=True)
if isinstance(link, Tag) and link.has_attr("title"):
return str(link["title"])
return cell.get_text(strip=True)
def parse_character_regions(html: str) -> dict[str, str]:
"""Extract ``{slug(name): region}`` from a rendered ``Character/List`` page."""
header, rows = _character_table(html)
name_index = header.index("Name")
region_index = header.index("Region")
regions: dict[str, str] = {}
for row in rows:
cells = row.find_all("td", recursive=False)
if len(cells) <= max(name_index, region_index):
continue
name = cells[name_index].get_text(strip=True)
region = _region_of(cells[region_index])
if name and region:
regions.setdefault(slug(name), region)
return regions
def parse_region_emblems(html: str) -> dict[str, str]:
"""Extract ``{region title: wiki emblem file name}`` from ``Character/List``.
The emblem shown next to each region is the file the wiki itself uses, so
the downloaded icon matches what a reader sees on the wiki.
"""
header, rows = _character_table(html)
region_index = header.index("Region")
emblems: dict[str, str] = {}
for row in rows:
cells = row.find_all("td", recursive=False)
if len(cells) <= region_index:
continue
cell = cells[region_index]
region = _region_of(cell)
image = cell.find("img")
file_name = str(image.get("data-image-name", "")) if isinstance(image, Tag) else ""
if region and file_name:
emblems.setdefault(region, file_name)
return emblems
def parse_enemy_families(html: str) -> dict[str, str]:
"""Extract ``{slug(name): family}`` from a rendered ``Enemy/List`` page.
The page groups enemies under ``<h2>`` family headings, each followed by a
``card-list-container`` listing the offending enemies.
"""
soup = BeautifulSoup(html, "html.parser")
root = soup.find("div", class_="mw-parser-output")
if not isinstance(root, Tag):
root = soup
families: dict[str, str] = {}
current: str | None = None
for element in root.find_all(True):
if element.name == "h2":
headline = element.find("span", class_="mw-headline")
current = (headline or element).get_text(strip=True)
elif "card-list-container" in (element.get("class") or []):
if current is None:
continue
for anchor in element.find_all("a", href=True):
href = str(anchor.get("href", ""))
title = str(anchor.get("title", ""))
# Keep only the textual title link, not the illustration/icon links.
if not href.startswith("/wiki/") or ":" in href or anchor.find("img"):
continue
if title:
families.setdefault(slug(title), current)
return families
@dataclass(frozen=True, slots=True)
class FactionResolution:
"""Outcome of mapping a set of cards to faction tokens."""
assignments: dict[str, str] = field(default_factory=dict)
playable: tuple[str, ...] = ()
enemies: tuple[str, ...] = ()
unmatched: tuple[str, ...] = ()
@property
def matched(self) -> int:
return len(self.playable) + len(self.enemies)
def resolve_factions(cards: Iterable[Card], *, source: FactionSource) -> FactionResolution:
"""Resolve a faction token for every :class:`CharacterCard` in *cards*.
Playable characters are matched against the character table first; anything
else (bosses, enemies) falls back to the enemy families table. Cards that
appear in neither are reported in :attr:`FactionResolution.unmatched`.
"""
regions = source.character_regions()
families = source.enemy_families()
assignments: dict[str, str] = {}
playable: list[str] = []
enemies: list[str] = []
unmatched: list[str] = []
for card in cards:
if not isinstance(card, CharacterCard):
continue
region = regions.get(card.id)
faction = faction_from_region(region) if region is not None else None
if faction is not None:
assignments[card.id] = faction.value
playable.append(card.id)
continue
family = families.get(card.id)
if family is not None:
assignments[card.id] = faction_from_family(family).value
enemies.append(card.id)
continue
unmatched.append(card.id)
return FactionResolution(
assignments=assignments,
playable=tuple(playable),
enemies=tuple(enemies),
unmatched=tuple(unmatched),
)
@dataclass(frozen=True, slots=True)
class FactionIconReport:
"""Outcome of syncing faction emblem images into the assets directory."""
downloaded: tuple[str, ...] = ()
skipped: tuple[str, ...] = ()
missing: tuple[str, ...] = ()
failed: dict[str, str] = field(default_factory=dict)
@property
def ok(self) -> bool:
return not self.failed
def sync_faction_icons(
resolution: FactionResolution,
*,
source: FactionIconSource,
assets_dir: Path,
force: bool = False,
) -> FactionIconReport:
"""Download the wiki emblem of every nation faction used by *resolution*.
Emblems are stored as ``<assets_dir>/<faction>.png`` — exactly where the
renderer looks them up (``tcg/icons/factions/<faction>.png``). Non-nation
factions (``fatui``, ``hilichurl``, ``monster``) have no emblem and are
reported in :attr:`FactionIconReport.missing`.
"""
used = sorted(set(resolution.assignments.values()))
by_faction: dict[str, str] = {}
for region, file_name in source.region_emblems().items():
nation = faction_from_region(region)
if nation is not None:
by_faction.setdefault(nation.value, file_name)
wanted = {faction: by_faction[faction] for faction in used if faction in by_faction}
missing = tuple(faction for faction in used if faction not in by_faction)
urls = source.file_urls(list(wanted.values()))
downloaded: list[str] = []
skipped: list[str] = []
failed: dict[str, str] = {}
assets_dir.mkdir(parents=True, exist_ok=True)
for faction, file_name in sorted(wanted.items()):
url = urls.get(file_name)
if url is None:
failed[faction] = f"no image URL for {file_name!r}"
continue
try:
wrote = source.download(url, assets_dir / f"{faction}.png", force=force)
except FandomError as exc:
failed[faction] = str(exc)
continue
(downloaded if wrote else skipped).append(faction)
return FactionIconReport(
downloaded=tuple(downloaded),
skipped=tuple(skipped),
missing=missing,
failed=failed,
)
class FandomSource:
"""A caching MediaWiki client for the Genshin Impact Fandom wiki."""
def __init__(
self,
base_url: str = DEFAULT_WIKI_BASE_URL,
*,
cache_dir: Path | None = None,
force: bool = False,
client: httpx.Client | None = None,
logger: logging.Logger | None = None,
) -> None:
self._base_url = base_url.rstrip("/")
self._api_url = f"{self._base_url}/api.php"
self._cache_dir = cache_dir
self._force = force
self._logger = logger or get_logger("data.fandom")
self._owns_client = client is None
self._client = client or httpx.Client(
timeout=httpx.Timeout(30.0, connect=15.0),
follow_redirects=True,
transport=httpx.HTTPTransport(retries=3),
headers={"User-Agent": "genshin-tcg/0.1 (+https://github.com/)"},
)
# -- Context manager -------------------------------------------------------
def __enter__(self) -> FandomSource:
return self
def __exit__(self, *exc: object) -> None:
self.close()
def close(self) -> None:
if self._owns_client:
self._client.close()
# -- MediaWiki transport ---------------------------------------------------
def render_page(self, page: str, *, force: bool = False) -> str:
"""Return the rendered HTML of *page* (cached on disk when configured)."""
cached = self._cache_dir / f"{slug(page)}.html" if self._cache_dir else None
if cached is not None and cached.is_file() and not force:
self._logger.debug("Using cached wiki page %s", cached)
return cached.read_text(encoding="utf-8")
params: dict[str, str] = {
"action": "parse",
"page": page,
"prop": "text",
"redirects": "1",
"format": "json",
"formatversion": "2",
}
try:
response = self._client.get(self._api_url, params=params)
response.raise_for_status()
except httpx.HTTPError as exc: # pragma: no cover - network dependent
raise FandomError(f"Failed to fetch wiki page {page!r}: {exc}") from exc
payload: Any = response.json()
html = payload.get("parse", {}).get("text") if isinstance(payload, dict) else None
if not isinstance(html, str):
raise FandomError(f"Wiki page {page!r} did not return HTML")
if cached is not None:
cached.parent.mkdir(parents=True, exist_ok=True)
cached.write_text(html, encoding="utf-8")
self._logger.info("Fetched wiki page %r", page)
return html
# -- FactionSource ---------------------------------------------------------
def character_regions(self) -> dict[str, str]:
return parse_character_regions(self.render_page(_CHARACTER_LIST_PAGE, force=self._force))
def enemy_families(self) -> dict[str, str]:
return parse_enemy_families(self.render_page(_ENEMY_LIST_PAGE, force=self._force))
# -- FactionIconSource -----------------------------------------------------
def region_emblems(self) -> dict[str, str]:
return parse_region_emblems(self.render_page(_CHARACTER_LIST_PAGE, force=self._force))
def file_urls(self, names: Sequence[str]) -> dict[str, str]:
"""Resolve wiki ``File:`` names to their original (full-size) URLs."""
unique = sorted({name for name in names if name})
if not unique:
return {}
params: dict[str, str] = {
"action": "query",
"titles": "|".join(f"File:{name}" for name in unique),
"prop": "imageinfo",
"iiprop": "url",
"redirects": "1",
"format": "json",
"formatversion": "2",
}
try:
response = self._client.get(self._api_url, params=params)
response.raise_for_status()
except httpx.HTTPError as exc: # pragma: no cover - network dependent
raise FandomError(f"Failed to resolve wiki files: {exc}") from exc
payload: Any = response.json()
pages = payload.get("query", {}).get("pages", []) if isinstance(payload, dict) else []
urls: dict[str, str] = {}
for page in pages:
info = page.get("imageinfo") if isinstance(page, dict) else None
if isinstance(info, list) and info and info[0].get("url"):
urls[str(page["title"]).removeprefix("File:")] = str(info[0]["url"])
return urls
def download(self, url: str, destination: Path, *, force: bool = False) -> bool:
"""Download a wiki image into *destination*, skipping when cached.
The Fandom image CDN sits behind Cloudflare: it needs a wiki ``Referer``
and browser-like ``Sec-Fetch-*`` headers, and otherwise re-encodes images
to WebP. ``format=original`` therefore keeps the source format so the
``.png`` suffix stays truthful.
"""
if not force and destination.is_file() and destination.stat().st_size > 0:
return False
destination.parent.mkdir(parents=True, exist_ok=True)
separator = "&" if "?" in url else "?"
headers = {
"Referer": f"{self._base_url}/",
"Accept-Language": "en-US,en;q=0.9",
"Sec-Fetch-Dest": "image",
"Sec-Fetch-Mode": "no-cors",
"Sec-Fetch-Site": "cross-site",
}
try:
response = self._client.get(f"{url}{separator}format=original", headers=headers)
response.raise_for_status()
except httpx.HTTPError as exc: # pragma: no cover - network dependent
raise FandomError(f"Failed to download {url}: {exc}") from exc
destination.write_bytes(response.content)
return True
+106
View File
@@ -7,6 +7,7 @@ each step can be inspected before continuing.
from __future__ import annotations
import json
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path
@@ -14,17 +15,29 @@ from pathlib import Path
from genshin_tcg.config import Settings
from genshin_tcg.data.assets import download_required_assets
from genshin_tcg.data.catalog import Catalog, load_catalog, load_raw_source, save_catalog
from genshin_tcg.data.fandom import (
FactionIconReport,
FactionIconSource,
FactionResolution,
FactionSource,
FandomSource,
resolve_factions,
sync_faction_icons,
)
from genshin_tcg.data.paimon import DownloadReport, PaimonSource
from genshin_tcg.logging_setup import get_logger
from genshin_tcg.models import parse_catalog
from genshin_tcg.pdf.imposition import ImpositionGrid, ImpositionResult, impose_pdf
from genshin_tcg.rendering.context import load_faction_overrides
from genshin_tcg.rendering.converters import HtmlConverter, PlaywrightConverter, Viewport
from genshin_tcg.rendering.renderer import CardRenderer, RenderedCard, safe_name
__all__ = [
"BuildResult",
"FactionResult",
"FetchResult",
"build",
"factions",
"fetch",
"impose",
"render",
@@ -47,6 +60,18 @@ class BuildResult:
imposition: ImpositionResult
@dataclass(frozen=True, slots=True)
class FactionResult:
"""Outcome of :func:`factions` (wiki resolution + override file update)."""
resolution: FactionResolution
output: Path
written: bool
added: tuple[str, ...] = ()
conflicts: tuple[str, ...] = ()
icons: FactionIconReport | None = None
def fetch(
settings: Settings,
*,
@@ -151,6 +176,87 @@ def build(
return BuildResult(fetch=fetch_result, rendered=rendered, imposition=imposition)
def factions(
settings: Settings,
*,
overwrite: bool = False,
force: bool = False,
write: bool = True,
icons: bool = True,
source: FactionSource | None = None,
icon_source: FactionIconSource | None = None,
) -> FactionResult:
"""Resolve card factions from the wiki and refresh the override file.
Playable characters are matched to their wiki region and enemies to their
family (see :mod:`genshin_tcg.data.fandom`). Unless *overwrite* is set, the
existing ``assets/overrides/factions.json`` is preserved: hand-made entries
win over scraped ones and only missing keys are added, so the file stays
safe to edit by hand.
When *icons* is set, the nation emblems are downloaded into
``assets/tcg/icons/factions/<faction>.png`` so the renderer shows an image
instead of the text label. *source* / *icon_source* inject a fake wiki
(tests); *write* ``False`` performs a dry run.
"""
catalog = _load_catalogue(settings)
icon_report: FactionIconReport | None = None
if source is None:
with FandomSource(cache_dir=settings.wiki_dir, force=force) as wiki:
resolution = resolve_factions(catalog, source=wiki)
if icons:
icon_report = sync_faction_icons(
resolution,
source=icon_source or wiki,
assets_dir=settings.faction_icons_dir,
force=force,
)
else:
resolution = resolve_factions(catalog, source=source)
if icons and icon_source is not None:
icon_report = sync_faction_icons(
resolution,
source=icon_source,
assets_dir=settings.faction_icons_dir,
force=force,
)
existing = {} if overwrite else load_faction_overrides(settings.factions_override)
added = tuple(sorted(set(resolution.assignments) - set(existing)))
conflicts = tuple(
sorted(
card_id
for card_id, faction in resolution.assignments.items()
if card_id in existing and existing[card_id] != faction
)
)
merged: dict[str, str] = {**resolution.assignments, **existing}
if conflicts:
_logger.warning(
"Kept %d hand-made faction entry/entries over the wiki values: %s",
len(conflicts),
", ".join(conflicts),
)
output = settings.factions_override
written = False
if write:
output.parent.mkdir(parents=True, exist_ok=True)
payload = json.dumps(dict(sorted(merged.items())), ensure_ascii=False, indent=2)
output.write_text(payload + "\n", encoding="utf-8")
written = True
_logger.info("Wrote %d faction override(s) to %s", len(merged), output)
return FactionResult(
resolution=resolution,
output=output,
written=written,
added=added,
conflicts=conflicts,
icons=icon_report,
)
def _load_catalogue(settings: Settings) -> Catalog:
if not settings.catalog_path.is_file():
raise FileNotFoundError(
+1
View File
@@ -68,6 +68,7 @@ _DEFAULT_LABELS: dict[str, str] = {
"fontaine": "Fontaine",
"natlan": "Natlan",
"snezhnaya": "Snezhnaya",
"nod_krai": "Nod-Krai",
"khaenriah": "Khaenri'ah",
"fatui": "Fatui",
"hilichurl": "Brutocollinus",