From 30eb1ed8df6b1c435ae69035c7d1e6e9b2618b2b Mon Sep 17 00:00:00 2001 From: Florian Greistorfer Date: Mon, 27 Oct 2025 09:51:00 +0100 Subject: [PATCH] added metadata dataclass --- .version | 2 +- builder.py | 17 ++-- modules/css_color.py | 3 +- modules/datatypes/metadata.py | 157 ++++++++++++++++++++++++++++++++++ modules/generate_html.py | 100 +++++++++++----------- 5 files changed, 218 insertions(+), 61 deletions(-) create mode 100644 modules/datatypes/metadata.py diff --git a/.version b/.version index adaf203..f3ac133 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.8.6 \ No newline at end of file +2.9.0 \ No newline at end of file diff --git a/builder.py b/builder.py index 05500c8..f521daf 100755 --- a/builder.py +++ b/builder.py @@ -37,8 +37,8 @@ NOT_LIST = ["*/Galleries/*", "Archives"] args = parse_arguments(VERSION) -lock_file = os.path.join(args.root_directory, ".lock") -if os.path.exists(lock_file): +LOCKFILE = os.path.join(args.root_directory, ".lock") +if os.path.exists(LOCKFILE): print("Another instance of this program is running.") sys.exit() else: @@ -168,12 +168,12 @@ def main(args) -> None: """ Main function to process images and generate a static image hosting website. """ - thumbnails: list[tuple[str, str, str, bool]] = [] + thumbnails: list[tuple[str, str, str]] = [] args, raw = init_globals(args, RAW_EXTENSIONS) try: - Path(lock_file).touch() + Path(LOCKFILE).touch() logger.info("starting builder", extra={"version": VERSION, "arguments": args}) logger.info("getting logo from sorogon.eu") @@ -195,10 +195,11 @@ def main(args) -> None: logger.warning("reread metadata flag is set to true, all image metadata will be reread") if args.regenerate_thumbnails: logger.warning("regenerate thumbnails flag is set to true, all thumbnails will be regenerated") - if os.path.exists(os.path.join(args.root_directory, ".thumbnails")): + thumbdir = os.path.join(args.root_directory, ".thumbnails") + if os.path.exists(thumbdir): logger.info("removing old thumbnails folder") - shutil.rmtree(os.path.join(args.root_directory, ".thumbnails")) - os.makedirs(os.path.join(args.root_directory, ".thumbnails"), exist_ok=True) + shutil.rmtree(thumbdir) + os.makedirs(thumbdir, exist_ok=True) copy_static_files(args) icons(args) @@ -230,7 +231,7 @@ def main(args) -> None: ): pass finally: - os.remove(lock_file) + os.remove(LOCKFILE) logger.info("finished builder", extra={"version": VERSION}) diff --git a/modules/css_color.py b/modules/css_color.py index fbc1c2f..1b46915 100644 --- a/modules/css_color.py +++ b/modules/css_color.py @@ -95,7 +95,8 @@ def css_color_to_hex(css_color: str) -> str: # Helper function to convert HSL tuple to RGB tuple def hsl_to_rgb(hsl: tuple[int, float, float]) -> tuple[int, int, int]: logger.debug("converting hsl tuple to rgb tuple", extra={"hsl": hsl}) - return tuple(round(c * 255) for c in colorsys.hls_to_rgb(hsl[0] / 360, hsl[1] / 100, hsl[2] / 100)) + r, g, b = colorsys.hls_to_rgb(hsl[0] / 360, hsl[1] / 100, hsl[2] / 100) + return (round(r * 255), round(g * 255), round(b * 255)) # Regular expression pattern to match CSS colors color_pattern = re.compile( diff --git a/modules/datatypes/metadata.py b/modules/datatypes/metadata.py new file mode 100644 index 0000000..7d0a3e7 --- /dev/null +++ b/modules/datatypes/metadata.py @@ -0,0 +1,157 @@ +from dataclasses import dataclass +from typing import List, Optional, Any, Dict, TypeVar, Callable, Type, cast + +T = TypeVar("T") + + +def from_int(x: Any) -> int: + assert isinstance(x, int) and not isinstance(x, bool) + return x + + +def from_str(x: Any) -> str: + assert isinstance(x, str) + return x + + +def from_list(f: Callable[[Any], T], x: Any) -> List[T]: + assert isinstance(x, list) + return [f(y) for y in x] + + +def from_none(x: Any) -> Any: + assert x is None + return x + + +def from_union(fs, x): + for f in fs: + try: + return f(x) + except AssertionError: + pass + assert False + + +def to_class(c: Type[T], x: Any) -> dict: + assert isinstance(x, c) + return cast(Any, x).to_dict() + + +def from_dict(f: Callable[[Any], T], x: Any) -> Dict[str, T]: + assert isinstance(x, dict) + return {k: f(v) for (k, v) in x.items()} + + +def from_native_dict(f: Callable[[Any], T], x: Any) -> Dict[Any, T]: + assert isinstance(x, dict) + return x + + +@dataclass +class ImageMetadata: + w: Optional[int] + h: Optional[int] + tags: List[str] + exifdata: Optional[Dict[str, Any]] + xmp: Optional[Dict[str, Any]] + src: str + msrc: str + name: str + title: str + tiff: Optional[str] = None + raw: Optional[str] = None + + @staticmethod + def from_dict(obj: Any) -> "ImageMetadata": + assert isinstance(obj, dict) + w = from_union([from_int, from_none], obj.get("w")) + h = from_union([from_int, from_none], obj.get("h")) + tags = from_list(from_str, obj.get("tags")) + exifdata = from_union([from_native_dict, from_none], obj.get("exifdata")) + xmp = from_union([from_native_dict, from_none], obj.get("xmp")) + src = from_str(obj.get("src")) + msrc = from_str(obj.get("msrc")) + name = from_str(obj.get("name")) + title = from_str(obj.get("title")) + tiff = from_union([from_str, from_none], obj.get("tiff")) + raw = from_union([from_str, from_none], obj.get("raw")) + return ImageMetadata(w, h, tags, exifdata, xmp, src, msrc, name, title, tiff, raw) + + def to_dict(self) -> dict: + result: dict = {} + if self.w is not None: + result["w"] = from_union([from_int, from_none], self.w) + if self.h is not None: + result["h"] = from_union([from_int, from_none], self.h) + result["tags"] = from_list(from_str, self.tags) + result["src"] = from_str(self.src) + result["msrc"] = from_str(self.msrc) + result["name"] = from_str(self.name) + result["title"] = from_str(self.title) + if self.tiff is not None: + result["tiff"] = from_union([from_str, from_none], self.tiff) + if self.raw is not None: + result["raw"] = from_union([from_str, from_none], self.raw) + if self.exifdata is not None: + result["exifdata"] = from_union([lambda x: from_native_dict(dict, x), from_none], self.exifdata) + if self.xmp is not None: + result["xmp"] = from_union([lambda x: from_native_dict(dict, x), from_none], self.xmp) + return result + + +@dataclass +class SubfolderMetadata: + url: str + name: str + metadata: Optional[str] = None + thumb: Optional[str] = None + + @staticmethod + def from_dict(obj: Any) -> "SubfolderMetadata": + assert isinstance(obj, dict) + url = from_str(obj.get("url")) + name = from_str(obj.get("name")) + metadata = from_union([from_none, from_str], obj.get("metadata")) + thumb = from_union([from_none, from_str], obj.get("thumb")) + return SubfolderMetadata(url, name, metadata, thumb) + + def to_dict(self) -> dict: + result: dict = {} + result["url"] = from_str(self.url) + result["name"] = from_str(self.name) + result["metadata"] = from_union([from_none, from_str], self.metadata) + result["thumb"] = from_union([from_none, from_str], self.thumb) + return result + + +@dataclass +class Metadata: + images: Dict[str, ImageMetadata] + subfolders: Optional[List[SubfolderMetadata]] = None + + @staticmethod + def from_dict(obj: Any) -> "Metadata": + assert isinstance(obj, dict) + images = from_dict(ImageMetadata.from_dict, obj.get("images")) + subfolders = from_union([lambda x: from_list(SubfolderMetadata.from_dict, x), from_none], obj.get("subfolders")) + return Metadata(images, subfolders) + + def to_dict(self) -> dict: + result: dict = {} + result["images"] = from_dict(lambda x: to_class(ImageMetadata, x), self.images) + if self.subfolders is not None: + result["subfolders"] = from_union([lambda x: from_list(lambda x: to_class(SubfolderMetadata, x), x), from_none], self.subfolders) + return result + + def sort(self, reverse=False) -> "Metadata": + self.images = {key: self.images[key] for key in sorted(self.images, reverse=reverse)} + return self + + +def top_level_from_dict(s: Any) -> Metadata: + return Metadata.from_dict(s) + + +def top_level_to_dict(x: Metadata) -> Any: + return to_class(Metadata, x) diff --git a/modules/generate_html.py b/modules/generate_html.py index 8912ad3..cc058a5 100644 --- a/modules/generate_html.py +++ b/modules/generate_html.py @@ -17,6 +17,7 @@ from bs4 import BeautifulSoup from modules.logger import logger from modules import cclicense from modules.argumentparser import Args +from modules.datatypes.metadata import Metadata, ImageMetadata, SubfolderMetadata # Constants for file paths and exclusions if __package__ is None: @@ -73,7 +74,7 @@ def getxmp(strbuffer: str) -> dict[str, Any]: return {get_name(root.tag): get_value(root)} -def initialize_metadata(folder: str) -> dict[str, dict[str, int]]: +def initialize_metadata(folder: str) -> Metadata: """ Initializes the metadata JSON file if it doesn't exist. @@ -128,10 +129,10 @@ def initialize_metadata(folder: str) -> dict[str, dict[str, int]]: if "title" not in v: metadata["images"][k]["title"] = v["name"] - return metadata + return Metadata.from_dict(metadata) -def update_metadata(metadata: dict[str, dict[str, Any]], folder: str) -> None: +def update_metadata(metadata: Metadata, folder: str) -> None: """ Updates the metadata JSON file. @@ -143,14 +144,14 @@ def update_metadata(metadata: dict[str, dict[str, Any]], folder: str) -> None: if metadata: with open(metadata_path, "w", encoding="utf-8") as metadatafile: logger.info("writing metadata file", extra={"file": metadata_path}) - metadatafile.write(json.dumps(metadata, indent=4)) + metadatafile.write(json.dumps(metadata.to_dict(), indent=4)) else: if os.path.exists(metadata_path): logger.info("deleting empty metadata file", extra={"file": metadata_path}) os.remove(metadata_path) -def get_image_info(item: str, folder: str) -> dict[str, Any]: +def get_image_info(item: str, folder: str) -> ImageMetadata: """ Extracts image information and EXIF data. @@ -172,7 +173,7 @@ def get_image_info(item: str, folder: str) -> dict[str, Any]: except UnidentifiedImageError: logger.error("cannot identify image file", extra={"file": file}) print(f"cannot identify image file: {file}") - return {"w": None, "h": None, "tags": None, "exifdata": None, "xmp": None} + return ImageMetadata(w=None, h=None, tags=[], exifdata=None, xmp=None, src="", msrc="", name="", title="") if exif: logger.info("extracting EXIF data", extra={"file": file}) ifd = exif.get_ifd(ExifTags.IFD.Exif) @@ -253,9 +254,11 @@ def get_image_info(item: str, folder: str) -> dict[str, Any]: tags = get_tags(sidecarfile) except Exception as e: logger.error(e) - if None in tags: - tags.remove(None) - return {"w": width, "h": height, "tags": tags, "exifdata": exifdata, "xmp": xmp} + if None in tags: # type: ignore + tags.remove(None) # type: ignore + if not isinstance(tags, list): + tags = [] + return ImageMetadata(w=width, h=height, tags=tags, exifdata=exifdata, xmp=xmp, src="", msrc="", name="", title="") def nested_dict(): @@ -331,12 +334,12 @@ def get_tags(sidecarfile: str) -> list[str]: pass except KeyError: pass - if None in tags: - tags.remove(None) - return tags + if None in tags: # type: ignore + tags.remove(None) # type: ignore + return tags # type: ignore -def process_image(item: str, folder: str, _args: Args, baseurl: str, metadata: dict[str, dict[str, int]], raw: list[str]) -> dict[str, Any]: +def process_image(item: str, folder: str, _args: Args, baseurl: str, metadata: Metadata, raw: list[str]) -> tuple[ImageMetadata, Metadata]: """ Processes an image and prepares its data for the HTML template. @@ -353,24 +356,21 @@ def process_image(item: str, folder: str, _args: Args, baseurl: str, metadata: d """ extsplit = os.path.splitext(item) sidecarfile = os.path.join(folder, item + ".xmp") - if item not in metadata["images"] or _args.reread_metadata: - metadata["images"][item] = get_image_info(item, folder) + if item not in metadata.images or _args.reread_metadata: + metadata.images[item] = get_image_info(item, folder) if _args.reread_sidecar and os.path.exists(sidecarfile): logger.info("xmp sidecar file found", extra={"file": sidecarfile}) try: - metadata["images"][item]["tags"] = get_tags(sidecarfile) + metadata.images[item].tags = get_tags(sidecarfile) except Exception as e: logger.error(e) - image = { - "src": f"{_args.web_root_url}{baseurl}{urllib.parse.quote(item)}", - "msrc": f"{_args.web_root_url}.thumbnails/{baseurl}{urllib.parse.quote(item)}.jpg", - "name": item, - "w": metadata["images"][item]["w"], - "h": metadata["images"][item]["h"], - "tags": metadata["images"][item]["tags"], - "title": item, - } + image = metadata.images[item] + image.src = f"{_args.web_root_url}{baseurl}{urllib.parse.quote(item)}" + image.msrc = f"{_args.web_root_url}.thumbnails/{baseurl}{urllib.parse.quote(item)}.jpg" + image.name = item + image.title = item + path = os.path.join(_args.root_directory, ".thumbnails", baseurl, item + ".jpg") if not os.path.exists(path) or _args.regenerate_thumbnails: if os.path.exists(path): @@ -383,17 +383,17 @@ def process_image(item: str, folder: str, _args: Args, baseurl: str, metadata: d url = f"{_args.web_root_url}{baseurl}{urllib.parse.quote(extsplit[0])}{_raw}" if _raw in (".tif", ".tiff"): logger.info("tiff file found", extra={"file": file}) - image["tiff"] = url + image.tiff = url else: logger.info("raw file found", extra={"file": file, "extension": _raw}) - image["raw"] = url + image.raw = url - metadata["images"][item].update(image) + metadata.images[item] = image return image, metadata -def generate_html(folder: str, title: str, _args: Args, raw: list[str], version: str, logo: str) -> list[str]: +def generate_html(folder: str, title: str, _args: Args, raw: list[str], version: str, logo: str) -> set[str]: """ Generates HTML content for a folder of images. @@ -412,16 +412,16 @@ def generate_html(folder: str, title: str, _args: Args, raw: list[str], version: items = sorted(os.listdir(folder)) contains_files = False - images = [] - subfolders = [] - subfoldertags = set() + images: list[ImageMetadata] = [] + subfolders: list[SubfolderMetadata] = [] + subfoldertags: set[str] = set() foldername = folder.removeprefix(_args.root_directory) foldername = f"{foldername}/" if foldername else "" baseurl = urllib.parse.quote(foldername) - gone = [item for item in metadata["images"] if item not in items] + gone = [item for item in metadata.images if item not in items] for gon in gone: - del metadata["images"][gon] + del metadata.images[gon] create_thumbnail_folder(foldername, _args.root_directory) @@ -455,11 +455,11 @@ def generate_html(folder: str, title: str, _args: Args, raw: list[str], version: if item == "LICENSE": process_license(folder, item) - metadata["subfolders"] = subfolders + metadata.subfolders = subfolders if _args.reverse_sort: - metadata["images"] = {key: metadata["images"][key] for key in sorted(metadata["images"], reverse=True)} + metadata.sort(reverse=True) else: - metadata["images"] = {key: metadata["images"][key] for key in sorted(metadata["images"])} + metadata.sort() update_metadata(metadata, folder) if should_generate_html(images, contains_files, _args): @@ -485,7 +485,7 @@ def create_thumbnail_folder(foldername: str, root_directory: str) -> None: os.mkdir(thumbnails_path) -def process_subfolder(item: str, folder: str, baseurl: str, subfolders: list[dict[str, str | None]], _args: Args, raw: list[str], version: str, logo: str) -> list[str]: +def process_subfolder(item: str, folder: str, baseurl: str, subfolders: list[SubfolderMetadata], _args: Args, raw: list[str], version: str, logo: str) -> set[str]: """ Processes a subfolder. @@ -513,10 +513,10 @@ def process_subfolder(item: str, folder: str, baseurl: str, subfolders: list[dic if item not in _args.exclude_folders: if not any(fnmatch.fnmatchcase(os.path.join(folder, item), exclude) for exclude in _args.exclude_folders): - subfolders.append({"url": subfolder_url, "name": item, "thumb": thumb, "metadata": f"{_args.web_root_url}{baseurl}{urllib.parse.quote(item)}/.metadata.json"}) + subfolders.append(SubfolderMetadata(url=subfolder_url, name=item, thumb=thumb, metadata=f"{_args.web_root_url}{baseurl}{urllib.parse.quote(item)}/.metadata.json")) return generate_html(os.path.join(folder, item), os.path.join(folder, item).removeprefix(_args.root_directory), _args, raw, version, logo) - subfolders.append({"url": subfolder_url, "name": item, "thumb": thumb, "metadata": None}) - return [] + subfolders.append(SubfolderMetadata(url=subfolder_url, name=item, thumb=thumb)) + return set() def process_license(folder: str, item: str) -> None: @@ -548,7 +548,7 @@ def process_info_file(folder: str, item: str) -> None: info[urllib.parse.quote(folder)] = f.read() -def should_generate_html(images: list[dict[str, Any]], contains_files, _args: Args) -> bool: +def should_generate_html(images: list[ImageMetadata], contains_files, _args: Args) -> bool: """ Determines if HTML should be generated. @@ -559,7 +559,7 @@ def should_generate_html(images: list[dict[str, Any]], contains_files, _args: Ar Returns: bool: True if HTML should be generated, False otherwise. """ - return images or (_args.use_fancy_folders and not contains_files) or (_args.use_fancy_folders and _args.ignore_other_files) + return bool(images) or (_args.use_fancy_folders and not contains_files) or (_args.use_fancy_folders and _args.ignore_other_files) def format_html(html: str) -> str: @@ -568,8 +568,8 @@ def format_html(html: str) -> str: def create_html_file( - folder: str, title: str, foldername: str, images: list[dict[str, Any]], subfolders: list[dict[str, str]], _args: Args, version: str, logo: str, subfoldertags: list[str] -) -> list[str]: + folder: str, title: str, foldername: str, images: list[ImageMetadata], subfolders: list[SubfolderMetadata], _args: Args, version: str, logo: str, subfoldertags: set[str] +) -> set[str]: """ Creates the HTML file using the template. @@ -602,15 +602,13 @@ def create_html_file( alltags = set() for img in images: - if img["tags"]: - alltags.update(img["tags"]) + if img.tags: + alltags.update(img.tags) - alltags.update(set(subfoldertags)) + alltags.update(subfoldertags) folder_info = info.get(urllib.parse.quote(folder), "").split("\n") _info = [i for i in folder_info if len(i) > 1] if folder_info else None - if _args.reverse_sort: - images.sort(key=lambda i: i["name"], reverse=True) folder_license = licens.get(urllib.parse.quote(folder), False) @@ -661,7 +659,7 @@ def create_html_file( logger.info("writing formatted html file", extra={"path": html_file}) f.write(format_html(content)) - return sorted(alltags) + return set(sorted(alltags)) def list_folder(folder: str, title: str, _args: Args, raw: list[str], version: str, logo: str) -> list[tuple[str, str, str]]: