diff --git a/CLAUDE.md b/CLAUDE.md index e09537d..01826b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,7 @@ An implicit configuration system for Python. Tracks constructor arguments, suppo - `src/confingy/` — library source (src layout) - `tracking.py` — core `@track`, `lazy`, `lens`, `Lazy[T]`, `disable_validation` + - `tree.py` — `get_init_args`, `is_fingy_of`, `walk_fingy`, `map_fingy`, `replace_args` (tree read/match/transform) - `fingy.py` — `serialize_fingy`, `deserialize_fingy`, `save_fingy`, `load_fingy`, `transpile_fingy`, `prettify_fingy` - `serde.py` — serialization/deserialization internals (handlers, registry, keys) - `exceptions.py` — `ValidationError`, `SerializationError`, `DeserializationError` diff --git a/docs/api/tree.md b/docs/api/tree.md new file mode 100644 index 0000000..054060b --- /dev/null +++ b/docs/api/tree.md @@ -0,0 +1,3 @@ +# confingy.tree + +::: confingy.tree diff --git a/docs/api/utils/containers.md b/docs/api/utils/containers.md new file mode 100644 index 0000000..dce4665 --- /dev/null +++ b/docs/api/utils/containers.md @@ -0,0 +1,3 @@ +# confingy.utils.containers + +::: confingy.utils.containers diff --git a/docs/diving_deeper.md b/docs/diving_deeper.md index e54f7c2..00dcb75 100644 --- a/docs/diving_deeper.md +++ b/docs/diving_deeper.md @@ -163,10 +163,67 @@ print(modified_trainer.optimizer.lr) # 0.01 ``` !!! warning - `unlens()` will **reinstantiate** any tracked objects with the updated arguments. This means the returned object is a new instance, not the original with mutated state. + `unlens()` will **reinstantiate** modified tracked objects, and their ancestors, with the updated arguments. The returned objects are new instances, not the originals with mutated state. Nodes you didn't modify are returned as the original objects, and an object shared by several parents stays shared. For more examples, see [Lensing and Unlensing](examples/lens.md). +#### Transforming Whole Trees + +`lens()` works well when you know the path to the value you want to change. To change *every* object of a given class, wherever it appears in the tree, use the functions in [confingy.tree](api/tree.md): + +```python +from confingy import get_init_args, is_fingy_of, map_fingy, replace_args, track, update, walk_fingy + +@track +class Scaler: + def __init__(self, factor: float = 1.0): + self.factor = factor + +@track +class Pipeline: + def __init__(self, steps: list): + self.steps = steps + +@track +class Registry: + def __init__(self, entries: dict): + self.entries = entries + +registry = Registry(entries={ + "a": Pipeline(steps=[Scaler(), "passthrough"]), + "b": Pipeline(steps=[Scaler(factor=2.0)]), +}) + +# Inspect every tracked or lazy node +for path, node in walk_fingy(registry): + print(repr(path), type(node).__name__, get_init_args(node)) +# '' Registry {...} +# "entries['a']" Pipeline {...} +# "entries['a'].steps[0]" Scaler {'factor': 1.0} +# "entries['b']" Pipeline {...} +# "entries['b'].steps[0]" Scaler {'factor': 2.0} + +# Set arguments on every matching node +new_registry, n_rebuilt = replace_args(registry, Scaler, factor=3.0) +print(n_rebuilt) # 2 +print(new_registry.entries["a"].steps[0].factor) # 3.0 +print(registry.entries["a"].steps[0].factor) # 1.0: the input is unchanged + +# Or apply any node-level transform +def cap(node): + if is_fingy_of(node, Scaler) and get_init_args(node)["factor"] > 1.0: + return update(node)(factor=1.0) + return node + +capped = map_fingy(registry, cap) +print(capped.entries["a"] is registry.entries["a"]) # True: untouched subtrees are reused +``` + +[map_fingy()][confingy.tree.map_fingy] rebuilds a node with [update()][confingy.tracking.update] only when one of its children changed, so untouched subtrees keep their identity. Lazy nodes stay lazy, tracked nodes stay tracked, and a node shared by several parents is transformed once and stays shared. [is_fingy_of()][confingy.tree.is_fingy_of] matches tracked and lazy objects of a class or any subclass, including the subclasses created by `track(SomeClass)`. + +!!! note + Rebuilding re-runs each rebuilt node's constructor (or, for `Lazy` nodes, its validation and `__post_config__` hook), so any side effects happen again. Children are found through recorded constructor arguments, so attributes set after construction are not carried over. + ## Migrating to Confingy One of `confingy`'s central tenets is that it should be easy to migrate to without too many changes to your non-configuration code. For example, it could be painful to add [@confingy.track()][confingy.tracking.track] decorators to all of your classes, so you can instead use its functional form only when configuring your system: diff --git a/mkdocs.yml b/mkdocs.yml index 187a219..2c73066 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,6 +76,7 @@ nav: - API Reference: - confingy.fingy: api/fingy.md - confingy.tracking: api/tracking.md + - confingy.tree: api/tree.md - confingy.serde: api/serde.md - confingy.exceptions: api/exceptions.md - confingy.mypy_plugin: api/mypy_plugin.md @@ -85,6 +86,7 @@ nav: - confingy.cli.transpile: api/cli/transpile.md - confingy.cli.viz: api/cli/viz.md - Utils: + - confingy.utils.containers: api/utils/containers.md - confingy.utils.hashing: api/utils/hashing.md - confingy.utils.imports: api/utils/imports.md - confingy.utils.types: api/utils/types.md diff --git a/src/confingy/__init__.py b/src/confingy/__init__.py index 217596b..17afac5 100644 --- a/src/confingy/__init__.py +++ b/src/confingy/__init__.py @@ -32,6 +32,13 @@ track, update, ) +from confingy.tree import ( + get_init_args, + is_fingy_of, + map_fingy, + replace_args, + walk_fingy, +) # Set up logging for better debugging logger = logging.getLogger(__name__) @@ -44,6 +51,12 @@ "track", "update", "disable_validation", + # Tree utilities + "get_init_args", + "is_fingy_of", + "map_fingy", + "replace_args", + "walk_fingy", # Type hints and Core classes "Lazy", "MaybeLazy", diff --git a/src/confingy/tracking.py b/src/confingy/tracking.py index 02aa67b..1d9a3c8 100644 --- a/src/confingy/tracking.py +++ b/src/confingy/tracking.py @@ -24,6 +24,7 @@ from confingy.exceptions import ( ValidationError, ) +from confingy.utils.containers import same_structure from confingy.utils.hashing import hash_class from confingy.utils.imports import get_module_name from confingy.utils.types import is_lazy_instance, is_tracked_instance @@ -158,6 +159,7 @@ def _build_validation_model(cls: type[Any]) -> type[BaseModel]: hints = get_type_hints(cls.__init__, include_extras=False) fields: dict[str, tuple[Any, Any]] = {} + accepts_var_keyword = False for param_name, param in init_signature.parameters.items(): if param_name == "self": continue @@ -170,6 +172,7 @@ def _build_validation_model(cls: type[Any]) -> type[BaseModel]: fields[param_name] = (param_type, param.default) elif param.kind == inspect.Parameter.VAR_KEYWORD: # **kwargs + accepts_var_keyword = True fields[param_name] = (param_type, {}) else: fields[param_name] = (param_type, ...) # Required field @@ -177,7 +180,12 @@ def _build_validation_model(cls: type[Any]) -> type[BaseModel]: # Create the model, suppressing pydantic warnings about field names that # shadow BaseModel attributes (e.g. "schema", "validate", "copy"). # These validation models are ephemeral and the shadowing is harmless. - model_config: ConfigDict = {"arbitrary_types_allowed": True} + # Unknown arguments are rejected unless __init__ accepts **kwargs, so a typo + # fails when a Lazy is created rather than at instantiate(). + model_config: ConfigDict = { + "arbitrary_types_allowed": True, + "extra": "allow" if accepts_var_keyword else "forbid", + } with warnings.catch_warnings(): warnings.filterwarnings( "ignore", @@ -503,6 +511,10 @@ def unlens(self) -> Any: This enables a round-trip: `lens(obj) -> modify -> unlens()` preserves whether each node was originally Lazy or instantiated. + Only modified nodes and their ancestors are rebuilt: a node that was not + changed after `lens()` is returned as the original object. A node shared + by several parents is rebuilt once and stays shared. + Returns: Either an instantiated object or a new Lazy, depending on how this Lazy was created. @@ -522,6 +534,13 @@ def unlens(self) -> Any: new_lazy = l.unlens() # Returns Lazy[Outer] ``` """ + return self._unlens({}) + + def _unlens(self, memo: dict[int, Any]) -> Any: + """Implementation of `unlens()`, memoised by id so shared nodes stay shared.""" + if id(self) in memo: + return memo[id(self)] + from confingy.serde import HandlerRegistry handlers = HandlerRegistry.get_default_handlers() @@ -529,7 +548,7 @@ def unlens(self) -> Any: def unlens_value(value: Any) -> Any: """Recursively unlens a value, using handlers for containers.""" if is_lazy_instance(value): - return value.unlens() + return value._unlens(memo) # Use handlers for container types for handler in handlers: @@ -541,17 +560,30 @@ def unlens_value(value: Any) -> Any: # Process all config values realized_config = {k: unlens_value(v) for k, v in self._confingy_config.items()} - if self._confingy_was_instantiated: + # Reuse the object this Lazy was lensed from if nothing changed + source = self.__dict__.get("_confingy_source") + source_config = None + if is_lazy_instance(source): + source_config = source._confingy_config + elif is_tracked_instance(source): + source_config = source._tracked_info["init_args"] + + result: Any + if source_config is not None and _same_config(realized_config, source_config): + result = source + elif self._confingy_was_instantiated: # This Lazy was created from a tracked instance - instantiate - return self._confingy_actual_cls(**realized_config) + result = self._confingy_actual_cls(**realized_config) else: # This was originally a Lazy - return new Lazy # Always skip the post-config hook since unlens() is a structural # transformation, not a semantic creation. Hooks already ran on # setattr when values were modified. - return Lazy( + result = Lazy( self._confingy_cls, realized_config, _skip_post_config_hook=True ) + memo[id(self)] = result + return result def __call__(self, *args: Any, **kwargs: Any) -> "Lazy[T]": """Make Lazy callable to support the lazy(Class)(...) pattern. @@ -720,18 +752,31 @@ def __init__(self, value: int): l.middle.inner.value = 100 new_lazy = l.unlens() # Returns Lazy since original was Lazy ``` + + An object shared by several parents is lensed once, so editing it through one + parent is visible through the others, and it stays shared after `unlens()`. """ + if not (is_lazy_instance(obj) or is_tracked_instance(obj)): + raise TypeError( + f"lens() requires a Lazy or tracked instance, got {type(obj).__name__}" + ) + return _lens(obj, {}) + + +def _lens(obj: Any, memo: dict[int, Any]) -> Any: + """Implementation of `lens()`, memoised by id so shared nodes stay shared.""" + if id(obj) in memo: + return memo[id(obj)] + from confingy.serde import HandlerRegistry handlers = HandlerRegistry.get_default_handlers() def lens_value(value: Any) -> Any: """Recursively convert tracked instances to Lazy.""" - if is_tracked_instance(value): - return lens(value) - if is_lazy_instance(value): - # Recurse into Lazy config in case it contains tracked instances - return lens(value) + if is_tracked_instance(value) or is_lazy_instance(value): + # Recurse into Lazy config too, in case it contains tracked instances + return _lens(value, memo) # Use handlers for container types for handler in handlers: @@ -740,35 +785,44 @@ def lens_value(value: Any) -> Any: return value + result: Any if is_lazy_instance(obj): # Recurse into config to convert any nested tracked instances new_config = {k: lens_value(v) for k, v in obj._confingy_config.items()} # Only create new Lazy if config actually changed if any(new_config[k] is not obj._confingy_config[k] for k in new_config): - return Lazy( + result = Lazy( obj._confingy_cls, new_config, skip_validation=True, # Config comes from valid source _was_instantiated=getattr(obj, "_confingy_was_instantiated", False), _skip_post_config_hook=True, # lens() is just wrapping, don't run hooks ) - return obj - - if is_tracked_instance(obj): + result._confingy_source = obj + else: + result = obj + else: # Convert tracked instance to Lazy with _was_instantiated=True # Skip validation since the tracked instance was already valid config = {k: lens_value(v) for k, v in obj._tracked_info["init_args"].items()} - return Lazy( + result = Lazy( type(obj), config, skip_validation=True, _was_instantiated=True, _skip_post_config_hook=True, # lens() is just wrapping, don't run hooks ) + result._confingy_source = obj + + memo[id(obj)] = result + return result + - raise TypeError( - f"lens() requires a Lazy or tracked instance, got {type(obj).__name__}" +def _same_config(config: dict[str, Any], source_config: dict[str, Any]) -> bool: + """Check whether a realized config still matches the config it was lensed from.""" + return config.keys() == source_config.keys() and all( + same_structure(config[k], source_config[k]) for k in config ) @@ -923,10 +977,20 @@ def init_with_tracking(self: Any, *args: Any, **kwargs: Any) -> None: ) if _validate and validation_model is not None: + to_validate = init_kwargs + if type(self).__init__ is not init_with_tracking: + # An untracked subclass overrode __init__, so init_kwargs follow + # its signature and may include arguments cls doesn't declare. + # Only check the ones cls does. + to_validate = { + k: v + for k, v in init_kwargs.items() + if k in validation_model.model_fields + } try: # Validate but keep original objects instead of converting to dict validation_model( - **init_kwargs + **to_validate ) # Just validate, don't use the result # Store the original init_kwargs, not the dumped version stored_kwargs = init_kwargs @@ -1075,6 +1139,9 @@ def _add_tracking_to_instance(instance: Any) -> Any: "module": get_module_name(cls), "init_args": init_kwargs, "class_hash": hash_class(cls), + # init_args holds attributes, not constructor arguments, so this + # instance can't be rebuilt from them + "from_instance": True, } return instance diff --git a/src/confingy/tree.py b/src/confingy/tree.py new file mode 100644 index 0000000..1f86271 --- /dev/null +++ b/src/confingy/tree.py @@ -0,0 +1,367 @@ +""" +Read, match, and transform utilities for trees of tracked and lazy objects. + +A fingy is usually a tree: a root object whose constructor arguments contain +other tracked or [Lazy][confingy.tracking.Lazy] objects, possibly nested inside +lists, tuples, dicts, sets, or dataclasses. The functions in this module let you +inspect and rewrite such trees without touching confingy internals. + +Examples: + ```python + from confingy import track, replace_args, walk_fingy + + @track + class Scaler: + def __init__(self, factor: float = 1.0): + self.factor = factor + + @track + class Pipeline: + def __init__(self, steps: list): + self.steps = steps + + pipeline = Pipeline(steps=[Scaler(), Scaler(factor=2.0)]) + + for path, node in walk_fingy(pipeline): + print(repr(path), type(node).__name__) + # '' Pipeline + # 'steps[0]' Scaler + # 'steps[1]' Scaler + + new_pipeline, n_rebuilt = replace_args(pipeline, Scaler, factor=3.0) + assert n_rebuilt == 2 + assert [s.factor for s in new_pipeline.steps] == [3.0, 3.0] + assert pipeline.steps[0].factor == 1.0 # The input is never mutated + ``` +""" + +from collections.abc import Callable, Iterator +from typing import Any, TypeVar + +from confingy.tracking import Lazy, update +from confingy.utils.containers import container_children, rebuild_container +from confingy.utils.types import ( + TrackedInstance, + is_lazy_instance, + is_tracked_instance, +) + +T = TypeVar("T") + +_MISSING = object() + + +def get_init_args(obj: TrackedInstance | Lazy[Any]) -> dict[str, Any]: + """Return the constructor arguments recorded for a tracked or lazy object. + + The returned dict is a shallow copy, so mutating it does not affect `obj`. + To change arguments, use [update][confingy.tracking.update], + [map_fingy][confingy.tree.map_fingy], or [replace_args][confingy.tree.replace_args]. + + Args: + obj: A tracked instance or a [Lazy][confingy.tracking.Lazy] instance. + + Returns: + A new dict mapping constructor argument names to their values. + + Raises: + TypeError: If `obj` is neither tracked nor lazy. + + Examples: + ```python + @track + class Foo: + def __init__(self, bar: int, baz: str = "x"): + self.bar = bar + self.baz = baz + + get_init_args(Foo(bar=1)) # {'bar': 1, 'baz': 'x'} + get_init_args(Foo.lazy(bar=1)) # {'bar': 1, 'baz': 'x'} + ``` + """ + if is_lazy_instance(obj): + return obj.get_config() + if is_tracked_instance(obj): + return dict(obj._tracked_info["init_args"]) + raise TypeError( + f"get_init_args() requires a tracked or Lazy instance, got {type(obj).__name__}" + ) + + +def is_fingy_of(obj: Any, expected: type) -> bool: + """Check whether `obj` is a tracked or lazy instance of `expected` or a subclass. + + Unlike [is_lazy_version_of][confingy.utils.types.is_lazy_version_of], this + matches the subclasses created by `track(expected)` and works for both tracked + and lazy objects, so a single check finds a class however it was constructed. + + Always pass the *original* class. Each call to `track(SomeClass)` creates a new + subclass, so `is_fingy_of(obj, track(SomeClass))` never matches anything. + + Args: + obj: Any value. Values that are neither tracked nor lazy return `False`. + expected: The class to match against. + + Returns: + True if `obj` is a tracked instance of `expected` (or a subclass), or a + `Lazy` whose target class is `expected` (or a subclass). + + Examples: + ```python + class Foo: + def __init__(self, bar: int): + self.bar = bar + + is_fingy_of(track(Foo)(bar=1), Foo) # True + is_fingy_of(lazy(track(Foo))(bar=1), Foo) # True + is_fingy_of(lens(track(Foo)(bar=1)), Foo) # True + is_fingy_of(Foo(bar=1), Foo) # False: not tracked + ``` + """ + if is_lazy_instance(obj): + actual_cls = obj._confingy_actual_cls + return isinstance(actual_cls, type) and issubclass(actual_cls, expected) + if is_tracked_instance(obj): + return isinstance(obj, expected) + return False + + +def walk_fingy(obj: Any) -> Iterator[tuple[str, Any]]: + """Yield `(path, node)` for every tracked or lazy node in a tree. + + Nodes are yielded depth-first, parents before children. A node reachable + from several parents is yielded once, at the first path it is found on. + Children are found through recorded constructor arguments, and through + lists, tuples, dicts, sets, and dataclasses nested inside them. + + Paths are built from argument names, sequence indices (`[0]`), dict keys + (`['key']`), and dataclass fields (`.field`). Set members have no stable + position and are shown as `{}`. The root node has the path `""`. + + Args: + obj: The root of the tree. May be a tracked or lazy node, or a container + holding them. + + Yields: + `(path, node)` tuples. + + Examples: + ```python + pipeline = Pipeline(steps=[Scaler(), Scaler(factor=2.0)]) + [path for path, _ in walk_fingy(pipeline)] + # ['', 'steps[0]', 'steps[1]'] + ``` + """ + seen: set[int] = set() + + def visit(value: Any, path: str) -> Iterator[tuple[str, Any]]: + if _is_node(value): + if id(value) in seen: + return + seen.add(id(value)) + yield path, value + for name, child in get_init_args(value).items(): + yield from visit(child, _join(path, name)) + return + + children = container_children(value) + if children is None: + return + if id(value) in seen: + return + seen.add(id(value)) + for suffix, child in children: + yield from visit(child, path + suffix) + + yield from visit(obj, "") + + +def map_fingy(obj: T, fn: Callable[[Any], Any]) -> T: + """Return a copy of a tree with `fn` applied to every tracked or lazy node. + + Nodes are visited bottom-up. For each node, its children are mapped first; if + any child changed, the node is rebuilt with + [update][confingy.tracking.update] using the new children. Then `fn` is + called on the (possibly rebuilt) node, and its return value takes the node's + place in the parent. A replacement returned by `fn` is not traversed again. + + Guarantees: + + - The input is never mutated. + - Structural sharing: if `fn` returns every node unchanged, the result is `obj` + itself. Only the ancestors of changed nodes are rebuilt, and untouched + subtrees and containers keep their identity. + - Laziness is preserved: lazy nodes are rebuilt as `Lazy`, tracked nodes as + tracked instances. + - A node shared by several parents is transformed once, and the result is + shared in the output. + + Notes: + + - Children are found through recorded constructor arguments, not live + attributes. Attributes set after construction are not part of the config + and are not carried over when a node is rebuilt (the same contract as + [update][confingy.tracking.update]). + - Rebuilding re-runs the constructor of each rebuilt tracked node, and the + validation and `__post_config__` hook of each rebuilt lazy node, so any side + effects happen again. Only the changed path is rebuilt. + - Nodes created with `track(existing_instance)` record attributes rather than + constructor arguments, so they cannot be rebuilt. A `ValueError` is raised + if one lies on a changed path. + + Args: + obj: The root of the tree. May be a tracked or lazy node, or a container + holding them. + fn: Called with each node. Return the node itself to leave it unchanged, + or a replacement. + + Returns: + The transformed tree, or `obj` itself if nothing changed. + + Raises: + ValueError: If the tree contains a cycle, or a node created with + `track(existing_instance)` needs to be rebuilt. + + Examples: + ```python + pipeline = Pipeline(steps=[Scaler(factor=0.5), Scaler(factor=2.0)]) + + def clamp(node): + if is_fingy_of(node, Scaler) and get_init_args(node)["factor"] > 1.0: + return update(node)(factor=1.0) + return node + + clamped = map_fingy(pipeline, clamp) + assert [s.factor for s in clamped.steps] == [0.5, 1.0] + assert clamped.steps[0] is pipeline.steps[0] # Untouched node is reused + ``` + """ + return _map_fingy(obj, lambda node, path: fn(node)) + + +def replace_args(obj: T, target: type, **init_args: Any) -> tuple[T, int]: + """Set constructor arguments on every node in a tree that matches `target`. + + This is [map_fingy][confingy.tree.map_fingy] with a function that rebuilds each + node matching [is_fingy_of(node, target)][confingy.tree.is_fingy_of] using + `update(node)(**init_args)`. Nodes whose arguments already have the requested + values are left untouched and not counted, so repeating a call is a no-op. + + Args: + obj: The root of the tree. + target: The class to match. Subclasses match too. + **init_args: Constructor arguments to set on each matching node. + + Returns: + A `(new_obj, n_rebuilt)` tuple, where `n_rebuilt` is the number of matching + nodes that were rebuilt. + + Raises: + ValidationError: If an argument is unknown or has an invalid value for a + matching node's class. + ValueError: See [map_fingy][confingy.tree.map_fingy]. + + Examples: + ```python + pipeline = Pipeline(steps=[Scaler(), Scaler(factor=2.0)]) + new_pipeline, n = replace_args(pipeline, Scaler, factor=2.0) + assert n == 1 # The second Scaler already had factor=2.0 + ``` + """ + n_rebuilt = 0 + + def apply(node: Any, path: str) -> Any: + nonlocal n_rebuilt + if not is_fingy_of(node, target): + return node + current = get_init_args(node) + if all(_same_value(current.get(k, _MISSING), v) for k, v in init_args.items()): + return node + n_rebuilt += 1 + return _rebuild(node, init_args, path) + + return _map_fingy(obj, apply), n_rebuilt + + +def _map_fingy(obj: T, fn: Callable[[Any, str], Any]) -> T: + """Implementation of [map_fingy][confingy.tree.map_fingy] with node paths.""" + # Keys are ids of objects in the input tree, which stays alive for the whole + # call, so ids cannot be reused. + memo: dict[int, Any] = {} + in_progress: set[int] = set() + + def visit(value: Any, path: str) -> Any: + is_node = _is_node(value) + children = None if is_node else container_children(value) + if not is_node and children is None: + return value + + key = id(value) + if key in memo: + return memo[key] + if key in in_progress: + raise ValueError(f"Cycle detected at {_describe(path)}") + in_progress.add(key) + try: + if is_node: + args = get_init_args(value) + changes = {} + for name, child in args.items(): + new_child = visit(child, _join(path, name)) + if new_child is not child: + changes[name] = new_child + node = _rebuild(value, changes, path) if changes else value + result = fn(node, path) + else: + assert children is not None + new_children = [ + visit(child, path + suffix) for suffix, child in children + ] + if all(new is old for new, (_, old) in zip(new_children, children)): + result = value + else: + result = rebuild_container(value, new_children) + finally: + in_progress.discard(key) + memo[key] = result + return result + + return visit(obj, "") + + +def _is_node(value: Any) -> bool: + return is_lazy_instance(value) or is_tracked_instance(value) + + +def _rebuild(node: Any, changes: dict[str, Any], path: str) -> Any: + """Rebuild a tracked or lazy node with some constructor arguments replaced.""" + if is_lazy_instance(node): + if node._confingy_was_instantiated: + # Lazies created by lens() hold Lazy children where the class expects + # instances, so update()'s validation would reject them. + return node.copy(**changes) + return update(node)(**changes) + if node._tracked_info.get("from_instance"): + raise ValueError( + f"Cannot rebuild {type(node).__name__} at {_describe(path)}: it was " + "tracked with track(existing_instance), so its recorded arguments are " + "attributes, not constructor arguments." + ) + return update(node)(**changes) + + +def _same_value(a: Any, b: Any) -> bool: + if a is b: + return True + try: + return bool(a == b) + except Exception: # e.g. arrays, where == is elementwise + return False + + +def _join(path: str, name: str) -> str: + return f"{path}.{name}" if path else name + + +def _describe(path: str) -> str: + return f"path '{path}'" if path else "the root" diff --git a/src/confingy/utils/containers.py b/src/confingy/utils/containers.py new file mode 100644 index 0000000..486dafa --- /dev/null +++ b/src/confingy/utils/containers.py @@ -0,0 +1,102 @@ +"""Helpers for traversing the containers that can hold tracked or lazy objects.""" + +import copy +import dataclasses +from typing import Any + + +def container_children(value: Any) -> list[tuple[str, Any]] | None: + """Return `(path_suffix, child)` pairs for a supported container. + + Supported containers are lists, tuples (including namedtuples), dicts, sets, + frozensets, and dataclass instances. Path suffixes look like `[0]` for + sequences, `['key']` for dicts, `.field` for dataclasses, and `{}` for set + members, which have no stable position. + + Args: + value: Any value. + + Returns: + The container's children in iteration order, or None if `value` is not a + supported container. + """ + if isinstance(value, (list, tuple)): + return [(f"[{i}]", v) for i, v in enumerate(value)] + if isinstance(value, dict): + return [(f"[{k!r}]", v) for k, v in value.items()] + if isinstance(value, (set, frozenset)): + return [("{}", v) for v in value] + if dataclasses.is_dataclass(value) and not isinstance(value, type): + # init=False fields are derived, and dataclasses.replace() can't set them + return [ + (f".{f.name}", getattr(value, f.name)) + for f in dataclasses.fields(value) + if f.init + ] + return None + + +def rebuild_container(value: Any, new_children: list[Any]) -> Any: + """Return a copy of a container with its children replaced. + + The container type is preserved where possible, including list and dict + subclasses such as `OrderedDict` and `defaultdict`. + + Args: + value: A container supported by + [container_children][confingy.utils.containers.container_children]. + new_children: The new children, in the order returned by + `container_children(value)`. + + Returns: + A new container of the same type as `value`. + """ + if isinstance(value, list): + new_list = copy.copy(value) + new_list[:] = new_children + return new_list + if isinstance(value, tuple): + if hasattr(type(value), "_fields"): # namedtuple + return type(value)(*new_children) + return type(value)(new_children) + if isinstance(value, dict): + new_dict = copy.copy(value) + for key, child in zip(value, new_children): + new_dict[key] = child + return new_dict + if isinstance(value, (set, frozenset)): + return type(value)(new_children) + names = [f.name for f in dataclasses.fields(value) if f.init] + return dataclasses.replace(value, **dict(zip(names, new_children))) + + +def same_structure(a: Any, b: Any) -> bool: + """Check whether two values are identical, or containers of identical values. + + Containers are compared recursively by type, keys, and child identity. Leaf + values are compared by identity only. + + Args: + a: The first value. + b: The second value. + + Returns: + True if `a` and `b` are the same object, or containers of the same type + whose children are recursively the same. + """ + if a is b: + return True + if type(a) is not type(b): + return False + if isinstance(a, (set, frozenset)): + # Members have no order to pair them up by. Unchanged objects compare + # equal by identity or value, which is enough here. + return bool(a == b) + children_a = container_children(a) + children_b = container_children(b) + if children_a is None or children_b is None or len(children_a) != len(children_b): + return False + return all( + suffix_a == suffix_b and same_structure(child_a, child_b) + for (suffix_a, child_a), (suffix_b, child_b) in zip(children_a, children_b) + ) diff --git a/src/confingy/utils/types.py b/src/confingy/utils/types.py index c0d626c..1750912 100644 --- a/src/confingy/utils/types.py +++ b/src/confingy/utils/types.py @@ -7,7 +7,13 @@ class TrackedInstance(Protocol): - """Protocol for objects decorated with @track.""" + """Protocol for objects decorated with @track. + + `_tracked_info` is private. Use + [get_init_args][confingy.tree.get_init_args] to read the recorded constructor + arguments, and [update][confingy.tracking.update] or + [map_fingy][confingy.tree.map_fingy] to change them. + """ _tracked_info: dict[str, Any] @@ -40,7 +46,13 @@ def is_tracked_instance(obj: Any) -> TypeGuard[TrackedInstance]: def is_lazy_version_of(obj: Any, expected_type: type) -> bool: - """Check if an object is a [Lazy][confingy.tracking.Lazy] version of a given type.""" + """Check if an object is a [Lazy][confingy.tracking.Lazy] version of a given type. + + This compares classes by exact equality, so it does not match Lazies built + from `track(expected_type)`, which creates a new subclass on every call. Use + [is_fingy_of][confingy.tree.is_fingy_of] to match tracked and lazy objects of + a class or its subclasses. + """ return is_lazy_instance(obj) and obj._confingy_cls == expected_type diff --git a/tests/test_tracking.py b/tests/test_tracking.py index 036f96c..a48ebcb 100644 --- a/tests/test_tracking.py +++ b/tests/test_tracking.py @@ -6,7 +6,7 @@ import pytest -from confingy import lazy, serialize_fingy, track, update +from confingy import disable_validation, lazy, serialize_fingy, track, update from confingy.exceptions import ValidationError from confingy.tracking import Lazy from confingy.utils.hashing import hash_class @@ -3223,3 +3223,55 @@ def __init__(self, value: int): gc.collect() assert ref() is None + + +class TestUnknownArguments: + """Unknown constructor arguments are rejected when a config is created.""" + + @staticmethod + def _make_class(): + class Widget: + def __init__(self, size: int = 1): + self.size = size + + return Widget + + def test_lazy_creation_rejects_unknown_arguments(self): + Widget = self._make_class() + with pytest.raises(ValidationError, match="typo"): + track(Widget).lazy(typo=1) + with pytest.raises(ValidationError, match="typo"): + lazy(track(Widget))(typo=1) + with pytest.raises(ValidationError, match="typo"): + lazy(Widget, typo=1) + + def test_update_lazy_rejects_unknown_arguments(self): + Widget = self._make_class() + with pytest.raises(ValidationError, match="typo"): + update(lazy(track(Widget))())(typo=1) + + def test_tracked_creation_rejects_unknown_arguments(self): + Widget = self._make_class() + with pytest.raises(ValidationError, match="typo"): + track(Widget)(typo=1) + with pytest.raises(ValidationError, match="typo"): + track(Widget, typo=1) + with pytest.raises(ValidationError, match="typo"): + update(track(Widget)())(typo=1) + + def test_var_keyword_accepts_extra_arguments(self): + @track + class Flexible: + def __init__(self, size: int = 1, **options: object): + self.size = size + self.options = options + + assert Flexible.lazy(anything=2).instantiate().options == {"anything": 2} + assert update(Flexible.lazy())(other=3).get_config()["other"] == 3 + assert Flexible(anything=2).options == {"anything": 2} + + def test_disable_validation_still_skips_check(self): + Widget = self._make_class() + with disable_validation(): + lazy_widget = lazy(track(Widget))(typo=1) + assert lazy_widget.get_config()["typo"] == 1 diff --git a/tests/test_tree.py b/tests/test_tree.py new file mode 100644 index 0000000..bec1c4f --- /dev/null +++ b/tests/test_tree.py @@ -0,0 +1,456 @@ +"""Tests for the tree read, match, and transform utilities.""" + +import dataclasses +from collections import OrderedDict, namedtuple + +import pytest + +from confingy import ( + Lazy, + ValidationError, + get_init_args, + is_fingy_of, + lazy, + lens, + map_fingy, + replace_args, + serialize_fingy, + track, + update, + walk_fingy, +) + + +class Scaler: + def __init__(self, factor: float = 1.0, offset: float = 0.0): + self.factor = factor + self.offset = offset + + +class SpecialScaler(Scaler): + pass + + +class Pipeline: + def __init__(self, steps: list): + self.steps = steps + + +class Registry: + def __init__(self, entries: dict): + self.entries = entries + + +class Pair: + def __init__(self, left: object, right: object): + self.left = left + self.right = right + + +@dataclasses.dataclass +class Holder: + item: object + note: str = "hi" + + +def identity(node): + return node + + +def build_registry(): + return track(Registry)( + entries={ + "a": track(Pipeline)(steps=[track(Scaler)(), "passthrough"]), + "b": track(Pipeline)(steps=[track(Scaler)(factor=2.0)]), + } + ) + + +# get_init_args + + +def test_get_init_args_tracked_and_lazy_match(): + tracked = track(Scaler)(factor=2.0) + lazy_obj = lazy(track(Scaler))(factor=2.0) + assert get_init_args(tracked) == get_init_args(lazy_obj) + assert get_init_args(tracked)["factor"] == 2.0 + + +def test_get_init_args_returns_copy(): + for obj in (track(Scaler)(factor=2.0), lazy(track(Scaler))(factor=2.0)): + args = get_init_args(obj) + args["factor"] = 99.0 + assert get_init_args(obj)["factor"] == 2.0 + + +def test_get_init_args_rejects_plain_objects(): + with pytest.raises(TypeError, match="tracked or Lazy"): + get_init_args(Scaler()) + + +# is_fingy_of + + +def test_is_fingy_of_matches_all_construction_styles(): + @track + class Decorated: + def __init__(self, x: int = 1): + self.x = x + + tracked = track(Scaler)() + assert is_fingy_of(tracked, Scaler) + assert is_fingy_of(track(Scaler, factor=2.0), Scaler) + assert is_fingy_of(lazy(track(Scaler))(), Scaler) + assert is_fingy_of(lazy(Scaler)(), Scaler) + assert is_fingy_of(lens(tracked), Scaler) + assert is_fingy_of(Decorated(), Decorated) + assert is_fingy_of(Decorated.lazy(), Decorated) + assert is_fingy_of(lazy(Decorated, x=2), Decorated) + + +def test_is_fingy_of_matches_subclasses(): + assert is_fingy_of(track(SpecialScaler)(), Scaler) + assert is_fingy_of(lazy(track(SpecialScaler))(), Scaler) + assert not is_fingy_of(track(Scaler)(), SpecialScaler) + + +def test_is_fingy_of_rejects_other_values(): + assert not is_fingy_of(track(Scaler)(), Pipeline) + assert not is_fingy_of(lazy(track(Scaler))(), Pipeline) + assert not is_fingy_of(Scaler(), Scaler) # Not tracked + assert not is_fingy_of("text", str) + assert not is_fingy_of(None, Scaler) + + +# walk_fingy + + +def test_walk_fingy_paths(): + registry = build_registry() + walked = [(path, type(node).__name__) for path, node in walk_fingy(registry)] + assert walked == [ + ("", "Registry"), + ("entries['a']", "Pipeline"), + ("entries['a'].steps[0]", "Scaler"), + ("entries['b']", "Pipeline"), + ("entries['b'].steps[0]", "Scaler"), + ] + + +def test_walk_fingy_yields_shared_nodes_once(): + shared = track(Scaler)() + pair = track(Pair)(left=shared, right=[shared]) + paths = [path for path, node in walk_fingy(pair) if node is shared] + assert paths == ["left"] + + +def test_walk_fingy_containers_and_lazy(): + tree = track(Pair)( + left=Holder(item=lazy(track(Scaler))()), + right=(track(Scaler)(), {track(Scaler)(factor=3.0)}), + ) + paths = [path for path, _ in walk_fingy(tree)] + assert paths == ["", "left.item", "right[0]", "right[1]{}"] + + +def test_walk_fingy_root_container(): + nodes = [track(Scaler)(), lazy(track(Scaler))()] + assert [path for path, _ in walk_fingy(nodes)] == ["[0]", "[1]"] + + +# map_fingy + + +def test_map_fingy_identity_returns_same_root(): + registry = build_registry() + assert map_fingy(registry, identity) is registry + containers = [registry, [registry], {"k": (registry,)}] + for container in containers: + assert map_fingy(container, identity) is container + + +def test_map_fingy_rebuilds_only_ancestors(): + registry = build_registry() + target = registry.entries["a"].steps[0] + + def change_target(node): + return update(node)(factor=5.0) if node is target else node + + new = map_fingy(registry, change_target) + assert new is not registry + assert new.entries["a"] is not registry.entries["a"] + assert new.entries["a"].steps[0].factor == 5.0 + assert new.entries["a"].steps[1] == "passthrough" + # Sibling subtree keeps its identity + assert new.entries["b"] is registry.entries["b"] + + +def test_map_fingy_does_not_mutate_input(): + registry = build_registry() + before = serialize_fingy(registry) + replace_args(registry, Scaler, factor=7.0) + assert serialize_fingy(registry) == before + assert registry.entries["a"].steps[0].factor == 1.0 + + +def test_map_fingy_rebuilt_node_attributes_match_config(): + registry = build_registry() + new, _ = replace_args(registry, Scaler, factor=7.0) + step = new.entries["a"].steps[0] + assert step.factor == 7.0 + assert get_init_args(step)["factor"] == 7.0 + + +def test_map_fingy_shared_node_transformed_once_and_stays_shared(): + shared = track(Scaler)() + pair = track(Pair)(left=shared, right=[shared]) + calls = [] + + def bump(node): + if is_fingy_of(node, Scaler): + calls.append(node) + return update(node)(factor=2.0) + return node + + new = map_fingy(pair, bump) + assert len(calls) == 1 + assert new.left is new.right[0] + assert new.left.factor == 2.0 + + +def test_map_fingy_preserves_laziness_in_mixed_trees(): + tree = track(Pair)( + left=lazy(track(Pipeline))(steps=[track(Scaler)()]), + right=track(Pipeline)(steps=[lazy(track(Scaler))()]), + ) + new, n = replace_args(tree, Scaler, factor=3.0) + assert n == 2 + assert isinstance(new.left, Lazy) + assert not isinstance(new.left.steps[0], Lazy) + assert new.left.steps[0].factor == 3.0 + assert not isinstance(new.right, Lazy) + assert isinstance(new.right.steps[0], Lazy) + assert new.right.steps[0].factor == 3.0 + # The lazy parent still instantiates correctly + assert new.left.instantiate().steps[0].factor == 3.0 + + +def test_map_fingy_reaches_all_container_types(): + Point = namedtuple("Point", ["x", "y"]) + tree = track(Pair)( + left=[ + (track(Scaler)(),), + {"k": track(Scaler)()}, + Holder(item=track(Scaler)()), + Point(x=track(Scaler)(), y=0), + ], + right=OrderedDict(z=track(Scaler)()), + ) + new, n = replace_args(tree, Scaler, factor=4.0) + assert n == 5 + assert new.left[0][0].factor == 4.0 + assert new.left[1]["k"].factor == 4.0 + assert new.left[2].item.factor == 4.0 + assert new.left[2].note == "hi" + assert isinstance(new.left[3], Point) + assert new.left[3].x.factor == 4.0 + assert isinstance(new.right, OrderedDict) + assert new.right["z"].factor == 4.0 + + +def test_map_fingy_reaches_sets(): + tree = track(Pair)(left={track(Scaler)()}, right=frozenset()) + new, n = replace_args(tree, Scaler, factor=4.0) + assert n == 1 + assert [s.factor for s in new.left] == [4.0] + assert new.right is tree.right + + +def test_map_fingy_untouched_containers_keep_identity(): + untouched = [track(Scaler)(factor=2.0), "x"] + tree = track(Pair)(left=untouched, right=[track(Scaler)()]) + new, _ = replace_args(tree, Scaler, factor=2.0) + assert new.left is untouched + assert new.right is not tree.right + + +def test_map_fingy_serializes_like_hand_built_tree(): + registry = build_registry() + new, _ = replace_args(registry, Scaler, offset=0.5) + expected = track(Registry)( + entries={ + "a": track(Pipeline)(steps=[track(Scaler)(offset=0.5), "passthrough"]), + "b": track(Pipeline)(steps=[track(Scaler)(factor=2.0, offset=0.5)]), + } + ) + assert serialize_fingy(new) == serialize_fingy(expected) + + +def test_map_fingy_fn_sees_rebuilt_children(): + tree = track(Pipeline)(steps=[track(Scaler)()]) + seen = {} + + def record(node): + if is_fingy_of(node, Scaler): + return update(node)(factor=9.0) + if is_fingy_of(node, Pipeline): + seen["factor"] = get_init_args(node)["steps"][0].factor + return node + + map_fingy(tree, record) + assert seen["factor"] == 9.0 + + +def test_map_fingy_replacement_is_not_traversed(): + tree = track(Pipeline)(steps=[track(Scaler)()]) + replacement = track(Pipeline)(steps=[track(Scaler)(factor=5.0)]) + visited = [] + + def swap(node): + visited.append(type(node).__name__) + return replacement if is_fingy_of(node, Pipeline) else node + + assert map_fingy(tree, swap) is replacement + assert visited == ["Scaler", "Pipeline"] + + +def test_map_fingy_detects_cycles(): + root = lazy(track(Pair))(left=None, right=None) + root.left = [root] + with pytest.raises(ValueError, match="Cycle detected"): + map_fingy(root, identity) + + +def test_map_fingy_rejects_rebuilding_instance_tracked_nodes(): + class Legacy: + def __init__(self, child: object): + self.child = child + self.label = "derived" + + legacy = track(Legacy(child=track(Scaler)())) + assert get_init_args(legacy) == {"child": legacy.child, "label": "derived"} + + # Fine when the node is not on a changed path + tree = track(Pair)(left=legacy, right=track(SpecialScaler)()) + new, _ = replace_args(tree, SpecialScaler, factor=2.0) + assert new.left is legacy + + # Rebuilding it would pass attributes as constructor arguments + tree = track(Pair)(left=[legacy], right=1) + with pytest.raises( + ValueError, match=r"Legacy at path 'left\[0\]'.*track\(existing_instance\)" + ): + replace_args(tree, Scaler, factor=2.0) + + with pytest.raises(ValueError, match="the root"): + replace_args(legacy, Legacy, child=None) + + +def test_map_fingy_on_lensed_tree(): + tree = track(Pipeline)(steps=[track(Scaler)()]) + lensed = lens(tree) + new, n = replace_args(lensed, Scaler, factor=6.0) + assert n == 1 + rebuilt = new.unlens() + assert rebuilt.steps[0].factor == 6.0 + + +def test_map_fingy_reruns_post_config_hook_on_rebuilt_lazy(): + calls = [] + + @track + class Hooked: + def __init__(self, child: object = None): + self.child = child + + @staticmethod + def __post_config__(lazy_obj, changed_key): + calls.append(changed_key) + return lazy_obj + + tree = Hooked.lazy(child=lazy(track(Scaler))()) + calls.clear() + replace_args(tree, Scaler, factor=2.0) + assert calls == [None] + + +# replace_args + + +def test_replace_args_counts_and_is_idempotent(): + registry = build_registry() + new, n = replace_args(registry, Scaler, factor=2.0) + assert n == 1 # The second Scaler already has factor=2.0 + again, n_again = replace_args(new, Scaler, factor=2.0) + assert n_again == 0 + assert again is new + + +def test_replace_args_nested_targets(): + tree = track(Pair)(left=track(Pair)(left=1, right=2), right=3) + new, n = replace_args(tree, Pair, right=0) + assert n == 2 + assert new.right == 0 + assert new.left.right == 0 + + +def test_replace_args_invalid_value_raises_validation_error(): + registry = build_registry() + with pytest.raises(ValidationError): + replace_args(registry, Scaler, factor="not a float") + + +def test_replace_args_unknown_arg_raises_validation_error(): + for tree in ( + build_registry(), + lazy(track(Pipeline))(steps=[lazy(track(Scaler))()]), + ): + with pytest.raises(ValidationError, match="not_an_arg"): + replace_args(tree, Scaler, not_an_arg=1) + + +# lens / unlens sharing + + +def test_unlens_preserves_sharing(): + shared = track(Scaler)() + lensed = lens(track(Pair)(left=shared, right=shared)) + assert lensed.left is lensed.right + lensed.left.factor = 3.0 + rebuilt = lensed.unlens() + assert rebuilt.left is rebuilt.right + assert rebuilt.right.factor == 3.0 + + +def test_unlens_reuses_unmodified_nodes(): + original = track(Pair)(left=track(Scaler)(), right=[track(Scaler)()]) + assert lens(original).unlens() is original + + lensed = lens(original) + lensed.left.factor = 2.0 + rebuilt = lensed.unlens() + assert rebuilt is not original + assert rebuilt.left.factor == 2.0 + assert rebuilt.right[0] is original.right[0] + + +def test_unlens_detects_container_edits(): + original = track(Pipeline)(steps=[track(Scaler)()]) + lensed = lens(original) + lensed.steps.append(track(Scaler)(factor=2.0)) + rebuilt = lensed.unlens() + assert rebuilt is not original + assert len(rebuilt.steps) == 2 + assert len(original.steps) == 1 + + +def test_unlens_lazy_root_with_tracked_children(): + original = lazy(track(Pair))(left=track(Scaler)(), right=None) + assert lens(original).unlens() is original + + lensed = lens(original) + lensed.left.factor = 2.0 + rebuilt = lensed.unlens() + assert isinstance(rebuilt, Lazy) + assert rebuilt.left.factor == 2.0 + assert original.left.factor == 1.0