diff --git a/.github/workflows/pubpyruntime.yml b/.github/workflows/pubpyruntime.yml new file mode 100644 index 00000000..542fc028 --- /dev/null +++ b/.github/workflows/pubpyruntime.yml @@ -0,0 +1,89 @@ +# Publish the vodml-runtime Python package to PyPI using trusted publishing (OIDC). +# +# Prerequisites (one-time setup on PyPI): +# 1. Create the project "vodml-runtime" on https://pypi.org +# 2. Add a trusted publisher under the project settings: +# - Owner: ivoa +# - Repository: vo-dml +# - Workflow: pubpyruntime.yml +# - Environment: pypi +# +# Triggering: +# Push a tag matching "pyruntime*" (e.g. pyruntime-0.1.0). + +name: Publish Python Runtime to PyPI + +on: + push: + tags: + - 'pyruntime*' + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + if: github.repository == 'ivoa/vo-dml' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tools + run: python -m pip install --upgrade pip build + + - name: Build sdist and wheel + working-directory: runtime/python + run: python -m build + + - name: Upload distribution artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: runtime/python/dist/ + + publish-pypi: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/vodml-runtime + permissions: + id-token: write # required for trusted publishing via OIDC + + steps: + - name: Download distribution artifacts + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-testpypi: + name: Publish to TestPyPI + needs: build + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/vodml-runtime + permissions: + id-token: write + + steps: + - name: Download distribution artifacts + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + diff --git a/.github/workflows/pydantic-test.yml b/.github/workflows/pydantic-test.yml new file mode 100644 index 00000000..8ec64710 --- /dev/null +++ b/.github/workflows/pydantic-test.yml @@ -0,0 +1,63 @@ +# This workflow builds the VO-DML tooling, generates pydantic-xml model classes from +# the sample VO-DML models, and runs the Python interoperability test suite +# (PydanticInteropTest.py). +# +# Build order: +# 1. Publish the shared Java runtime library to Maven Local. +# 2. Publish the IVOA base model to Maven Local (sample depends on it). +# 3. Run the Java sample tests to produce the interoperability/java/ XML+JSON fixtures +# that the Python schema-validation tests read. +# 4. Generate the IVOA pydantic package (org.ivoa.dm.ivoa) into the shared generated dir. +# 5. Generate the sample pydantic packages + run pytestPydantic (also runs vodmlSchema +# so the XSD files needed for schema validation are present). + +name: Python pydantic model tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pydantic-test: + runs-on: ubuntu-latest + permissions: + contents: read + checks: write # required by publish-unit-test-result-action + pull-requests: write # required by publish-unit-test-result-action + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + - name: Set up Python 3 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Build runtime library + run: ./gradlew :java:publishToMavenLocal -PskipSigning=True + + - name: Build and publish IVOA base model + run: ./gradlew :ivoa:publishToMavenLocal -PskipSigning=True + + - name: Run Java sample tests (generates interoperability/java/ fixtures) + run: ./gradlew :sample:test -PskipSigning=True + + - name: Generate ivoa pydantic package and run pydantic interop tests + run: ./gradlew :ivoa:vodmlPydanticGenerate :sample:pytestPydantic -PskipSigning=True + + - name: Publish pydantic test results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: tools/gradletooling/sample/build/reports/pytestPydantic/results.xml diff --git a/.gitignore b/.gitignore index 8d3c303c..eb84ce35 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ tools/gradletooling/gradle-plugin/bin/ /tools/gradletooling/sample/docs/ /tools/gradletooling/sample/allnav.yml /tools/gradletooling/sample/pythontest/generated/ +**/__pycache__/ +**/*.pyc /tools/gradletooling/sample/mkdocs.yml /runtime/java/bin/ /tools/gradletooling/sample/tmp/ @@ -21,6 +23,6 @@ tools/gradletooling/gradle-plugin/bin/ /tools/pythontooling/.venv/ /tools/pythontooling/src/vodmltools/schxslt/ /tools/pythontooling/dist/ -__pycache__/ -*.pyc +/runtime/python/dist/ +/runtime/python/*.egg-info/ .pytest_cache/ diff --git a/AGENTS.md b/AGENTS.md index 1b023e51..7c40bda0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ - Plugin tests: `./gradlew :gradle-plugin:test` (passes in this workspace). - Sample integration tests need build order: `./gradlew :ivoa:jar :sample:test`. - Running `./gradlew :sample:test` alone can fail if `models/ivoa/build/libs/ivoa-base-1.0-SNAPSHOT.jar` is missing. +- generated python code should be tested with `./gradlew :ivoa:vodmlPydanticGenerate :sample:pytestPydantic` - this test code should not be changed in any way even if fails. - Discover task surface quickly: `./gradlew :sample:tasks --all`. ## Project-specific conventions @@ -37,6 +38,7 @@ - Python tests should be run with the via gradle tasks that activate the venv and set the PYTHONPATH to include the generated code; see `tools/gradletooling/sample/build.gradle.kts` for examples of how to do this. + ## Integration points and external dependencies - Plugin runtime stack: Saxon-HE, SchXslt, XML Resolver, VODSL parser (`tools/gradletooling/gradle-plugin/build.gradle.kts`). - Generated Java models depend on `org.javastro.ivoa.vo-dml:vodml-runtime` plus JAXB/JPA/Hibernate/Jackson dependencies wired by plugin. diff --git a/models/ivoa/build.gradle.kts b/models/ivoa/build.gradle.kts index 18bc7454..c702ed46 100644 --- a/models/ivoa/build.gradle.kts +++ b/models/ivoa/build.gradle.kts @@ -14,7 +14,7 @@ version = "1.0-SNAPSHOT" vodml { vodmlDir.set(file("vo-dml")) bindingFiles.setFrom(file("vo-dml/ivoa_base.vodml-binding.xml")) - outputPythonDir.set(layout.projectDirectory.dir("../../tools/gradletooling/sample/pythontest/generated")) + outputPythonDir.set(layout.projectDirectory.dir("../../tools/gradletooling/sample/pythontest/generated")) // FIXME when this is eventually packaged it should be local to this project, but for now this is just to allow testing of the generated code in the python test project } diff --git a/models/ivoa/vo-dml/ivoa_base.vodml-binding.xml b/models/ivoa/vo-dml/ivoa_base.vodml-binding.xml index 177e0265..5a03cf90 100644 --- a/models/ivoa/vo-dml/ivoa_base.vodml-binding.xml +++ b/models/ivoa/vo-dml/ivoa_base.vodml-binding.xml @@ -46,6 +46,7 @@ anyURI String + str xsd:anyURI string diff --git a/models/sample/test/serializationExample.vo-dml.xml b/models/sample/test/serializationExample.vo-dml.xml index 16db494e..d69e78af 100644 --- a/models/sample/test/serializationExample.vo-dml.xml +++ b/models/sample/test/serializationExample.vo-dml.xml @@ -8,7 +8,7 @@ 1.0 - 2026-04-01T12:30:22Z + 2026-04-10T16:35:10Z ivoa 1.0 @@ -132,6 +132,18 @@ -1 + + SomeContent.uri + uri + + + ivoa:anyURI + + + 1 + 1 + + types diff --git a/models/sample/test/serializationExample.vodsl b/models/sample/test/serializationExample.vodsl index d378e98a..6432887a 100644 --- a/models/sample/test/serializationExample.vodsl +++ b/models/sample/test/serializationExample.vodsl @@ -24,6 +24,7 @@ package types "" { abstract otype BaseC { !xmlmeta isAttribute="true"! bname: ivoa:string ""; + } otype Dcont -> BaseC { @@ -38,7 +39,9 @@ otype SomeContent "" { ref1 references Refa ""; ref2 references Refb ""; zval : ivoa:string @+ ""; - con: types:BaseC @+ as composition ""; + con: types:BaseC @+ as composition ""; + uri : ivoa:anyURI ""; + } primitive ivoid -> ivoa:anyURI "a specialization for IVOIDs" diff --git a/runtime/java/src/main/java/org/ivoa/vodml/nav/ModelInstanceTraverser.java b/runtime/java/src/main/java/org/ivoa/vodml/nav/ModelInstanceTraverser.java index 15108908..8ea14948 100644 --- a/runtime/java/src/main/java/org/ivoa/vodml/nav/ModelInstanceTraverser.java +++ b/runtime/java/src/main/java/org/ivoa/vodml/nav/ModelInstanceTraverser.java @@ -269,20 +269,20 @@ private ObjInfo(ObjBase inob) { else { alreadyVisited = false; - if(ob.o.getClass().isArray()) { //FIXME think about arrays of primitives.... not yet in our models... - children = Arrays.stream((Object[]) ob.o).map(ao->{return ao != null ? new ObjBase(ao): null;}).iterator(); + if(ob.o.getClass().isArray()) { + children = Arrays.stream((Object[]) ob.o).map(ao->{return ao != null ? new ObjBase(ao,ob.vodmlt): null;}).iterator(); } else if(ob.o instanceof Collection) { Collection col = (Collection) ob.o; List vals = new ArrayList<>(col.size()); - col.forEach(co -> {if (co != null)vals.add(new ObjBase(co));}); + col.forEach(co -> {if (co != null)vals.add(new ObjBase(co,ob.vodmlt));}); children = vals.iterator(); } else if (ob.o instanceof Map) { ///IMPL there are actually no maps created by current model generation.... Map m = ((Map) ob.o); List vals = new ArrayList<>(m.size()*2); - m.forEach((t, u) -> {vals.add(new ObjBase(t));vals.add(new ObjBase(u));}); + m.forEach((t, u) -> {vals.add(new ObjBase(t));vals.add(new ObjBase(u,ob.vodmlt));}); children = vals.iterator(); } else if (ob.vodmlt.role != VodmlRole.primitiveType) diff --git a/runtime/python/README.md b/runtime/python/README.md new file mode 100644 index 00000000..bb557691 --- /dev/null +++ b/runtime/python/README.md @@ -0,0 +1,51 @@ +# vodml-runtime + +Python runtime support library for code generated from [VO-DML](https://www.ivoa.net/documents/VODML/index.html) models. + +This is the Python counterpart of the Java +[vodml-runtime](https://search.maven.org/artifact/org.javastro.ivoa.vo-dml/vodml-runtime/) +library. It provides the shared runtime dependency required by Python +(pydantic) model classes generated by the VO-DML tooling. + +## Installation + +```bash +pip install vodml-runtime +``` + +If you need the optional SQLAlchemy registry support: + +```bash +pip install vodml-runtime[sqlalchemy] +``` + +## Use + +In general, you should not need to import anything from this library directly. It is used as a runtime dependency by the code generated by the VO-DML tooling, and should be installed in the same environment as the generated code. + +## How to publish + +To release: Push a tag like pyruntime-0.1.0: +```shell +git tag pyruntime-0.1.0 +git push origin pyruntime-0.1.0 +``` + +Local build (for testing): +```shell + +cd runtime/python && python -m build +``` + +## Links + +* [VO-DML tooling guide](https://ivoa.github.io/vo-dml/) +* [GitHub repository](https://github.com/ivoa/vo-dml) +* [Java runtime library](https://search.maven.org/artifact/org.javastro.ivoa.vo-dml/vodml-runtime/) + +## License + +This project is licensed under the +[Creative Commons Attribution-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-sa/4.0/) +license — see the [LICENSE](../../LICENSE) file for details. + diff --git a/runtime/python/pyproject.toml b/runtime/python/pyproject.toml new file mode 100644 index 00000000..450711ac --- /dev/null +++ b/runtime/python/pyproject.toml @@ -0,0 +1,45 @@ +[project] +name = "vodml-runtime" +version = "0.1.0" +description = "Python runtime support library for code generated from VO-DML models" +readme = "README.md" +license = "CC-BY-SA-4.0" +requires-python = ">=3.10" +authors = [ + { name = "Paul Harrison", email = "paul.harrison@manchester.ac.uk" }, +] +keywords = ["ivoa", "vo-dml", "astronomy", "data-model"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Astronomy", + "Operating System :: OS Independent", +] +dependencies = [ + "pydantic>=2.0", + "xsdata-pydantic>=24.0", +] + +[project.optional-dependencies] +sqlalchemy = [ + "sqlalchemy>=2.0", +] + +[project.urls] +Homepage = "https://github.com/ivoa/vo-dml" +Documentation = "https://ivoa.github.io/vo-dml/" +Repository = "https://github.com/ivoa/vo-dml" +Issues = "https://github.com/ivoa/vo-dml/issues" + +[build-system] +requires = ["setuptools>=77.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["vodml_runtime*"] + diff --git a/runtime/python/vodml_runtime/VodmlXmlBase.py b/runtime/python/vodml_runtime/VodmlXmlBase.py new file mode 100644 index 00000000..ba7c1cb7 --- /dev/null +++ b/runtime/python/vodml_runtime/VodmlXmlBase.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026. Paul Harrison, University of Manchester +from pydantic import BaseModel, ConfigDict, field_serializer +from xsdata.formats.dataclass.context import XmlContext +from xsdata.formats.dataclass.parsers.config import ParserConfig +from xsdata.formats.dataclass.serializers import XmlSerializer + +from vodml_runtime.xsdata.parsers.json import JsonParser +from vodml_runtime.xsdata.serializers import JsonSerializer +from xsdata.formats.dataclass.serializers.config import SerializerConfig +from xsdata.formats.dataclass.parsers import XmlParser + +class _VodmlXmlBase(BaseModel): + """Base class providing Pydantic BaseModel with xsdata XML serialisation.""" + model_config = ConfigDict(arbitrary_types_allowed=True) + + + def to_xml(self, nsmap: dict[str, str] | None = None, pretty_print: bool = False) -> bytes: + config = SerializerConfig(indent=" " if pretty_print else None) + ctx = XmlContext(class_type="pydantic") + return XmlSerializer(config=config, context=ctx).render(self, ns_map=nsmap).encode("utf-8") + + @classmethod + def from_xml(cls, xml_bytes: bytes): + ctx = XmlContext(class_type="pydantic") + data = xml_bytes.decode("utf-8") if isinstance(xml_bytes, bytes) else xml_bytes + return XmlParser(context=ctx).from_string(data, cls) + + + def to_xsjson(self,pretty_print: bool = False) -> bytes: # TODO should try to override the pydantic BaseModel to_json method. + config = SerializerConfig(indent=" " if pretty_print else None) + ctx = XmlContext(class_type="pydantic") + return JsonSerializer(config=config, context=ctx).render(self).encode("utf-8") + + @classmethod + def from_xsjson(cls, json_bytes: bytes): + ctx = XmlContext(class_type="pydantic") + return JsonParser(config=ParserConfig(), context=ctx).from_bytes(json_bytes,cls) diff --git a/runtime/python/vodml_runtime/__init__.py b/runtime/python/vodml_runtime/__init__.py index e69de29b..89175519 100644 --- a/runtime/python/vodml_runtime/__init__.py +++ b/runtime/python/vodml_runtime/__init__.py @@ -0,0 +1,9 @@ +"""VO-DML Python runtime support library.""" + +from vodml_runtime.references import resolve_references, VodmlIdRegistry +from vodml_runtime.VodmlXmlBase import _VodmlXmlBase + +__all__ = ["resolve_references", "VodmlIdRegistry", "_VodmlXmlBase" ] + +__version__ = "0.1.0" + diff --git a/runtime/python/vodml_runtime/py.typed b/runtime/python/vodml_runtime/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/runtime/python/vodml_runtime/references.py b/runtime/python/vodml_runtime/references.py new file mode 100644 index 00000000..04d81161 --- /dev/null +++ b/runtime/python/vodml_runtime/references.py @@ -0,0 +1,259 @@ +""" +XML ID/IDREF resolution for VO-DML pydantic models. + +TODO this should be mainly replaced by the xsdata https://github.com/tefra/xsdata/pull/1222 + +This module provides the Python equivalent of the Java JAXB ``@XmlID`` / +``@XmlIDREF`` mechanism. After xsdata deserialises an XML document into +pydantic model instances, ``resolve_references`` walks the object tree, +builds a registry of all objects that carry an identifier, and replaces +every string-valued IDREF field with the actual referenced object. + +The resolution uses explicit metadata emitted by the XSLT code generator +rather than runtime type introspection heuristics: + +1. Referenceable objects declare ``_vodml_id_field`` (a ``ClassVar[str]``) + naming the field that carries their identity — either a natural-key + attribute (e.g. ``"name"``) or the surrogate ``id`` field mapped to the + ``_id`` XML attribute. + +2. Reference fields are listed in ``_vodml_refs`` (a ``ClassVar[list[str]]``) + on each class that contains VO-DML ```` elements. This is + emitted on both ``objectType`` and ``dataType`` classes. + +Typical usage (called automatically by the generated ``from_xml`` override on +model-wrapper classes):: + + model = MyModelModel.from_xml(xml_bytes) + # model.someContent[0].ref1 is now a Refa instance, not a string + +Note that this file was largely AI generated - it might be over complex... +""" + +# Copyright (c) 2026. Paul Harrison, University of Manchester + +from __future__ import annotations + +import logging +from typing import Any + +from pydantic import BaseModel + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# ID registry +# --------------------------------------------------------------------------- + +class VodmlIdRegistry: + """Maps XML ID strings to deserialised model objects.""" + + def __init__(self) -> None: + self._by_id: dict[str, Any] = {} + + def register(self, id_value: str, obj: Any) -> None: + """Register *obj* under the given *id_value*.""" + if id_value is not None: + self._by_id[id_value] = obj + + def resolve(self, id_value: str) -> Any | None: + """Return the object registered for *id_value*, or ``None``.""" + return self._by_id.get(id_value) + + def __len__(self) -> int: + return len(self._by_id) + + def __repr__(self) -> str: + return f"VodmlIdRegistry({len(self._by_id)} entries)" + + +# --------------------------------------------------------------------------- +# Metadata helpers +# --------------------------------------------------------------------------- + +def _find_id_field(cls: type) -> str | None: + """Walk the MRO to find ``_vodml_id_field``. + + ``_vodml_id_field`` is a ``ClassVar[str]`` emitted by the XSLT code + generator on every referenceable type. It names the field that carries + the object's XML ID — either a natural-key attribute or the surrogate + ``id`` field. + + Because Python attribute lookup follows the MRO, a subclass that does + not itself declare ``_vodml_id_field`` will inherit it from its parent + (e.g. ``ReferredTo1(Refbase)`` inherits ``Refbase._vodml_id_field``). + """ + for klass in cls.__mro__: + field = klass.__dict__.get('_vodml_id_field') + if field is not None: + return field + return None + + +def _collect_vodml_refs(cls: type) -> set[str]: + """Collect all ``_vodml_refs`` entries from the class's MRO. + + Each class in the hierarchy may declare its own ``_vodml_refs`` listing + the reference field names defined at *that* level. This helper merges + them all so that a subclass inherits reference metadata from its parents. + """ + refs: set[str] = set() + for klass in cls.__mro__: + r = klass.__dict__.get('_vodml_refs') + if r is not None: + refs.update(r) + return refs + + +def _get_object_id(obj: Any) -> str | None: + """Return the XML ID string for a model object, or ``None``. + + Uses the ``_vodml_id_field`` class-level metadata emitted by the + XSLT code generator to determine which field carries the object's + identity. + """ + id_field = _find_id_field(obj.__class__) + if id_field is not None: + val = getattr(obj, id_field, None) + if val is not None: + return str(val) + return None + + +# --------------------------------------------------------------------------- +# Registry builder +# --------------------------------------------------------------------------- + +def _build_registry_from_refs(refs_obj: Any, registry: VodmlIdRegistry) -> None: + """Walk every list-valued field in a *Refs* container and register items.""" + if refs_obj is None: + return + for field_name in refs_obj.__class__.model_fields: + value = getattr(refs_obj, field_name, None) + if value is None: + continue + if isinstance(value, list): + for item in value: + obj_id = _get_object_id(item) + if obj_id is not None: + registry.register(obj_id, item) + elif isinstance(value, BaseModel): + obj_id = _get_object_id(value) + if obj_id is not None: + registry.register(obj_id, value) + + +# --------------------------------------------------------------------------- +# Recursive reference resolver +# --------------------------------------------------------------------------- + +def _resolve_fields(obj: Any, registry: VodmlIdRegistry) -> Any: + """Recursively resolve string IDREFs inside *obj* to actual objects. + + Uses the ``_vodml_refs`` class metadata to identify which fields are + VO-DML references. For each such field, if the current value is a + ``str``, it is looked up in the *registry* and replaced with the + resolved object. Non-reference fields that contain nested + ``BaseModel`` instances are recursed into. + + Returns the (possibly mutated) *obj*. + """ + if not isinstance(obj, BaseModel): + return obj + + ref_fields = _collect_vodml_refs(obj.__class__) + updates: dict[str, Any] = {} + + for field_name in obj.__class__.model_fields: + value = getattr(obj, field_name, None) + if value is None: + continue + + if field_name in ref_fields: + # --- reference field --- + if isinstance(value, str): + resolved = registry.resolve(value) + if resolved is not None: + updates[field_name] = resolved + elif isinstance(value, list): + new_list = [] + changed = False + for item in value: + if isinstance(item, str): + resolved = registry.resolve(item) + if resolved is not None: + new_list.append(resolved) + changed = True + else: + new_list.append(item) + else: + if isinstance(item, BaseModel): + _resolve_fields(item, registry) + new_list.append(item) + if changed: + updates[field_name] = new_list + elif isinstance(value, BaseModel): + # Value is already an object (e.g. embedded inline rather + # than via IDREF) — recurse in case it has nested refs. + _resolve_fields(value, registry) + else: + # --- non-reference field: recurse into nested models --- + if isinstance(value, BaseModel): + _resolve_fields(value, registry) + elif isinstance(value, list): + for item in value: + if isinstance(item, BaseModel): + _resolve_fields(item, registry) + + if updates: + for k, v in updates.items(): + object.__setattr__(obj, k, v) + + return obj + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def resolve_references(model_instance: Any) -> Any: + """Post-deserialisation pass: replace string IDREFs with actual objects. + + This is the Python equivalent of Java's ``processReferences()`` (read + direction). It inspects the ``refs`` field of the model wrapper, builds + a :class:`VodmlIdRegistry`, then recursively walks the model tree + replacing every ``str``-valued reference field with the looked-up object. + + The resolution is driven entirely by two pieces of class-level metadata + emitted by the XSLT code generator: + + * ``_vodml_id_field`` — names the identity field on referenceable types + * ``_vodml_refs`` — lists the reference field names on each class + + Parameters + ---------- + model_instance + A top-level model wrapper (e.g. ``MyModelModel``) that has a ``refs`` + attribute containing the referenceable objects. + + Returns + ------- + The same *model_instance*, mutated in place. + """ + registry = VodmlIdRegistry() + + # Phase 1: build ID registry from the refs section + refs = getattr(model_instance, 'refs', None) + _build_registry_from_refs(refs, registry) + + if len(registry) == 0: + return model_instance + + logger.debug("Built ID registry with %d entries, resolving references …", len(registry)) + + # Phase 2: resolve IDREF strings in reference fields + _resolve_fields(model_instance, registry) + + return model_instance + diff --git a/runtime/python/vodml_runtime/xsdata/__init__.py b/runtime/python/vodml_runtime/xsdata/__init__.py new file mode 100644 index 00000000..a30c6cbe --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026. Paul Harrison, University of Manchester diff --git a/runtime/python/vodml_runtime/xsdata/parsers/__init__.py b/runtime/python/vodml_runtime/xsdata/parsers/__init__.py new file mode 100644 index 00000000..c7728dca --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/parsers/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) 2026. Paul Harrison, University of Manchester + +from vodml_runtime.xsdata.parsers.dict import DictDecoder +from vodml_runtime.xsdata.parsers.json import JsonParser + +__all__ = [ + "DictDecoder", + "JsonParser", +] diff --git a/runtime/python/vodml_runtime/xsdata/parsers/dict.py b/runtime/python/vodml_runtime/xsdata/parsers/dict.py new file mode 100644 index 00000000..38f7d0e5 --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/parsers/dict.py @@ -0,0 +1,521 @@ +from collections.abc import Iterable +from contextlib import suppress +from dataclasses import dataclass, field, replace +from typing import Any, get_args, get_origin, cast + +from xsdata.exceptions import ParserError +from xsdata.formats.converter import converter +from xsdata.formats.dataclass.context import XmlContext +from xsdata.formats.dataclass.models.elements import XmlMeta, XmlVar +from xsdata.formats.dataclass.parsers.config import ParserConfig +from xsdata.formats.dataclass.parsers.nodes.idref import get_obj_key +from xsdata.formats.dataclass.parsers.utils import ParserUtils +from xsdata.formats.types import T +from xsdata.utils import collections +from xsdata.utils.constants import EMPTY_MAP + + +def _utype_of(clazz: type) -> str | None: + """Return the ``Meta.utype`` declared directly on *clazz*, if any.""" + cls_meta = clazz.__dict__.get("Meta") + if cls_meta is not None: + return cls_meta.__dict__.get("utype") + return None + +def _is_referenced(clazz: type) -> bool: + """Return True if *clazz* is a referenced type.""" + return sum(1 for b in clazz.__mro__ if hasattr(b, 'Meta') and hasattr(b.Meta, 'key')) > 0 + +def _is_polymorphic(clazz: type) -> bool: + """Return True if *clazz* is a polymorphic type.""" + return sum(1 for b in clazz.__mro__ if hasattr(b, 'Meta') and hasattr(b.Meta, 'utype')) > 0 + +@dataclass +class DictDecoder: + """Bind a dictionary or a list of dictionaries to data models. + + Args: + config: Parser configuration + context: The models context instance + """ + + config: ParserConfig = field(default_factory=ParserConfig) + context: XmlContext = field(default_factory=XmlContext) + + def decode(self, data: list | dict, clazz: type[T] | None = None) -> T: + """Parse the input stream into the target class type. + + If no clazz is provided, the binding context will try + to locate it from imported dataclasses. + + Args: + data: A dictionary or list of dictionaries + clazz: The target class type to decode the input data + + Returns: + An instance of the specified class representing the decoded content. + """ + tp = self.verify_type(clazz, data) + if not isinstance(data, list): #impl not envisaged that lists will be presented + result = self.bind_dataclass(data, tp) + else: + result = [ + self.bind_dataclass(obj, tp) for obj in data + ] # type: ignore + return result + + def resolve_polymorphic(self, data: dict, clazz: type[T]) -> tuple[dict, type[T]]: + """Unwrap a VO-DML polymorphic type-hint wrapper, if present. + + Polymorphic values are serialised as a single-key object whose key is + the concrete class' ``Meta.utype`` and whose value is the actual + field data, e.g. ``{"MyModel:types.Dcont": {...}}``. This resolves the + real target class (which may be ``clazz`` itself, or one of its + subclasses) and returns the unwrapped data alongside it. + + Args: + data: The (possibly wrapped) data value + clazz: The statically declared/expected class + + Returns: + A tuple of (unwrapped data, resolved class). + """ + + + (key, inner), = data.items() + candidates = set(self.context.get_subclasses(clazz)) + candidates.add(clazz) + for candidate in candidates: + if _utype_of(candidate) == key: + return inner, candidate + raise ParserError(f"Failed to resolve polymorphic type {key} for {clazz.__qualname__}") + + + def _find_text_var(self, clazz: type | None) -> XmlVar | None: + """Return the sole ``Text`` var of *clazz*, if it is a simple-content + wrapper model (e.g. a VO-DML PrimitiveType specialisation), else None. + """ + if clazz is None or not self.context.class_type.is_model(clazz): + return None + meta = self.context.build(clazz) + for var in meta.get_all_vars(): + if var.is_text: + return var + return None + + def verify_type(self, clazz: type[T] | None, data: dict | list) -> type[T]: + """Verify the given data matches the given clazz. + + If no clazz is provided, the binding context will try + to locate it from imported dataclasses. + + Args: + clazz: The target class type to parse object + data: The loaded dictionary or list of dictionaries + + Returns: + The clazz type to bind the loaded data. + """ + if clazz is None: + return self.detect_type(data) + + try: + origin = get_origin(clazz) + list_type = False + if origin is list: + list_type = True + args = get_args(clazz) + + if len(args) != 1 or not self.context.class_type.is_model(args[0]): + raise TypeError + + clazz = args[0] + elif origin is not None: + raise TypeError + except TypeError: + raise ParserError(f"Invalid clazz argument: {clazz}") + + if list_type != isinstance(data, list): + if list_type: + raise ParserError("Document is object, expected array") + raise ParserError("Document is array, expected object") + + return clazz # type: ignore + + def detect_type(self, data: dict | list) -> type[T]: + """Locate the target clazz type from the data keys. + + Args: + data: The loaded dictionary or list of dictionaries + + Returns: + The clazz type to bind the loaded data. + """ + if not data: + raise ParserError("Document is empty, can not detect type") + + keys = data[0].keys() if isinstance(data, list) else data.keys() + clazz: type[T] | None = self.context.find_type_by_fields(set(keys)) + + if clazz: + return clazz + + raise ParserError(f"Unable to locate model with properties({list(keys)})") + + def bind_dataclass(self, data: dict, clazz: type[T]) -> T: + """Create a new instance of the given class type with the given data. + + Args: + data: The loaded data + clazz: The target class type to bind the input data + + Returns: + An instance of the class type representing the parsed content. + """ + if set(data.keys()) == self.context.class_type.derived_keys: + return self.bind_derived_dataclass(data, clazz) + + meta = self.context.build(clazz) + xml_vars = meta.get_all_vars() + + params = {} + for key, value in data.items(): + findkey = key if key != '_id' else 'id' #kludge + var = self.find_var(xml_vars, findkey, value) + + if var is None: + if self.config.fail_on_unknown_properties: + raise ParserError(f"Unknown property {clazz.__qualname__}.{findkey}") + continue + + # Note: unlike xsdata's default XML-oriented behaviour, VO-DML JSON + # stores the wrapped collection directly under the wrapper key + # (no extra nesting under var.local_name), so no further unwrap + # is required here. + + value = self.bind_value(meta, var, value) + if var.init: + params[var.name] = value + else: + ParserUtils.validate_fixed_value(meta, var, value) + + try: + retval = self.config.class_factory(clazz, params) + if _is_referenced(clazz): + idref = get_obj_key(retval) + if idref is not None: + self.context.idref_registry[str(idref)] = retval # this is a bit of a hack to use the idref_registry + return retval + except TypeError as e: + raise ParserError(e) + + def bind_derived_dataclass(self, data: dict, clazz: type[T]) -> Any: + """Bind the input data to the given class type. + + Examples: + { + "qname": "foo", + "@type": "my:type", + "value": {"prop": "value"} + } + + Args: + data: The derived element dictionary + clazz: The target class type to bind the input data + + Returns: + An instance of the class type representing the parsed content. + """ + qname = data["qname"] + xsi_type = data["@type"] + params = data["value"] + + generic = self.context.class_type.derived_element + + if clazz is generic: + real_clazz: type[T] | None = None + if xsi_type: + real_clazz = self.context.find_type(xsi_type) + + if real_clazz is None: + raise ParserError( + f"Unable to locate derived model " + f"with properties({list(params.keys())})" + ) + + value = self.bind_dataclass(params, real_clazz) + else: + value = self.bind_dataclass(params, clazz) + + return generic(qname=qname, type=xsi_type, value=value) + + def bind_best_dataclass(self, data: dict, classes: Iterable[type[T]]) -> T: + """Bind the input data to all the given classes and return best match. + + Args: + data: The derived element dictionary + classes: The target class types to try + + Returns: + An instance of one of the class types representing the parsed content. + """ + obj = None + keys = set(data.keys()) + max_score = -1.0 + config = replace(self.config, fail_on_converter_warnings=True) + decoder = DictDecoder(config=config, context=self.context) + + for clazz in classes: + if not self.context.class_type.is_model(clazz): + continue + + if self.context.local_names_match(keys, clazz): + candidate = None + with suppress(Exception): + candidate = decoder.bind_dataclass(data, clazz) + + score = self.context.class_type.score_object(candidate) + if score > max_score: + max_score = score + obj = candidate + + if obj: + return obj + + raise ParserError( + f"Failed to bind object with properties({list(data.keys())}) " + f"to any of the {[cls.__qualname__ for cls in classes]}" + ) + + def bind_value( + self, + meta: XmlMeta, + var: XmlVar, + value: Any, + recursive: bool = False, + ) -> Any: + """Main entry point for binding values. + + Args: + meta: The parent xml meta instance + var: The xml var descriptor for the field + value: The data value + recursive: Whether this is a recursive call + + Returns: + The parsed object + """ + # xs:anyAttributes get it out of the way, it's the mapping exception! + if var.is_attributes: + return dict(value) + + # VO-DML idref fields are serialised as plain identifier tokens + # (matching the referenced object's Meta.key attributes) rather than + # embedded objects. + def _get_idref(_val): + retval = self.context.idref_registry.get(str(_val)) # hack to + if retval is not None: + return retval + else: + raise ParserError( + f"Failed to find reference with key({_val}) " + f"for {meta.clazz.__qualname__}.{var.name} field") + + if var.is_idref and value is not None: + if collections.is_array(value): + return [_get_idref(val) for val in value] + else: + return _get_idref(value) + + + # Repeating element, recursively bind the values + if not recursive and var.list_element and isinstance(value, list): + assert var.factory is not None + return var.factory( + self.bind_value(meta, var, val, recursive=True) for val in value + ) + + # If not dict this is a text or tokens value - unless the field's + # declared class is itself a simple-content wrapper (a model with a + # single Text var, e.g. a VO-DML PrimitiveType specialisation such as + # ``altURL``/``ivoid``). VO-DML JSON (and Java's serialisation) + # writes those as a bare scalar rather than as ``{"value": ...}``. + if not isinstance(value, dict): + if not var.is_idref: + text_var = self._find_text_var(var.clazz) + if text_var is not None: + return self.bind_complex_type( + meta, var, {text_var.local_name: value} + ) + return self.bind_text(meta, var, value) + + keys = value.keys() + if keys == self.context.class_type.any_keys: + # Bind data to AnyElement dataclass + return self.bind_dataclass(value, self.context.class_type.any_element) + + if keys == self.context.class_type.derived_keys: + # Bind data to AnyElement dataclass + return self.bind_derived_value(meta, var, value) + + # Bind data to a user defined dataclass + return self.bind_complex_type(meta, var, value) + + def bind_text(self, meta: XmlMeta, var: XmlVar, value: Any) -> Any: + """Bind text/tokens value entrypoint. + + Args: + meta: The parent xml meta instance + var: The xml var descriptor for the field + value: The data value + + Returns: + The parsed tokens or text value. + """ + if var.is_elements: + # Compound field we need to match the value to one of the choice elements + check_subclass = self.context.class_type.is_model(value) + choice = var.find_value_choice(value, check_subclass) + if choice: + return self.bind_text(meta, choice, value) + + if value is None: + return value + + raise ParserError( + f"Failed to bind '{value}' " + f"to {meta.clazz.__qualname__}.{var.name} field" + ) + + if var.any_type or var.is_wildcard: + # field can support any object return the value as it is + return value + + value = converter.serialize(value) + + # Convert value according to the field types + return ParserUtils.parse_var( + meta=meta, + var=var, + config=self.config, + value=value, + ns_map=EMPTY_MAP, + ) + + def bind_complex_type(self, meta: XmlMeta, var: XmlVar, data: dict) -> Any: + """Bind complex values entrypoint. + + Args: + meta: The parent xml meta instance + var: The xml var descriptor for the field + data: The complex data value + + Returns: + The parsed dataclass instance. + """ + if var.is_clazz_union: + # Union of dataclasses + return self.bind_best_dataclass(data, var.types) + if var.elements: + # Compound field with multiple choices + return self.bind_best_dataclass(data, var.element_types) + if var.any_type or var.is_wildcard: + # xs:anyType element, check all meta classes + return self.bind_best_dataclass(data, meta.element_types) + + assert var.clazz is not None + + # VO-DML JSON tags polymorphic values with a single-key wrapper naming + # the concrete class' Meta.utype, regardless of whether the declared + # field type has any further subclasses. + + if _is_polymorphic(var.clazz): + return self.bind_dataclass(*self.resolve_polymorphic(data, var.clazz)) + else: + return self.bind_dataclass(data, var.clazz) + + def bind_derived_value(self, meta: XmlMeta, var: XmlVar, data: dict) -> Any: + """Bind derived data entrypoint. + + The data is representation of a derived element, e.g. { + "qname": "foo", + "type": "my:type" + "value": Any + } + + The data value can be a primitive value or a complex value. + + Args: + meta: The parent xml meta instance + var: The xml var descriptor for the field + data: The derived element data + + Returns: + The parsed object. + """ + qname = data["qname"] + xsi_type = data["type"] + params = data["value"] + + if var.elements: + choice = var.find_choice(qname) + if choice is None: + raise ParserError( + f"Unable to locate compound element" + f" {meta.clazz.__qualname__}.{var.name}[{qname}]" + ) + return self.bind_derived_value(meta, choice, data) + + if not isinstance(params, dict): + # Is this scenario still possible??? + value = self.bind_text(meta, var, params) + elif xsi_type: + clazz: type | None = self.context.find_type(xsi_type) + + if clazz is None: + raise ParserError(f"Unable to locate xsi:type `{xsi_type}`") + + value = self.bind_dataclass(params, clazz) + elif var.clazz: + value = self.bind_complex_type(meta, var, params) + else: + value = self.bind_best_dataclass(params, meta.element_types) + + generic = self.context.class_type.derived_element + return generic(qname=qname, value=value, type=xsi_type) + + @classmethod + def find_var( + cls, + xml_vars: list[XmlVar], + key: str, + value: Any, + ) -> XmlVar | None: + """Match the name to a xml variable. + + VO-DML JSON keys fields by their Python attribute name (``var.name``) + rather than the XML local name, and wrapped collections are emitted + directly under the wrapper key (no extra nesting). This matches + against ``var.name``, ``var.local_name`` and ``var.wrapper`` to + support both styles. + + Args: + xml_vars: A list of xml vars + key: A key from the loaded data + value: The data assigned to the key + + Returns: + One of the xml vars, if all search attributes match, None otherwise. + """ + for var in xml_vars: + if var.wrapper == key: + return var + + if var.local_name == key or var.name == key: + var_is_list = var.list_element or var.tokens + is_array = collections.is_array(value) + if is_array == var_is_list: + return var + return None + + diff --git a/runtime/python/vodml_runtime/xsdata/parsers/json.py b/runtime/python/vodml_runtime/xsdata/parsers/json.py new file mode 100644 index 00000000..1a123135 --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/parsers/json.py @@ -0,0 +1,105 @@ +import io +import json +import pathlib +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +from xsdata.exceptions import ParserError +from xsdata.formats.types import T + +from vodml_runtime.xsdata.parsers.dict import DictDecoder, _utype_of + + +@dataclass +class JsonParser(DictDecoder): + """Json parser for data classes. + + Args: + config: Parser configuration + context: The models context instance + load_factory: Json loader factory + """ + + load_factory: Callable = field(default=json.load) + + def from_path(self, path: pathlib.Path, clazz: type[T] | None = None) -> T: + """Parse the input file into the target class type. + + If no clazz is provided, the binding context will try + to locate it from imported dataclasses. + + Args: + path: The path to the input file + clazz: The target class type to parse the file into + + Returns: + An instance of the specified class representing the parsed content. + """ + return self.parse(str(path.resolve()), clazz) + + def from_string(self, source: str, clazz: type[T] | None = None) -> T: + """Parse the input source string into the target class type. + + If no clazz is provided, the binding context will try + to locate it from imported dataclasses. + + Args: + source: The source string to parse + clazz: The target class type to parse the source string into + + Returns: + An instance of the specified class representing the parsed content. + """ + return self.from_bytes(source.encode(), clazz) + + def from_bytes(self, source: bytes, clazz: type[T] | None = None) -> T: + """Parse the input source bytes object into the target class type. + + If no clazz is provided, the binding context will try + to locate it from imported dataclasses. + + Args: + source: The source bytes object to parse + clazz: The target class type to parse the source bytes object + + Returns: + An instance of the specified class representing the parsed content. + """ + return self.parse(io.BytesIO(source), clazz) + + def parse(self, source: Any, clazz: type[T] ) -> T: + """Parse the input stream into the target class type. + + Args: + source: The source file name or stream to parse + clazz: The target class type to parse the source object + + Returns: + An instance of the specified class representing the parsed content. + """ + data = self.load_json(source) + if len(data) == 1: + inner = data.get(_utype_of(clazz),None) + if inner is not None: + return self.decode(inner, clazz) + else: + raise ParserError(f"The input JSON data's root key '{key}' does not match the expected utype '{_utype_of(clazz)}' for the target class type '{clazz.__name__}'.") + else: + raise ParserError("The input JSON data must contain exactly one object to parse into the target class type.") + + + def load_json(self, source: Any) -> dict | list: + """Load the given json source filename or stream. + + Args: + source: A file name or file stream + + Returns: + The loaded dictionary or list of dictionaries. + """ + if not hasattr(source, "read"): + with open(source, "rb") as fp: + return self.load_factory(fp) + + return self.load_factory(source) diff --git a/runtime/python/vodml_runtime/xsdata/serializers/__init__.py b/runtime/python/vodml_runtime/xsdata/serializers/__init__.py new file mode 100644 index 00000000..28da23ab --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/serializers/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2026. Paul Harrison, University of Manchester + +from vodml_runtime.xsdata.serializers.dict import DictEncoder +from vodml_runtime.xsdata.serializers.dict import DictFactory +from vodml_runtime.xsdata.serializers.json import JsonSerializer + +__all__ = [ + "DictEncoder", + "DictFactory", + "JsonSerializer", +] \ No newline at end of file diff --git a/runtime/python/vodml_runtime/xsdata/serializers/dict.py b/runtime/python/vodml_runtime/xsdata/serializers/dict.py new file mode 100644 index 00000000..edac89c2 --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/serializers/dict.py @@ -0,0 +1,128 @@ +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + +from xsdata.formats.converter import converter +from xsdata.formats.dataclass.context import XmlContext +from xsdata.formats.dataclass.models.elements import XmlVar +from xsdata.formats.dataclass.serializers.config import SerializerConfig +from xsdata.utils import collections + + +def filter_none(x: tuple) -> dict: + """Convert a key-value pairs to dict, ignoring None values. + + Args: + x: Key-value pairs + + Returns: + The filtered dictionary. + """ + return {k: v for k, v in x if v is not None} + + +class DictFactory: + """Dictionary factory types.""" + + FILTER_NONE = filter_none + + +@dataclass +class DictEncoder: + """Json serializer for data classes. + + Args: + config: The serializer config instance + context: The models context instance + dict_factory: Dictionary factory + """ + + config: SerializerConfig = field(default_factory=SerializerConfig) + context: XmlContext = field(default_factory=XmlContext) + dict_factory: Callable = field(default=dict) + + def encode( + self, + value: Any, + var: XmlVar | None = None, + wrapped: bool = False, + ) -> Any: + """Convert a value to a dictionary object. + + Args: + value: The input value + var: The xml var instance + wrapped: Whether this is a wrapped value + + Returns: + The converted json serializable value. + """ + if value is None: + return None + + if var is None: + if collections.is_array(value): + return list(map(self.encode, value)) + + return self.dict_factory(self.next_value(value)) + + if var.is_idref and self.context.class_type.is_model(value): + key: list[str] = [] + for cls in value.__class__.__mro__: + cls_meta = cls.__dict__.get("Meta") + if cls_meta is not None and hasattr(cls_meta, "key"): + key = list(cls_meta.key) + break + key_value = "_".join(str(getattr(value, k)) for k in key) + return key_value + + if var and var.wrapper and not wrapped: + return self.dict_factory(((var.local_name, self.encode(value, var, True)),)) + + if self.context.class_type.is_model(value): + polymorphic_count = sum(1 for b in value.__class__.__mro__ if hasattr(b, 'Meta') and hasattr(b.Meta, 'utype')) + encoded = self.dict_factory(self.next_value(value)) + if polymorphic_count > 0: + utype = value.__class__.Meta.utype + return self.dict_factory(((utype, encoded),)) + return encoded + + if collections.is_array(value): + return type(value)(self.encode(val, var, wrapped) for val in value) + + if isinstance(value, (dict, int, float, str, bool)): + return value + + if isinstance(value, Enum): + return self.encode(value.value, var, wrapped) + + return converter.serialize(value, format=var.format) + + def next_value(self, obj: Any) -> Iterator[tuple[str, Any]]: + """Fetch the next value of a model instance to convert. + + Args: + obj: The input model instance + + Yields: + An iterator of field name and value tuples. + """ + ignore_optionals = self.config.ignore_default_attributes + meta = self.context.build(obj.__class__, globalns=self.config.globalns) + + for var in meta.get_all_vars(): + value = getattr(obj, var.name) + if ( + not ignore_optionals + or not var.is_optional(value) + ): + name = var.name + if name == "id": # this is kludge! + name = "_id" + + if var.wrapper: + yield var.wrapper, self.encode(value, var, True) + else: + yield name, self.encode(value, var) + diff --git a/runtime/python/vodml_runtime/xsdata/serializers/json.py b/runtime/python/vodml_runtime/xsdata/serializers/json.py new file mode 100644 index 00000000..8b80fc07 --- /dev/null +++ b/runtime/python/vodml_runtime/xsdata/serializers/json.py @@ -0,0 +1,48 @@ +import json +from collections.abc import Callable +from dataclasses import dataclass, field +from io import StringIO +from typing import Any, TextIO + +from xsdata.formats.dataclass.models.elements import XmlVar + +from vodml_runtime.xsdata.serializers import DictEncoder + + +@dataclass +class JsonSerializer(DictEncoder): + """Json serializer for data classes. + + Args: + config: The serializer config instance + context: The models context instance + dict_factory: Dictionary factory + dump_factory: Json dump factory e.g. json.dump + """ + + dump_factory: Callable = field(default=json.dump) + + def render(self, obj: Any) -> str: + """Serialize the input model instance to json string. + + Args: + obj: The input model instance + + Returns: + The serialized json string output. + """ + output = StringIO() + self.write(output, obj) + return output.getvalue() + + def write(self, out: TextIO, obj: Any) -> None: + """Serialize the given object to the output text stream. + + Args: + out: The output text stream + obj: The input model instance to serialize + """ + if hasattr(obj, "Meta"): + self.dump_factory(self.dict_factory(((obj.Meta.utype, self.encode(obj)), )), out, indent=self.config.indent) + else: + self.dump_factory(self.encode(obj), out, indent=self.config.indent) diff --git a/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlGradlePlugin.kt b/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlGradlePlugin.kt index 7bb717fe..5abfcb85 100644 --- a/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlGradlePlugin.kt +++ b/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlGradlePlugin.kt @@ -33,6 +33,7 @@ class VodmlGradlePlugin: Plugin { const val VODML_VODSL_TASK_NAME = "vodslToVodml" const val VODML_TO_VODSL_TASK_NAME = "vodmlToVodsl" const val VODML_TO_PYTHON_TASK_NAME = "vodmlPythonGenerate" + const val VODML_TO_PYDANTIC_TASK_NAME = "vodmlPydanticGenerate" const val VODML_SCHEMA_TASK_NAME = "vodmlSchema" const val XSD_TO_VODSL_TASK_NAME = "vodmlXsdToVodsl" } @@ -162,6 +163,13 @@ class VodmlGradlePlugin: Plugin { task.pythonGenDir.set(extension.outputPythonDir) } + // pydantic task + project.tasks.register(VODML_TO_PYDANTIC_TASK_NAME, VodmlPydanticTask::class.java) { task -> + task.description = "generate pydantic model classes from VO-DML models" + setVodmlFiles(task, extension, project) + task.pythonGenDir.set(extension.outputPythonDir) + } + //add the dependencies for JAXB and JPA - using the hibernate implementation diff --git a/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlPydanticTask.kt b/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlPydanticTask.kt new file mode 100644 index 00000000..d17e582e --- /dev/null +++ b/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/VodmlPydanticTask.kt @@ -0,0 +1,41 @@ +package net.ivoa.vodml.gradle.plugin + +import org.gradle.api.file.ArchiveOperations +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.tasks.* +import javax.inject.Inject + + +/** + * Generates Pydantic model code from the VO-DML models. + * Uses xsdata-pydantic for XML/JSON serialisation support. + */ +open class VodmlPydanticTask @Inject constructor(ao1: ArchiveOperations) : VodmlBaseTask(ao1) { + + @get:OutputDirectory + val pythonGenDir: DirectoryProperty = project.objects.directoryProperty() + + @TaskAction + fun doGeneration() { + logger.info("Generating Pydantic for VO-DML files ${vodmlFiles.files.joinToString { it.name }}") + logger.info("Looked in ${vodmlDir.get()}") + val eh = ExternalModelHelper(project, ao, logger) + val actualCatalog = eh.makeCatalog(vodmlFiles, catalogFile) + + val allBinding = bindingFiles.files.plus(eh.externalBinding()) + + var index = 0 + vodmlFiles.forEach { v -> + val shortname = v.nameWithoutExtension + val outfile = pythonGenDir.file("$shortname.pydantictrans.txt") + Vodml2Pydantic.doTransform( + v.absoluteFile, mapOf( + "binding" to allBinding.joinToString(separator = ",") { it.toURI().toURL().toString() }, + "output_root" to pythonGenDir.get().asFile.toURI().toURL().toString(), + "isMain" to (if (index++ == 0) "True" else "False") + ), + actualCatalog, outfile.get().asFile + ) + } + } +} diff --git a/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/XSLTTransform.kt b/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/XSLTTransform.kt index aa9b9c52..10ab7f98 100644 --- a/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/XSLTTransform.kt +++ b/tools/gradletooling/gradle-plugin/src/main/kotlin/net/ivoa/vodml/gradle/plugin/XSLTTransform.kt @@ -144,6 +144,7 @@ object Vodml2Java : XSLTTransformer("vo-dml2java.xsl", "text") object Vodml2Latex : XSLTTransformer("vo-dml2Latex.xsl", "text") object Vodml2Vodsl : XSLTTransformer("vo-dml2dsl.xsl", "text") object Vodml2Python : XSLTTransformer("vo-dml2python.xsl", "text") +object Vodml2Pydantic : XSLTTransformer("vo-dml2pydantic.xsl", "text") object Xsd2Vodsl : XSLTTransformer("xsd2dsl.xsl", "text") object Vodml2json : XSLTTransformer("vo-dml2jsonschema.xsl", "text") object Vodml2Catalogues : XSLTExecutionOnlyTransformer("create-catalogues.xsl", "main") diff --git a/tools/gradletooling/sample/build.gradle.kts b/tools/gradletooling/sample/build.gradle.kts index 4958305c..ed7f816f 100644 --- a/tools/gradletooling/sample/build.gradle.kts +++ b/tools/gradletooling/sample/build.gradle.kts @@ -114,15 +114,19 @@ python { // +":"+layout.projectDirectory.dir("../../../models/ivoa/build/generated/sources/vodml/python").asFile.absolutePath ) - pip("pytest:7.3.1") - pip("SQLAlchemy:2.0.30") - pip("xsdata[lxml,cli]:24.5") - pip("pydantic:2.9.2") + pip("pytest:9.0.2") + pip("SQLAlchemy:2.0.48") + pip("xsdata[lxml,cli]:26.2") + pip("pydantic:2.12.5") pip("sqlmodel:0.0.22") pip("xsdata-pydantic:24.5") + // pip("pydantic-xml:2.19.0") } +tasks.named("clean") { + delete(vodml.outputPythonDir) //additional clean up of generated python code when doing a clean build - note that even though it looks like it default behaviour is not being overridden +} tasks.register("tpath") { group = "Other" @@ -152,6 +156,13 @@ tasks.register("pytest", PythonTask::class.java) { dependsOn("vodmlPythonGenerate") } +tasks.register("pytestPydantic", PythonTask::class.java) { + group = "verification" + description = "run pydantic interoperability tests against generated pydantic models" + command = "-m pytest pythontest/src/PydanticInteropTest.py -v --junit-xml=build/reports/pytestPydantic/results.xml" + dependsOn("vodmlPydanticGenerate", "vodmlSchema") +} + tasks.register("siteNav") { commandLine("yq", "eval", "(.nav | .. |select(has(\"AutoGenerated Documentation\"))|.[\"AutoGenerated Documentation\"]) += load(\"docs/generated/allnav.yml\")", "mkdocs_template.yml") diff --git a/tools/gradletooling/sample/interoperability/java/jpatest.json b/tools/gradletooling/sample/interoperability/java/jpatest.json index ada113b8..5f7d8c83 100644 --- a/tools/gradletooling/sample/interoperability/java/jpatest.json +++ b/tools/gradletooling/sample/interoperability/java/jpatest.json @@ -1,37 +1,37 @@ { "JpatestModel" : { "refs" : { - "jpatest:Refbase" : [ { + "refbase" : [ { + "jpatest:ReferredTo2" : { + "_id" : 1016, + "sval" : "lower ref" + } + }, { "jpatest:ReferredTo3" : { - "_id" : 1024, - "sval" : "ref in sub", + "_id" : 1014, + "sval" : "ref in dtype", "ival" : 3 } }, { "jpatest:ReferredTo1" : { - "_id" : 1022, + "_id" : 1015, "sval" : "top level ref" } - }, { - "jpatest:ReferredTo2" : { - "_id" : 1023, - "sval" : "lower ref" - } }, { "jpatest:ReferredTo3" : { - "_id" : 1021, - "sval" : "ref in dtype", + "_id" : 1017, + "sval" : "ref in sub", "ival" : 3 } } ] }, - "jpatest:Parent" : [ { + "parent" : [ { "jpatest:Parent" : { "_id" : 0, "dval" : { "jpatest:ADtype" : { "basestr" : "base", - "dref" : 1021, + "dref" : 1014, "intatt" : "intatt", "dvalr" : 1.1, "dvals" : "astring" @@ -40,16 +40,16 @@ "eval" : { "jpatest:AEtype" : { "basestr" : "basestre_e", - "dref" : 1021, + "dref" : 1014, "intatt" : "intatt_e", "evalr" : 1.2, "evals" : "evals" } }, - "rval" : 1022, + "rval" : 1015, "cval" : { "_id" : 0, - "rval" : 1023 + "rval" : 1016 }, "lval" : [ { "_id" : 0, @@ -78,7 +78,7 @@ "dval" : { "jpatest:ADtype" : { "basestr" : "base", - "dref" : 1021, + "dref" : 1014, "intatt" : "intatt", "dvalr" : 1.1, "dvals" : "astring" @@ -87,16 +87,16 @@ "eval" : { "jpatest:AEtype" : { "basestr" : "basestre_e", - "dref" : 1021, + "dref" : 1014, "intatt" : "intatt_e", "evalr" : 1.2, "evals" : "evals" } }, - "rval" : 1022, + "rval" : 1015, "cval" : { "_id" : 0, - "rval" : 1023 + "rval" : 1016 }, "lval" : [ { "_id" : 0, @@ -119,10 +119,10 @@ "dt" : "thing" }, "subval" : "this is a subtype", - "slist" : [ { + "slists" : [ { "jpatest:AEtype" : { "basestr" : "basestre_sub", - "dref" : 1024, + "dref" : 1017, "intatt" : "intatt_sub", "evalr" : 1.3, "evals" : "subevals" diff --git a/tools/gradletooling/sample/interoperability/java/jpatest.xml b/tools/gradletooling/sample/interoperability/java/jpatest.xml index b6061aa7..58291373 100644 --- a/tools/gradletooling/sample/interoperability/java/jpatest.xml +++ b/tools/gradletooling/sample/interoperability/java/jpatest.xml @@ -1,38 +1,38 @@ - - lower ref - - - ref in sub - 3 + + top level ref - + ref in dtype 3 - - top level ref + + lower ref + + + ref in sub + 3 base - jpatest-ReferredTo3_1017 + jpatest-ReferredTo3_1010 intatt 1.1 astring basestre_e - jpatest-ReferredTo3_1017 + jpatest-ReferredTo3_1010 intatt_e 1.2 evals - jpatest-ReferredTo1_1018 + jpatest-ReferredTo1_1011 - jpatest-ReferredTo2_1019 + jpatest-ReferredTo2_1012 @@ -59,21 +59,21 @@ base - jpatest-ReferredTo3_1017 + jpatest-ReferredTo3_1010 intatt 1.1 astring basestre_e - jpatest-ReferredTo3_1017 + jpatest-ReferredTo3_1010 intatt_e 1.2 evals - jpatest-ReferredTo1_1018 + jpatest-ReferredTo1_1011 - jpatest-ReferredTo2_1019 + jpatest-ReferredTo2_1012 @@ -100,7 +100,7 @@ basestre_sub - jpatest-ReferredTo3_1020 + jpatest-ReferredTo3_1013 intatt_sub 1.3 subevals diff --git a/tools/gradletooling/sample/interoperability/java/lifecycle.json b/tools/gradletooling/sample/interoperability/java/lifecycle.json index c750b468..46165315 100644 --- a/tools/gradletooling/sample/interoperability/java/lifecycle.json +++ b/tools/gradletooling/sample/interoperability/java/lifecycle.json @@ -1,18 +1,21 @@ { "LifecycleTestModel" : { "refs" : { - "lifecycleTest:ReferredTo" : [ { - "_id" : 1031, + "referredTo" : [ { + "_id" : 1005, + "test1" : 4 + }, { + "_id" : 1004, "test1" : 3 } ] }, - "lifecycleTest:ATest3" : [ ], - "lifecycleTest:ATest2" : [ { + "aTest3" : [ ], + "aTest2" : [ { "_id" : 0, - "refagg" : [ 1031 ], + "refagg" : [ 1004, 1005 ], "atest" : { "_id" : 0, - "ref1" : 1031, + "ref1" : 1004, "contained" : [ { "_id" : 0, "test2" : "firstcontained" @@ -21,18 +24,18 @@ "test2" : "secondContained" } ], "refandcontained" : [ { - "_id" : 1032, + "_id" : 1006, "test3" : "rc1" }, { - "_id" : 1033, + "_id" : 1007, "test3" : "rc2" } ], "contained2" : { "_id" : 0, - "lowr" : 1032 + "lowr" : 1006 } }, - "refcont" : 1032 + "refcont" : 1006 } ] } } \ No newline at end of file diff --git a/tools/gradletooling/sample/interoperability/java/lifecycle.xml b/tools/gradletooling/sample/interoperability/java/lifecycle.xml index 22eac122..056f2928 100644 --- a/tools/gradletooling/sample/interoperability/java/lifecycle.xml +++ b/tools/gradletooling/sample/interoperability/java/lifecycle.xml @@ -1,13 +1,19 @@ - + 3 + + 4 + - lifecycleTest-ReferredTo_1028 + + lifecycleTest-ReferredTo_1000 + lifecycleTest-ReferredTo_1001 + - lifecycleTest-ReferredTo_1028 + lifecycleTest-ReferredTo_1000 firstcontained @@ -17,17 +23,17 @@ - + rc1 - + rc2 - lifecycleTest-ReferredLifeCycle_1029 + lifecycleTest-ReferredLifeCycle_1002 - lifecycleTest-ReferredLifeCycle_1029 + lifecycleTest-ReferredLifeCycle_1002 diff --git a/tools/gradletooling/sample/interoperability/java/notstccoords.json b/tools/gradletooling/sample/interoperability/java/notstccoords.json index 102d507e..98ca5a6a 100644 --- a/tools/gradletooling/sample/interoperability/java/notstccoords.json +++ b/tools/gradletooling/sample/interoperability/java/notstccoords.json @@ -1,9 +1,26 @@ { "CoordsModel" : { "refs" : { - "coords:CoordSys" : [ { + "coordSys" : [ { + "coords:SpaceSys" : { + "_id" : 1030, + "frame" : { + "coords:SpaceFrame" : { + "_id" : 0, + "refPosition" : { + "coords:StdRefLocation" : { + "_id" : 0, + "position" : "TOPOCENTER" + } + }, + "spaceRefFrame" : "ICRS", + "planetaryEphem" : "DE432" + } + } + } + }, { "coords:TimeSys" : { - "_id" : 1010, + "_id" : 1031, "frame" : { "coords:TimeFrame" : { "_id" : 0, @@ -20,24 +37,7 @@ "epoch" : "J2014.25", "position" : { "coords:LonLatPoint" : { - "coordSys" : { - "coords:SpaceSys" : { - "_id" : 1009, - "frame" : { - "coords:SpaceFrame" : { - "_id" : 0, - "refPosition" : { - "coords:StdRefLocation" : { - "_id" : 0, - "position" : "TOPOCENTER" - } - }, - "spaceRefFrame" : "ICRS", - "planetaryEphem" : "DE432" - } - } - } - }, + "coordSys" : 1030, "lon" : { "ivoa:RealQuantity" : { "unit" : { @@ -69,14 +69,14 @@ } } } - }, 1009 ] + } ] }, - "coords:CoordSpace" : [ ], - "coords:AnObject" : [ { + "coordSpace" : [ ], + "anObject" : [ { "_id" : 0, "position" : { "coords:LonLatPoint" : { - "coordSys" : 1009, + "coordSys" : 1030, "lon" : { "ivoa:RealQuantity" : { "unit" : { @@ -105,13 +105,13 @@ }, "time" : { "coords:MJD" : { - "coordSys" : 1010, + "coordSys" : 1031, "date" : 60000.0 } }, "sys" : { "coords:GenericSys" : { - "_id" : 1011, + "_id" : 1032, "frame" : { "coords:GenericFrame" : { "_id" : 0, diff --git a/tools/gradletooling/sample/interoperability/java/notstccoords.xml b/tools/gradletooling/sample/interoperability/java/notstccoords.xml index 48efeda3..fa8cb5b7 100644 --- a/tools/gradletooling/sample/interoperability/java/notstccoords.xml +++ b/tools/gradletooling/sample/interoperability/java/notstccoords.xml @@ -1,6 +1,6 @@ - + TOPOCENTER @@ -9,7 +9,7 @@ DE432 - + TOPOCENTER @@ -18,7 +18,7 @@ J2014.25 - coords-SpaceSys_1006 + coords-SpaceSys_1027 deg 6.752477 @@ -38,7 +38,7 @@ - coords-SpaceSys_1006 + coords-SpaceSys_1027 deg 45.0 @@ -53,10 +53,10 @@ - + TOPOCENTER diff --git a/tools/gradletooling/sample/interoperability/java/sample.json b/tools/gradletooling/sample/interoperability/java/sample.json index 4d4b2a37..e05d879c 100644 --- a/tools/gradletooling/sample/interoperability/java/sample.json +++ b/tools/gradletooling/sample/interoperability/java/sample.json @@ -1,19 +1,19 @@ { "SampleModel" : { "refs" : { - "sample:catalog.SkyCoordinateFrame" : [ { + "catalog.SkyCoordinateFrame" : [ { "name" : "J2000", "documentURI" : "http://coord.net", "equinox" : "J2000.0" } ] }, - "sample:catalog.AstroObject" : [ ], - "filter:PhotometricSystem" : [ { + "catalog.AstroObject" : [ ], + "photometricSystem" : [ { "_id" : 0, "description" : "test photometric system", "detectorType" : 1, "photometryFilter" : [ { - "_id" : 1036, + "_id" : 1038, "name" : "C-Band", "description" : "radio band", "bandName" : "C-Band", @@ -28,7 +28,7 @@ } } }, { - "_id" : 1037, + "_id" : 1039, "name" : "L-Band", "description" : "radio band", "bandName" : "L-Band", @@ -44,7 +44,7 @@ } } ] } ], - "sample:catalog.inner.SourceCatalogue" : [ { + "catalog.inner.SourceCatalogue" : [ { "_id" : 0, "name" : "testCat", "entry" : [ { @@ -98,7 +98,7 @@ }, "description" : "lummeas", "type" : "flux", - "filter" : 1036 + "filter" : 1038 }, { "_id" : 0, "value" : { @@ -119,7 +119,7 @@ }, "description" : "lummeas2", "type" : "flux", - "filter" : 1037 + "filter" : 1039 } ] } } ] diff --git a/tools/gradletooling/sample/interoperability/java/sample.xml b/tools/gradletooling/sample/interoperability/java/sample.xml index f49e5e2c..41e8f68c 100644 --- a/tools/gradletooling/sample/interoperability/java/sample.xml +++ b/tools/gradletooling/sample/interoperability/java/sample.xml @@ -9,7 +9,7 @@ test photometric system 1 - + C-Band radio band C-Band @@ -20,7 +20,7 @@ 5.0 - + L-Band radio band L-Band @@ -65,7 +65,7 @@ lummeas flux - filter-PhotometryFilter_1034 + filter-PhotometryFilter_1036 @@ -78,7 +78,7 @@ lummeas2 flux - filter-PhotometryFilter_1035 + filter-PhotometryFilter_1037 diff --git a/tools/gradletooling/sample/interoperability/java/serializationsample.json b/tools/gradletooling/sample/interoperability/java/serializationsample.json index 8b710e77..b4402c30 100644 --- a/tools/gradletooling/sample/interoperability/java/serializationsample.json +++ b/tools/gradletooling/sample/interoperability/java/serializationsample.json @@ -1,20 +1,19 @@ { "MyModelModel" : { "refs" : { - "MyModel:Refa" : [ { - "_id" : 1047, + "refa" : [ { + "_id" : 1049, "val" : "urn:value" } ], - "MyModel:Refb" : [ { + "refb" : [ { "name" : "naturalkey", "val" : "ivo:val" } ] }, - "MyModel:SomeContent" : [ { + "someContent" : [ { "_id" : 0, - "ref1" : 1047, + "ref1" : 1049, "ref2" : "naturalkey", - "zval" : [ "some", "z", "values" ], "con" : [ { "MyModel:types.Dcont" : { "_id" : 0, @@ -27,7 +26,9 @@ "bname" : "eval", "evalue" : "cube" } - } ] + } ], + "uri" : "urn:uri", + "zvals" : [ "some", "z", "values" ] } ] } } \ No newline at end of file diff --git a/tools/gradletooling/sample/interoperability/java/serializationsample.xml b/tools/gradletooling/sample/interoperability/java/serializationsample.xml index 4e330720..c8fa391d 100644 --- a/tools/gradletooling/sample/interoperability/java/serializationsample.xml +++ b/tools/gradletooling/sample/interoperability/java/serializationsample.xml @@ -1,6 +1,6 @@ - + urn:value @@ -9,7 +9,7 @@ - MyModel-Refa_1046 + MyModel-Refa_1048 naturalkey some @@ -26,5 +26,6 @@ cube + urn:uri diff --git a/tools/gradletooling/sample/interoperability/python/.gitkeep b/tools/gradletooling/sample/interoperability/python/.gitkeep new file mode 100644 index 00000000..0d873a19 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/.gitkeep @@ -0,0 +1 @@ +# Placeholder - files in this directory are written by the pydantic interoperability tests diff --git a/tools/gradletooling/sample/interoperability/python/jpatest.json b/tools/gradletooling/sample/interoperability/python/jpatest.json new file mode 100644 index 00000000..3af2ed04 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/jpatest.json @@ -0,0 +1,152 @@ +{ + "JpatestModel": { + "refs": { + "refbase": [ + { + "jpatest:ReferredTo3": { + "_id": "jpatest-ReferredTo3_1002", + "sval": "ref in dtype", + "ival": 3 + } + }, + { + "jpatest:ReferredTo1": { + "_id": "jpatest-ReferredTo1_1003", + "sval": "top level ref" + } + }, + { + "jpatest:ReferredTo2": { + "_id": "jpatest-ReferredTo2_1004", + "sval": "lower ref" + } + }, + { + "jpatest:ReferredTo3": { + "_id": "jpatest-ReferredTo3_1005", + "sval": "ref in sub", + "ival": 3 + } + } + ] + }, + "parent": [ + { + "jpatest:Parent": { + "_id": null, + "dval": { + "jpatest:ADtype": { + "basestr": "base", + "dref": "jpatest-ReferredTo3_1002", + "intatt": "intatt", + "dvalr": 1.1, + "dvals": "astring" + } + }, + "eval": { + "jpatest:AEtype": { + "basestr": "basestre_e", + "dref": "jpatest-ReferredTo3_1002", + "intatt": "intatt_e", + "evalr": 1.2, + "evals": "evals" + } + }, + "rval": "jpatest-ReferredTo1_1003", + "cval": { + "_id": null, + "rval": "jpatest-ReferredTo2_1004" + }, + "lval": [ + { + "_id": null, + "sval": "First", + "ival": 1 + }, + { + "_id": null, + "sval": "Second", + "ival": 2 + }, + { + "_id": null, + "sval": "Third", + "ival": 3 + } + ], + "tval": { + "p": { + "x": 1.5, + "y": 3.0 + }, + "dt": "thing" + } + } + }, + { + "jpatest:Sub": { + "_id": null, + "dval": { + "jpatest:ADtype": { + "basestr": "base", + "dref": "jpatest-ReferredTo3_1002", + "intatt": "intatt", + "dvalr": 1.1, + "dvals": "astring" + } + }, + "eval": { + "jpatest:AEtype": { + "basestr": "basestre_e", + "dref": "jpatest-ReferredTo3_1002", + "intatt": "intatt_e", + "evalr": 1.2, + "evals": "evals" + } + }, + "rval": "jpatest-ReferredTo1_1003", + "cval": { + "_id": null, + "rval": "jpatest-ReferredTo2_1004" + }, + "lval": [ + { + "_id": null, + "sval": "First", + "ival": 1 + }, + { + "_id": null, + "sval": "Second", + "ival": 2 + }, + { + "_id": null, + "sval": "Third", + "ival": 3 + } + ], + "tval": { + "p": { + "x": 1.5, + "y": 3.0 + }, + "dt": "thing" + }, + "subval": "this is a subtype", + "slists": [ + { + "jpatest:AEtype": { + "basestr": "basestre_sub", + "dref": "jpatest-ReferredTo3_1005", + "intatt": "intatt_sub", + "evalr": 1.3, + "evals": "subevals" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/tools/gradletooling/sample/interoperability/python/jpatest.xml b/tools/gradletooling/sample/interoperability/python/jpatest.xml new file mode 100644 index 00000000..d17af4a2 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/jpatest.xml @@ -0,0 +1,111 @@ + + + + + ref in dtype + 3 + + + top level ref + + + lower ref + + + ref in sub + 3 + + + + + base + jpatest-ReferredTo3_1002 + intatt + 1.1 + astring + + + basestre_e + jpatest-ReferredTo3_1002 + intatt_e + 1.2 + evals + + jpatest-ReferredTo1_1003 + + jpatest-ReferredTo2_1004 + + + + First + 1 + + + Second + 2 + + + Third + 3 + + + +

+ 1.5 + 3.0 +

+
thing
+ + + + + base + jpatest-ReferredTo3_1002 + intatt + 1.1 + astring + + + basestre_e + jpatest-ReferredTo3_1002 + intatt_e + 1.2 + evals + + jpatest-ReferredTo1_1003 + + jpatest-ReferredTo2_1004 + + + + First + 1 + + + Second + 2 + + + Third + 3 + + + +

+ 1.5 + 3.0 +

+
thing
+ + this is a subtype + + + basestre_sub + jpatest-ReferredTo3_1005 + intatt_sub + 1.3 + subevals + + + + diff --git a/tools/gradletooling/sample/interoperability/python/lifecycle.json b/tools/gradletooling/sample/interoperability/python/lifecycle.json new file mode 100644 index 00000000..e7463750 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/lifecycle.json @@ -0,0 +1,55 @@ +{ + "LifecycleTestModel": { + "refs": { + "referredTo": [ + { + "_id": "lifecycleTest-ReferredTo_1000", + "test1": 3 + }, + { + "_id": "lifecycleTest-ReferredTo_1001", + "test1": 4 + } + ] + }, + "aTest3": [], + "aTest2": [ + { + "_id": null, + "refagg": [ + "lifecycleTest-ReferredTo_1000", + "lifecycleTest-ReferredTo_1001" + ], + "atest": { + "_id": null, + "ref1": "lifecycleTest-ReferredTo_1000", + "contained": [ + { + "_id": null, + "test2": "firstcontained" + }, + { + "_id": null, + "test2": "secondContained" + } + ], + "refandcontained": [ + { + "_id": "lifecycleTest-ReferredLifeCycle_1002", + "test3": "rc1" + }, + { + "_id": "lifecycleTest-ReferredLifeCycle_1003", + "test3": "rc2" + } + ], + "contained2": { + "_id": null, + "lowr": "lifecycleTest-ReferredLifeCycle_1002" + } + }, + "refcont": "lifecycleTest-ReferredLifeCycle_1002" + } + ] + } +} \ No newline at end of file diff --git a/tools/gradletooling/sample/interoperability/python/lifecycle.xml b/tools/gradletooling/sample/interoperability/python/lifecycle.xml new file mode 100644 index 00000000..630b27e5 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/lifecycle.xml @@ -0,0 +1,40 @@ + + + + + 3 + + + 4 + + + + + lifecycleTest-ReferredTo_1000 + lifecycleTest-ReferredTo_1001 + + + lifecycleTest-ReferredTo_1000 + + + firstcontained + + + secondContained + + + + + rc1 + + + rc2 + + + + lifecycleTest-ReferredLifeCycle_1002 + + + lifecycleTest-ReferredLifeCycle_1002 + + diff --git a/tools/gradletooling/sample/interoperability/python/sample.json b/tools/gradletooling/sample/interoperability/python/sample.json new file mode 100644 index 00000000..aaced0d0 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/sample.json @@ -0,0 +1,148 @@ +{ + "SampleModel": { + "refs": { + "catalog_SkyCoordinateFrame": [ + { + "name": "J2000", + "documentURI": "http://coord.net", + "equinox": "J2000.0", + "system": null + } + ] + }, + "astroObject": [], + "photometricSystem": [ + { + "_id": null, + "description": "test photometric system", + "detectorType": 1, + "photometryFilter": [ + { + "_id": "filter-PhotometryFilter_1034", + "fpsIdentifier": null, + "name": "C-Band", + "description": "radio band", + "bandName": "C-Band", + "dataValidityFrom": "2020-01-01T00:00:00Z", + "dataValidityTo": "2025-01-01T20:12:16Z", + "spectralLocation": { + "ivoa:RealQuantity": { + "unit": { + "value": "GHz" + }, + "value": 5.0 + } + } + }, + { + "_id": "filter-PhotometryFilter_1035", + "fpsIdentifier": null, + "name": "L-Band", + "description": "radio band", + "bandName": "L-Band", + "dataValidityFrom": "2020-01-01T00:00:00Z", + "dataValidityTo": "2025-01-01T13:12:00Z", + "spectralLocation": { + "ivoa:RealQuantity": { + "unit": { + "value": "GHz" + }, + "value": 1.5 + } + } + } + ] + } + ], + "sourceCatalogue": [ + { + "_id": null, + "name": "testCat", + "entry": [ + { + "sample:catalog.SDSSSource": { + "_id": null, + "label": "cepheid", + "name": "testSource", + "description": null, + "position": { + "longitude": { + "ivoa:RealQuantity": { + "unit": { + "value": "degree" + }, + "value": 2.5 + } + }, + "latitude": { + "ivoa:RealQuantity": { + "unit": { + "value": "degree" + }, + "value": 52.5 + } + }, + "frame": "J2000" + }, + "positionError": { + "sample:catalog.AlignedEllipse": { + "longError": 0.2, + "latError": 0.1 + } + }, + "classification": "AGN", + "luminosity": [ + { + "_id": null, + "value": { + "ivoa:RealQuantity": { + "unit": { + "value": "Jy" + }, + "value": 2.5 + } + }, + "error": { + "ivoa:RealQuantity": { + "unit": { + "value": "Jy" + }, + "value": 0.25 + } + }, + "description": "lummeas", + "type": "flux", + "filter": "filter-PhotometryFilter_1034" + }, + { + "_id": null, + "value": { + "ivoa:RealQuantity": { + "unit": { + "value": "Jy" + }, + "value": 3.5 + } + }, + "error": { + "ivoa:RealQuantity": { + "unit": { + "value": "Jy" + }, + "value": 0.25 + } + }, + "description": "lummeas2", + "type": "flux", + "filter": "filter-PhotometryFilter_1035" + } + ] + } + } + ], + "aTest": null, + "aTestMore": null + } + ] + } +} \ No newline at end of file diff --git a/tools/gradletooling/sample/interoperability/python/sample.xml b/tools/gradletooling/sample/interoperability/python/sample.xml new file mode 100644 index 00000000..f0dcc9c5 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/sample.xml @@ -0,0 +1,86 @@ + + + + + http://coord.net + J2000.0 + + + + test photometric system + 1 + + + C-Band + radio band + C-Band + 2020-01-01T00:00:00Z + 2025-01-01T20:12:16Z + + GHz + 5.0 + + + + L-Band + radio band + L-Band + 2020-01-01T00:00:00Z + 2025-01-01T13:12:00Z + + GHz + 1.5 + + + + + + testCat + + + testSource + + + degree + 2.5 + + + degree + 52.5 + + J2000 + + + 0.2 + 0.1 + + AGN + + + Jy + 2.5 + + + Jy + 0.25 + + lummeas + flux + filter-PhotometryFilter_1034 + + + + Jy + 3.5 + + + Jy + 0.25 + + lummeas2 + flux + filter-PhotometryFilter_1035 + + + + diff --git a/tools/gradletooling/sample/interoperability/python/serializationsample.json b/tools/gradletooling/sample/interoperability/python/serializationsample.json new file mode 100644 index 00000000..1ddfb401 --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/serializationsample.json @@ -0,0 +1,51 @@ +{ + "MyModelModel": { + "refs": { + "refa": [ + { + "_id": "MyModel-Refa_1000", + "val": { + "value": "urn:value" + } + } + ], + "refb": [ + { + "name": "naturalkey", + "val": { + "value": "ivo:val" + } + } + ] + }, + "someContent": [ + { + "_id": null, + "ref1": "MyModel-Refa_1000", + "ref2": "naturalkey", + "zvals": [ + "some", + "z", + "values" + ], + "con": [ + { + "MyModel:types.Dcont": { + "_id": null, + "bname": "dval", + "dval": "N1" + } + }, + { + "MyModel:types.Econt": { + "_id": null, + "bname": "eval", + "evalue": "cube" + } + } + ], + "uri": "urn:uri" + } + ] + } +} \ No newline at end of file diff --git a/tools/gradletooling/sample/interoperability/python/serializationsample.xml b/tools/gradletooling/sample/interoperability/python/serializationsample.xml new file mode 100644 index 00000000..511fe9ed --- /dev/null +++ b/tools/gradletooling/sample/interoperability/python/serializationsample.xml @@ -0,0 +1,32 @@ + + + + + urn:value + + + naturalkey + ivo:val + + + + MyModel-Refa_1000 + naturalkey + + some + z + values + + + + dval + N1 + + + eval + cube + + + urn:uri + + diff --git a/tools/gradletooling/sample/pythontest/src/PydanticInteropTest.py b/tools/gradletooling/sample/pythontest/src/PydanticInteropTest.py new file mode 100644 index 00000000..0c80dae7 --- /dev/null +++ b/tools/gradletooling/sample/pythontest/src/PydanticInteropTest.py @@ -0,0 +1,656 @@ +""" +Pydantic interoperability tests. + +These tests create the same model instances that the Java serialisation tests produce, +re-serialise them to XML and JSON using the generated pydantic-xml models, and write +the results to interoperability/python/. The output can then be compared with the +files in interoperability/java/ to evaluate how close the two serialisations are. +""" + +import json +import unittest +import xml.etree.ElementTree as ET +from datetime import datetime, timezone +from pathlib import Path + +from lxml import etree as _etree +from sqlalchemy import true + +from org.ivoa.dm.filter.filter import PhotometricSystem, PhotometryFilter +from org.ivoa.dm.ivoa import RealQuantity, Unit +from org.ivoa.dm.jpatest.jpatest import JpatestModel + +from org.ivoa.dm.lifecycle.lifecycleTest import LifecycleTestModel, LifecycleTestRefs +from org.ivoa.dm.samplemodel.sample import SampleModel, SampleRefs + +from org.ivoa.dm.samplemodel.sample_catalog_inner import SourceCatalogue +from org.ivoa.dm.serializationsample.MyModel import MyModelModel, MyModelRefs + +# Output directory (relative to the sample project root). +_SAMPLE_DIR = Path(__file__).parent.parent.parent +_INTEROP_DIR = _SAMPLE_DIR / "interoperability" / "python" +_JAVA_DIR = _SAMPLE_DIR / "interoperability" / "java" + +# Directory containing the generated XSD schemas (populated by :sample:vodmlSchema). +_SCHEMA_DIR = _SAMPLE_DIR / "docs" / "schema" + +# The IVOA base XSD lives in the ivoa model build resources (present after :ivoa:jar). +_IVOA_XSD = _SAMPLE_DIR.parent.parent.parent / "models" / "ivoa" / "build" / "resources" / "main" / "IVOA-v1.0.vo-dml.xsd" + + +class _LocalSchemaResolver(_etree.Resolver): + """Resolve relative schema imports from ``_SCHEMA_DIR`` or the ivoa build output.""" + + def resolve(self, url: str, id, context): + name = Path(url).name + local = _SCHEMA_DIR / name + if local.exists(): + return self.resolve_filename(str(local), context) + if name == "IVOA-v1.0.vo-dml.xsd" and _IVOA_XSD.exists(): + return self.resolve_filename(str(_IVOA_XSD), context) + return None + + +def _load_schema(xsd_name: str) -> "_etree.XMLSchema | None": + """Load an XMLSchema from *_SCHEMA_DIR*, returning ``None`` when unavailable.""" + xsd_path = _SCHEMA_DIR / xsd_name + if not xsd_path.exists(): + return None + parser = _etree.XMLParser() + parser.resolvers.add(_LocalSchemaResolver()) + return _etree.XMLSchema(_etree.parse(str(xsd_path), parser)) + + +def _validate_xml(xml_bytes: bytes, xsd_name: str, test_case: unittest.TestCase) -> None: + """Assert that *xml_bytes* validates against the named XSD schema. + + If the schema file is not present (e.g. ``vodmlSchema`` was not run), the + check is silently skipped so the test does not fail in minimal build + environments. + """ + schema = _load_schema(xsd_name) + if schema is None: + return + doc = _etree.fromstring(xml_bytes) + result = schema.validate(doc) + if not result: + errors = "\n".join(str(e) for e in schema.error_log) + test_case.fail(f"XML failed schema validation against {xsd_name}:\n{errors}") + + +def _write(filename: str, content: str | bytes) -> None: + """Write *content* to interoperability/python/.""" + _INTEROP_DIR.mkdir(parents=True, exist_ok=True) + dest = _INTEROP_DIR / filename + if isinstance(content, bytes): + dest.write_bytes(content) + else: + dest.write_text(content, encoding="utf-8") + + +def _local_name(tag: str) -> str: + """Return the local part of an XML tag name.""" + return tag.split("}", 1)[-1] + + +def _find_first(root: ET.Element, local_name: str) -> ET.Element | None: + """Return the first element in *root* whose local name matches *local_name*.""" + return next((el for el in root.iter() if _local_name(el.tag) == local_name), None) + + +def _children_named(element: ET.Element, local_name: str) -> list[ET.Element]: + """Return the direct children of *element* with the given local name.""" + return [child for child in list(element) if _local_name(child.tag) == local_name] + + +def _first_child_text(element: ET.Element, local_name: str) -> str | None: + """Return the text of the first direct child called *local_name*.""" + child = next((c for c in list(element) if _local_name(c.tag) == local_name), None) + return child.text if child is not None else None + + +def _read_json(filename: str) -> dict: + with open(_INTEROP_DIR / filename, encoding="utf-8") as fh: + return json.load(fh) + + +def _read_xml_root(filename: str) -> ET.Element: + return ET.parse(str(_INTEROP_DIR / filename)).getroot() + +def _read_java_file_as_bytes(path: str) -> bytes: + with open(_JAVA_DIR / path, "rb") as f: + return f.read() + +class SampleModelInteropTest(unittest.TestCase): + """ + Tests for the Sample model (SourceCatalogue / SDSSSource). + + Mirrors the Java BaseSourceCatalogueTest / SourceCatalogueTest which produce + interoperability/java/sample.xml and interoperability/java/sample.json. + """ + + @classmethod + def setUpClass(cls): + from org.ivoa.dm.samplemodel.sample_catalog import ( + AlignedEllipse, + LuminosityMeasurement, + LuminosityType, + SDSSSource, + SkyCoordinate, + SkyCoordinateFrame, + SourceClassification, + ) + jansky = Unit(value="Jy") + degree = Unit(value="degree") + ghz = Unit(value="GHz") + + frame = SkyCoordinateFrame( + name="J2000", + equinox="J2000.0", + documentURI="http://coord.net", + ) + + c_band = PhotometryFilter( + id="filter-PhotometryFilter_1034", # TODO this should be autogenerated + name="C-Band", + description="radio band", + bandName="C-Band", + dataValidityFrom=datetime(2020, 1, 1, tzinfo=timezone.utc), + dataValidityTo=datetime(2025, 1, 1, 20, 12, 16, tzinfo=timezone.utc), + spectralLocation=RealQuantity(value=5.0, unit=ghz), + ) + l_band = PhotometryFilter( + id="filter-PhotometryFilter_1035", # TODO this should be autogenerated + name="L-Band", + description="radio band", + bandName="L-Band", + dataValidityFrom=datetime(2020, 1, 1, tzinfo=timezone.utc), + dataValidityTo=datetime(2025, 1, 1, 13, 12, 0, tzinfo=timezone.utc), + spectralLocation=RealQuantity(value=1.5, unit=ghz), + ) + + cls.ps = PhotometricSystem( + description="test photometric system", + detectorType=1, + photometryFilter=[c_band, l_band], + ) + + source = SDSSSource( + name="testSource", + label="cepheid", + classification=SourceClassification.AGN, + position=SkyCoordinate( + longitude=RealQuantity(value=2.5, unit=degree), + latitude=RealQuantity(value=52.5, unit=degree), + frame=frame, + ), + positionError=AlignedEllipse(longError=0.2, latError=0.1), + luminosity=[ + LuminosityMeasurement( + description="lummeas", + type=LuminosityType.FLUX, + value=RealQuantity(value=2.5, unit=jansky), + error=RealQuantity(value=0.25, unit=jansky), + filter=c_band, + ), + LuminosityMeasurement( + description="lummeas2", + type=LuminosityType.FLUX, + value=RealQuantity(value=3.5, unit=jansky), + error=RealQuantity(value=0.25, unit=jansky), + filter=l_band, + ), + ], + ) + + cls.sc = SourceCatalogue(name="testCat", entry=[source]) + + cls.model = SampleModel(photometricSystem=[cls.ps], sourceCatalogue=[cls.sc],refs=SampleRefs(catalog_SkyCoordinateFrame=[frame])) + + # ------------------------------------------------------------------ + # JSON + # ------------------------------------------------------------------ + + def test_json_serialise(self): + json_str = self.model.to_xsjson(pretty_print=True) + _write("sample.json", json_str) + + data = json.loads(json_str) + self.assertEqual(data["SampleModel"]["sourceCatalogue"][0]["name"], "testCat") + entry = data["SampleModel"]["sourceCatalogue"][0]["entry"][0]["sample:catalog.SDSSSource"] + self.assertEqual(entry["name"], "testSource") + self.assertEqual(entry["position"]["frame"], "J2000") + self.assertEqual(len(entry["luminosity"]), 2) + self.assertAlmostEqual(entry["position"]["longitude"]["ivoa:RealQuantity"]["value"], 2.5) + self.assertEqual(data["SampleModel"]["photometricSystem"][0]["photometryFilter"][1]["name"], "L-Band") + + def test_json_round_trip(self): + recovered = SampleModel.model_validate_json(self.model.model_dump_json()) + self.assertEqual(recovered.sourceCatalogue[0].name, "testCat") + self.assertEqual(recovered.sourceCatalogue[0].entry[0].name, "testSource") + self.assertEqual(recovered.refs.catalog_SkyCoordinateFrame[0].name, "J2000") + + def test_xml_serialise(self): + xml_bytes = self.model.full_model_to_xml(pretty_print=True) + _write("sample.xml", xml_bytes) + _validate_xml(xml_bytes, "Sample.vo-dml.xsd", self) + root = ET.fromstring(xml_bytes) + self.assertEqual(_local_name(root.tag), "sampleModel") + source_catalogue = _find_first(root, "catalog.inner.SourceCatalogue") + self.assertIsNotNone(source_catalogue) + self.assertEqual(_first_child_text(source_catalogue, "name"), "testCat") + self.assertEqual(_first_child_text(_find_first(source_catalogue, "entry"), "name"), "testSource") + + def test_read_java_serialization_xml(self): + recovered = SampleModel.from_xml( _read_java_file_as_bytes("sample.xml")) + self.assertEqual(recovered.sourceCatalogue[0].name, "testCat") + self.assertEqual(recovered.sourceCatalogue[0].entry[0].name, "testSource") + self.assertEqual(recovered.photometricSystem[0].photometryFilter[0].name, "C-Band") + + def test_read_java_serialization_json(self): + from_java = SampleModel.from_xsjson( _read_java_file_as_bytes("sample.json")) + # would like to do the below, but cannot because os the ids + # self.assertEqual(from_java, self.model) + self.assertIsNotNone(from_java) + +class LifecycleModelInteropTest(unittest.TestCase): + """ + Tests for the Lifecycle test model. + + Mirrors the Java LifecycleTestModelTest which produces + interoperability/java/lifecycle.xml and interoperability/java/lifecycle.json. + """ + + @classmethod + def setUpClass(cls): + from org.ivoa.dm.lifecycle.lifecycleTest import ( + ATest, + ATest2, + ATest4, + Contained, + ReferredLifeCycle, + ReferredTo, + ) + #FIXME really need to add equivalent of the java processReferences() runtime functionality to set up the ids for references - and make tests not dependent on the exact values... + ref1 = ReferredTo(id="lifecycleTest-ReferredTo_1000", test1=3) + ref2 = ReferredTo(id="lifecycleTest-ReferredTo_1001", test1=4) + rc1 = ReferredLifeCycle(id="lifecycleTest-ReferredLifeCycle_1002", test3="rc1") + rc2 = ReferredLifeCycle(id="lifecycleTest-ReferredLifeCycle_1003", test3="rc2") + atest = ATest( + ref1=ref1, + contained=[ + Contained(test2="firstcontained"), + Contained(test2="secondContained"), + ], + refandcontained=[rc1, rc2], + contained2=ATest4(lowr=rc1), + ) + top = ATest2(atest=atest, refcont=rc1, refagg=[ref1, ref2]) + cls.model=LifecycleTestModel(aTest2=[top],refs=LifecycleTestRefs(referredTo=[ref1, ref2])) + print(f"LifecycleModelInteropTest: setUpClass created model {cls.model}") + + def test_json_serialise(self): + json_str = self.model.to_xsjson(pretty_print=True) + _write("lifecycle.json", json_str) + + data = json.loads(json_str) + self.assertEqual(data["LifecycleTestModel"]["refs"]["referredTo"][0]["test1"], 3) + atest2 = data["LifecycleTestModel"]["aTest2"][0] + self.assertEqual(atest2["refcont"], "lifecycleTest-ReferredLifeCycle_1002") + self.assertEqual(len(atest2["atest"]["contained"]), 2) + self.assertEqual(atest2["atest"]["refandcontained"][1]["test3"], "rc2") + + def test_json_round_trip(self): + recovered = LifecycleTestModel.model_validate_json(self.model.model_dump_json()) + self.assertEqual(recovered.aTest2[0].atest.contained[0].test2, "firstcontained") + self.assertEqual(recovered.aTest2[0].refcont.id, "lifecycleTest-ReferredLifeCycle_1002") + + def test_xml_serialise(self): + xml_bytes = self.model.full_model_to_xml(pretty_print=True) + _write("lifecycle.xml", xml_bytes) + _validate_xml(xml_bytes, "lifecycleTest.vo-dml.xsd", self) + root = ET.fromstring(xml_bytes) + self.assertEqual(_local_name(root.tag), "lifecycleTestModel") + atest2 = _find_first(root, "aTest2") + self.assertIsNotNone(atest2) + self.assertEqual(_first_child_text(atest2, "refcont"), "lifecycleTest-ReferredLifeCycle_1002") + self.assertIn("firstcontained", "".join(el.text or "" for el in root.iter() if _local_name(el.tag) == "test2")) + + def test_read_java_serialization_xml(self): + recovered = LifecycleTestModel.from_xml( _read_java_file_as_bytes("lifecycle.xml")) + self.assertEqual(len(recovered.aTest2[0].atest.contained), 2) + self.assertEqual(recovered.aTest2[0].atest.contained2.lowr.id, "lifecycleTest-ReferredLifeCycle_1002") + self.assertEqual(recovered, self.model) + + def test_read_java_serialization_json(self): + from_java = LifecycleTestModel.from_xsjson( _read_java_file_as_bytes("lifecycle.json")) + # would like to do the below, but cannot because os the ids + # self.assertEqual(from_java, self.model) + self.assertIsNotNone(from_java) + + +class SerializationExampleInteropTest(unittest.TestCase): + """ + Tests for the serialisation example model (MyModel). + + Mirrors the Java SerializationExampleTest which produces + interoperability/java/serializationsample.xml and + interoperability/java/serializationsample.json. + """ + + @classmethod + def setUpClass(cls): + from org.ivoa.dm.serializationsample.MyModel import Refa, Refb, SomeContent, altURL, ivoid + from org.ivoa.dm.serializationsample.MyModel_types import Dcont, Econt + + refa = Refa(id="MyModel-Refa_1000", val=altURL(value="urn:value")) + refb = Refb(name="naturalkey", val=ivoid(value="ivo:val")) + cls.model = MyModelModel( + someContent=[ + SomeContent( + ref1=refa, + ref2=refb, + zval=["some", "z", "values"], + con=[ + Dcont(bname="dval", dval="N1"), + Econt(bname="eval", evalue="cube"), + ], + uri="urn:uri" + ) + ], + refs=MyModelRefs(refa=[refa], refb=[refb]), + ) + + def test_json_serialise(self): + json_str = self.model.to_xsjson(pretty_print=True) + _write("serializationsample.json", json_str) + + data = json.loads(json_str) + self.assertEqual(data["MyModelModel"]["refs"]["refa"][0]["_id"], "MyModel-Refa_1000") + content = data["MyModelModel"]["someContent"][0] + self.assertEqual(content["ref1"], "MyModel-Refa_1000") + self.assertEqual(content["ref2"], "naturalkey") + self.assertEqual(content["zvals"], ["some", "z", "values"]) + self.assertEqual(len(content["con"]), 2) + + def test_json_round_trip(self): + recovered = MyModelModel.model_validate_json(self.model.model_dump_json()) + self.assertEqual(recovered.someContent[0].zval, ["some", "z", "values"]) + self.assertEqual(recovered.refs.refb[0].name, "naturalkey") + + + + def test_xml_serialise(self): + xml_bytes = self.model.full_model_to_xml(pretty_print=True) + _write("serializationsample.xml", xml_bytes) + _validate_xml(xml_bytes, "serializationExample.vo-dml.xsd", self) + root = ET.fromstring(xml_bytes) + self.assertEqual(_local_name(root.tag), "MyModelModel") + some_content = _find_first(root, "someContent") + self.assertIsNotNone(some_content) + self.assertEqual(_first_child_text(some_content, "ref1"), "MyModel-Refa_1000") + zvals = [el.text for el in root.iter() if _local_name(el.tag) == "zval"] + self.assertEqual(zvals, ["some", "z", "values"]) + + + def test_read_java_serialization_xml(self): + from org.ivoa.dm.serializationsample.MyModel import Refa + from_java = MyModelModel.from_xml( _read_java_file_as_bytes("serializationsample.xml")) + self.assertIsInstance(from_java.someContent[0].ref1, Refa) + + def test_read_java_serialization_json(self): + from org.ivoa.dm.serializationsample.MyModel import Refa + from_java = MyModelModel.from_xsjson( _read_java_file_as_bytes("serializationsample.json")) + # would like to do the below, but cannot because os the ids + # self.assertEqual(from_java, self.model) + self.assertIsNotNone(from_java) + + +class JpatestModelInteropTest(unittest.TestCase): + """Round-trip tests for the jpatest model wrapper.""" + + @classmethod + def setUpClass(cls): + from org.ivoa.dm.jpatest.jpatest import ( + ADtype, + AEtype, + Child, + DThing, + JpatestModel, + JpatestRefs, + LChild, + Parent, + Sub, + Point, + ReferredTo1, + ReferredTo2, + ReferredTo3, + ) + ref3 = ReferredTo3(id="jpatest-ReferredTo3_1002", sval="ref in dtype", ival=3) + ref2 = ReferredTo2(id="jpatest-ReferredTo2_1004", sval="lower ref") + ref1 = ReferredTo1(id="jpatest-ReferredTo1_1003", sval="top level ref") + refInSub = ReferredTo3(id="jpatest-ReferredTo3_1005", sval="ref in sub", ival=3) + + parent = Parent( + dval=ADtype( + basestr="base", + dref=ref3, + intatt="intatt", + dvalr=1.1, + dvals="astring", + ), + eval=AEtype( + basestr="basestre_e", + dref=ref3, + intatt="intatt_e", + evalr=1.2, + evals="evals", + ), + rval=ref1, + cval=Child(rval=ref2), + lval=[ + LChild(sval="First", ival=1), + LChild(sval="Second", ival=2), + LChild(sval="Third", ival=3), + ], + tval=DThing(p=Point(x=1.5, y=3.0), dt="thing"), + ) + + # NB no convenient subtype from supertype constructor like java! + sub = Sub(dval=ADtype( + basestr="base", + dref=ref3, + intatt="intatt", + dvalr=1.1, + dvals="astring", + ), + eval=AEtype( + basestr="basestre_e", + dref=ref3, + intatt="intatt_e", + evalr=1.2, + evals="evals", + ), + rval=ref1, + cval=Child(rval=ref2), + lval=[ + LChild(sval="First", ival=1), + LChild(sval="Second", ival=2), + LChild(sval="Third", ival=3), + ], + tval=DThing(p=Point(x=1.5, y=3.0), dt="thing"), + subval="this is a subtype", + slist=[AEtype(evalr= 1.3, evals="subevals", intatt="intatt_sub", basestr="basestre_sub", dref=refInSub)]) + + + cls.model = JpatestModel( + refs=JpatestRefs(refbase=[ref3, ref1, ref2, refInSub]), + parent=[parent,sub], + ) + + def test_json_serialise(self): + json_str = self.model.to_xsjson(pretty_print=True) + _write("jpatest.json", json_str) + + data = json.loads(json_str) + self.assertEqual(data["JpatestModel"]["refs"]["refbase"][0]["jpatest:ReferredTo3"]["ival"], 3) + parent = data["JpatestModel"]["parent"][0]["jpatest:Parent"] + self.assertEqual(parent["dval"]["jpatest:ADtype"]["dref"], "jpatest-ReferredTo3_1002") + self.assertEqual(parent["rval"], "jpatest-ReferredTo1_1003") + self.assertEqual(parent["cval"]["rval"], "jpatest-ReferredTo2_1004") + self.assertEqual([child["sval"] for child in parent["lval"]], ["First", "Second", "Third"]) + self.assertAlmostEqual(parent["tval"]["p"]["x"], 1.5) + + def test_json_round_trip(self): + recovered = JpatestModel.model_validate_json(self.model.model_dump_json()) + parent = recovered.parent[0] + self.assertEqual(parent.dval.intatt, "intatt") + self.assertEqual(parent.eval.intatt, "intatt_e") + self.assertEqual(parent.tval.p.x, 1.5) + self.assertEqual(parent.cval.rval.id, "jpatest-ReferredTo2_1004") + + def test_xml_serialise(self): + xml_bytes = self.model.full_model_to_xml(pretty_print=True) + _write("jpatest.xml", xml_bytes) + _validate_xml(xml_bytes, "jpatest.vo-dml.xsd", self) + root = ET.fromstring(xml_bytes) + self.assertEqual(_local_name(root.tag), "jpatestModel") + parent = _find_first(root, "parent") + self.assertIsNotNone(parent) + self.assertEqual(_first_child_text(parent, "rval"), "jpatest-ReferredTo1_1003") + dval = _find_first(parent, "dval") + self.assertEqual(_first_child_text(dval, "dvals"), "astring") + lval = _children_named(parent, "lval") + self.assertEqual(len(lval), 1) + lchildren = _children_named(lval[0], "lChild") + self.assertEqual([_first_child_text(child, "sval") for child in lchildren], ["First", "Second", "Third"]) + + def test_xml_round_trip(self): + recovered = JpatestModel.from_xml(self.model.to_xml(pretty_print=True)) + parent = recovered.parent[0] + self.assertEqual(parent.dval.basestr, "base") + self.assertEqual(parent.eval.evals, "evals") + self.assertEqual(parent.lval[1].ival, 2) + self.assertEqual(parent.tval.dt, "thing") + + def test_read_java_serialization_xml(self): + from_java = JpatestModel.from_xml(_read_java_file_as_bytes("jpatest.xml")) + # would like to do the below, but cannot because os the ids + # self.assertEqual(from_java, self.model) + self.assertIsNotNone(from_java) + + + def test_read_java_serialization_json(self): + from_java = JpatestModel.from_xsjson( _read_java_file_as_bytes("jpatest.json")) + # would like to do the below, but cannot because os the ids + # self.assertEqual(from_java, self.model) + self.assertIsNotNone(from_java) + + +class PythonNonModelReadTest(unittest.TestCase): + """Validate the Python-written interoperability files without using the generated models.""" + + def test_sample_json_source_catalogue(self): + data = _read_json("sample.json") + self.assertIn("sourceCatalogue", data["SampleModel"]) + self.assertIn("photometricSystem", data["SampleModel"]) + + catalogue = data["SampleModel"]["sourceCatalogue"][0] + self.assertEqual(catalogue["name"], "testCat") + entry = catalogue["entry"][0]["sample:catalog.SDSSSource"] + self.assertEqual(entry["name"], "testSource") + self.assertEqual(entry["classification"], "AGN") + self.assertEqual(entry["position"]["frame"], "J2000") + + def test_sample_json_photometric_system(self): + data = _read_json("sample.json") + ps = data["SampleModel"]["photometricSystem"][0] + self.assertEqual(ps["detectorType"], 1) + names = [item["name"] for item in ps["photometryFilter"]] + self.assertEqual(names, ["C-Band", "L-Band"]) + + def test_sample_xml_source_catalogue(self): + root = _read_xml_root("sample.xml") + catalogue = _find_first(root, "catalog.inner.SourceCatalogue") # note name includes package parts.... + self.assertIsNotNone(catalogue) + self.assertEqual(_first_child_text(catalogue, "name"), "testCat") + entry = _find_first(catalogue, "entry") + self.assertEqual(_first_child_text(entry, "name"), "testSource") + self.assertEqual(_first_child_text(entry, "classification"), "AGN") + position = _find_first(entry, "position") + self.assertEqual(_first_child_text(position, "frame"), "J2000") #note that this should be a reference tpo the frame not a frame instance... + + def test_lifecycle_json_atest2(self): + data = _read_json("lifecycle.json") + atest2 = data["LifecycleTestModel"]["aTest2"][0] + self.assertEqual(atest2["refagg"], ["lifecycleTest-ReferredTo_1000","lifecycleTest-ReferredTo_1001"]) + self.assertEqual(len(atest2["atest"]["contained"]), 2) + self.assertEqual(atest2["atest"]["contained2"]["lowr"], "lifecycleTest-ReferredLifeCycle_1002") + + def test_lifecycle_xml_atest2(self): + root = _read_xml_root("lifecycle.xml") + atest2 = _find_first(root, "aTest2") + self.assertIsNotNone(atest2) + self.assertEqual(_first_child_text(atest2, "refcont"), "lifecycleTest-ReferredLifeCycle_1002") + contained_values = [el.text for el in root.iter() if _local_name(el.tag) == "test2"] + self.assertEqual(contained_values, ["firstcontained", "secondContained"]) + + def test_serializationsample_json_somecontent(self): + data = _read_json("serializationsample.json") + content = data["MyModelModel"]["someContent"][0] + self.assertEqual(content["zvals"], ["some", "z", "values"]) + self.assertEqual(content["ref1"], "MyModel-Refa_1000") + self.assertEqual(content["ref2"], "naturalkey") + self.assertEqual(len(content["con"]), 2) + + def test_serializationsample_xml_somecontent(self): + root = _read_xml_root("serializationsample.xml") + some_content = _find_first(root, "someContent") + self.assertIsNotNone(some_content) + self.assertEqual(_first_child_text(some_content, "ref1"), "MyModel-Refa_1000") + zvals = [el.text for el in root.iter() if _local_name(el.tag) == "zval"] + self.assertEqual(zvals, ["some", "z", "values"]) + + def test_jpatest_json_parent(self): + data = _read_json("jpatest.json") + parent = data["JpatestModel"]["parent"][0]["jpatest:Parent"] + self.assertEqual(parent["dval"]["jpatest:ADtype"]["dvals"], "astring") + self.assertEqual(parent["eval"]["jpatest:AEtype"]["evals"], "evals") + self.assertEqual(parent["cval"]["rval"], "jpatest-ReferredTo2_1004") + self.assertEqual(parent["tval"]["dt"], "thing") + + def test_jpatest_xml_parent(self): + root = _read_xml_root("jpatest.xml") + parent = _find_first(root, "parent") + self.assertIsNotNone(parent) + self.assertEqual(_first_child_text(parent, "rval"), "jpatest-ReferredTo1_1003") + self.assertEqual(_first_child_text(_find_first(parent, "tval"), "dt"), "thing") + lvals = _children_named(parent, "lval") + self.assertEqual(len(lvals), 1) + lchildren = _children_named(lvals[0], "lChild") + self.assertEqual([_first_child_text(child, "sval") for child in lchildren], ["First", "Second", "Third"]) + + +class PythonModelReadJavaTest(unittest.TestCase): + """Sanity-check that the Java interoperability fixtures exist.""" + + def test_java_interop_files_exist(self): + expected = [ + "sample.json", + "sample.xml", + "lifecycle.json", + "lifecycle.xml", + "serializationsample.json", + "serializationsample.xml", + "jpatest.json", + "jpatest.xml", + ] + missing = [f for f in expected if not (_JAVA_DIR / f).exists()] + self.assertFalse( + missing, + f"Java interop files not found: {missing} (run :sample:test first)", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/gradletooling/sample/pythontest/src/SourceCatalogueTest.py b/tools/gradletooling/sample/pythontest/src/SourceCatalogueTest.py index 8c002d5b..f9df8945 100644 --- a/tools/gradletooling/sample/pythontest/src/SourceCatalogueTest.py +++ b/tools/gradletooling/sample/pythontest/src/SourceCatalogueTest.py @@ -28,7 +28,7 @@ def setUpClass(cls): jansky = Unit("Jy") degree = Unit("degree") GHz = Unit("GHz") - frame = SkyCoordinateFrame(name="J2000", equinox="J2000.0", documentURI=anyURI("http://coord.net")) + frame = SkyCoordinateFrame(name="J2000", equinox="J2000.0", documentURI="http://coord.net") ellipseError = AlignedEllipse(longError=.2, latError=.1) # sdss = SDSSSource(positionError=ellipseError)# UNUSED, but just checking position error subsetting. diff --git a/tools/gradletooling/sample/pythontest/src/vodml_runtime b/tools/gradletooling/sample/pythontest/src/vodml_runtime new file mode 120000 index 00000000..8fc4d971 --- /dev/null +++ b/tools/gradletooling/sample/pythontest/src/vodml_runtime @@ -0,0 +1 @@ +../../../../../runtime/python/vodml_runtime \ No newline at end of file diff --git a/tools/gradletooling/sample/src/test/java/org/ivoa/dm/lifecycle/LifecycleTestModelTest.java b/tools/gradletooling/sample/src/test/java/org/ivoa/dm/lifecycle/LifecycleTestModelTest.java index 6fb80a87..ddd161f5 100644 --- a/tools/gradletooling/sample/src/test/java/org/ivoa/dm/lifecycle/LifecycleTestModelTest.java +++ b/tools/gradletooling/sample/src/test/java/org/ivoa/dm/lifecycle/LifecycleTestModelTest.java @@ -61,6 +61,7 @@ protected String setSerializationDumpPrefix() { @Override public LifecycleTestModel createModel() { final ReferredTo referredTo = new ReferredTo(3); + final ReferredTo referredTo2 = new ReferredTo(4); List refcont = Arrays.asList(new ReferredLifeCycle("rc1"), new ReferredLifeCycle("rc2")); List contained = @@ -74,7 +75,7 @@ public LifecycleTestModel createModel() { a.refandcontained = refcont; a.contained2 = new ATest4( refcont.get(0)); }); - atest2 = new ATest2( Arrays.asList(referredTo), atest, refcont.get(0)); + atest2 = new ATest2( Arrays.asList(referredTo, referredTo2), atest, refcont.get(0)); LifecycleTestModel model = new LifecycleTestModel(); // model.addContent(atest); diff --git a/tools/gradletooling/sample/src/test/java/org/ivoa/dm/serializationsample/SerializationExampleTest.java b/tools/gradletooling/sample/src/test/java/org/ivoa/dm/serializationsample/SerializationExampleTest.java index 8f97fd36..6b1aad01 100644 --- a/tools/gradletooling/sample/src/test/java/org/ivoa/dm/serializationsample/SerializationExampleTest.java +++ b/tools/gradletooling/sample/src/test/java/org/ivoa/dm/serializationsample/SerializationExampleTest.java @@ -37,7 +37,7 @@ public MyModelModel createModel() { List clist = List.of(new Dcont("N1", "dval"), new Econt("cube", "eval")); - someContent = new SomeContent(refa, refb, List.of("some","z","values"), clist); + someContent = new SomeContent(refa, refb, List.of("some","z","values"), clist, "urn:uri"); themodel.addContent(someContent); return themodel; diff --git a/tools/xslt/common-binding.xsl b/tools/xslt/common-binding.xsl index e938b26b..31c0cae5 100644 --- a/tools/xslt/common-binding.xsl +++ b/tools/xslt/common-binding.xsl @@ -169,7 +169,7 @@ - + @@ -603,12 +603,16 @@ Long - + + + + + diff --git a/tools/xslt/jaxb.xsl b/tools/xslt/jaxb.xsl index 5f8231fa..355ac167 100644 --- a/tools/xslt/jaxb.xsl +++ b/tools/xslt/jaxb.xsl @@ -102,7 +102,7 @@ - + @jakarta.xml.bind.annotation.XmlAttribute(name = "", required =) @@ -124,7 +124,7 @@ - + cannot map to attribute with multiplicity > 1 @@ -141,6 +141,7 @@ @jakarta.xml.bind.annotation.XmlElementWrapper( name = "") @jakarta.xml.bind.annotation.XmlElement( name = "", required = , type = .class) + @com.fasterxml.jackson.annotation.JsonProperty("") @@ -150,7 +151,7 @@ - + @@ -158,10 +159,23 @@ @jakarta.xml.bind.annotation.XmlIDREF - - - @jakarta.xml.bind.annotation.XmlElement( name = "", required = , type = Reference.class) - + + + + + + + + @jakarta.xml.bind.annotation.XmlElement( name = "", required = , type = .class) + + + @jakarta.xml.bind.annotation.XmlElementWrapper( name = "") + @jakarta.xml.bind.annotation.XmlElement( name = "", required = , type = .class) + + + @jakarta.xml.bind.annotation.XmlIDREF + + @@ -281,7 +295,7 @@ ref= @XmlElement(name="") - @JsonProperty("") + @JsonProperty("") @com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver(value = org.ivoa.vodml.json.VodmlTypeResolver.class) @@ -370,7 +384,7 @@ public (@JsonProperty("refs") References refs_, - @JsonProperty("") List<> _ + @JsonProperty("") List<> _ , ){ @@ -451,7 +465,7 @@ } } - @JsonProperty("") + @JsonProperty("")//IMPL there is a chance of a name clash - would probably be better to use utype public () { return getContent(); diff --git a/tools/xslt/vo-dml2jsonschema.xsl b/tools/xslt/vo-dml2jsonschema.xsl index 663eb19c..473be0df 100644 --- a/tools/xslt/vo-dml2jsonschema.xsl +++ b/tools/xslt/vo-dml2jsonschema.xsl @@ -107,7 +107,7 @@ TODO subsetting not specifically checked ,"properties" : { "$comment" : "placeholder to make commas easier!" - ,"" : { + ,"" : { "type": "array" ,"items" : { "anyOf" : [ @@ -129,7 +129,7 @@ TODO subsetting not specifically checked - ,"" : { + ,"" : { "type" : "array" ,"items" : { @@ -283,8 +283,15 @@ TODO subsetting not specifically checked - - + + + + + + + + + @@ -337,7 +344,14 @@ TODO subsetting not specifically checked + + , "" : { + + + , "" : { + + "type":"array" ,"items": { diff --git a/tools/xslt/vo-dml2pydantic.xsl b/tools/xslt/vo-dml2pydantic.xsl new file mode 100644 index 00000000..53bd3b1e --- /dev/null +++ b/tools/xslt/vo-dml2pydantic.xsl @@ -0,0 +1,624 @@ + + + +"> + "> +]> + + + + + + + + + + + + + + + + + + + + " + + ' + + + + + + + Generating Pydantic - considering models + + + + + + +------------------------------------------------------------------------------------------------------- +-- Generating Pydantic code for model []. +-- last modification date of the model +------------------------------------------------------------------------------------------------------- + + + + + There is no binding for model + + + + + + + + + + + + + + + + + + + + + + + + package = + + + + +from __future__ import annotations +from typing import Optional, List, Any, Union, ClassVar +from enum import Enum +from xsdata_pydantic.fields import field as xsfield +from vodml_runtime.VodmlXmlBase import _VodmlXmlBase + +import xsdata_pydantic.hooks.class_type # register pydantic support with xsdata + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Writing pydantic code for base= haschildren= + + + + 1) Mapped type for = '' + + + + + + + + + + + + + + +class ( + + _VodmlXmlBase + ): + class Meta: + name = "" + namespace = "" + + + + utype = "" + + + + + + key = [] + + + + + + """ + * + * + * : + * + * + """ + + + + id: Union[str|int] = xsfield({'type': 'Attribute', 'name': '_id'}, default=None) # surrogate identifier for XML IDREF resolution (allowing str or int for now) + + + + id: Optional[str|int] = xsfield({'type': 'Attribute', 'name': '_id'}, default=None) # surrogate identifier for DB ID (allowing str or int for now) + + + + + + + + + + + pass + + + + + + + + + + + + + + +class ( + + _VodmlXmlBase + ): + class Meta: + name = "" + namespace = "" + + + + utype = "" + + + + + """ + * + * + * : + * + * + """ + + + _vodml_refs: ClassVar[list[str]] = [, ""] + + + + + + + + + + + pass + + + + + + + + + + +class + + """ + * + * + * Enumeration : + * + * + """ + + +&cr; + + + + + + + + + + + + + Primitive type represented as str + + + + + + + +class (_VodmlXmlBase): + class Meta: + name = "" + """ + * + * PrimitiveType : + * + * + """ + + value: = xsfield({'type': 'Text'}) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + Attribute : multiplicity + + + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + * + * Composition : ( Multiplicity : ) + + * + """ + + + + + + + + + + + + + + + + + + + + + """ + * Composition : ( Multiplicity : ) + * + * + """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """ + * Reference : + * + * ( Multiplicity : ) """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +class (_VodmlXmlBase): + class Meta: + name = "refs" + + + + + : List[] = xsfield({'type': 'Element', 'name': '', 'namespace': ''}, default_factory=list) + + + + pass + + + +class (_VodmlXmlBase): + class Meta: + name = "" + namespace = "" + utype = "" + refs: Optional[] = xsfield({'type': 'Element', 'name': 'refs', 'namespace': ''}, default=None) + + + + + + : List[] = xsfield({'type': 'Element', 'name': '', 'namespace': ''}, default_factory=list) + + + + def full_model_to_xml(self, pretty_print: bool = False) -> bytes: + """Serialise the whole model to XML, converting any object references to IDREF strings. with namespace declarations on the root element.""" + return self.to_xml(, pretty_print=pretty_print) + + + + + @classmethod + def from_xml(cls, xml_bytes: bytes): + """Deserialise from XML, resolving IDREF strings to object instances.""" + instance = super().from_xml(xml_bytes) + from vodml_runtime.references import resolve_references + resolve_references(instance) + return instance + + + + + + + + + + + + + + = "" + """ + * Value : + * + * + """ + + + + + + + + + + + + + + + + + + + Writing package info file + +""" + + + +""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/xslt/vo-dml2xsdNew.xsl b/tools/xslt/vo-dml2xsdNew.xsl index 8f8e94c7..d6950011 100644 --- a/tools/xslt/vo-dml2xsdNew.xsl +++ b/tools/xslt/vo-dml2xsdNew.xsl @@ -211,9 +211,9 @@ note that this schema is substantially different from the era when this code was - + - + @@ -256,9 +256,9 @@ note that this schema is substantially different from the era when this code was - + - + @@ -317,7 +317,7 @@ note that this schema is substantially different from the era when this code was - + @@ -342,7 +342,7 @@ note that this schema is substantially different from the era when this code was - + @@ -461,7 +461,7 @@ note that this schema is substantially different from the era when this code was - + this is a reference @@ -475,6 +475,25 @@ note that this schema is substantially different from the era when this code was + + this is a reference + + + + + + + + + + + xsd:IDREF + + + + + +