add support for CWL ingest

Author: kalxas

registration_api/parsers/__init__.py

@@ -0,0 +1,20 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################

registration_api/parsers/cwl.py

@@ -0,0 +1,58 @@
+###############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+###############################################################################
+
+import logging
+
+from pygeometa.core import import_metadata
+import yaml
+
+LOGGER = logging.getLogger(__name__)
+
+
+def from_cwl(cwl: dict, url: str = None,
+             parent_identifier: str = None) -> dict:
+    """
+    Transforms a CWL object to pygeometa MCF model
+
+    :param cwl: `dict` of CWL object
+    :param url: optional `str` of URL
+    :param parent_identifier: optional `str` of parent identifier
+
+    :returns: `dict` of MCF object
+    """
+
+    mcf = import_metadata('cwl', yaml.safe_dump(cwl))
+
+    if parent_identifier is not None:
+        mcf['metadata']['parentidentifier'] = parent_identifier
+
+    if url is not None:
+        mcf['distribution']['http'] = {
+            'rel': 'canonical',
+            'url': url,
+            'type': 'application/x-yaml',
+            'name': mcf['identification']['title'],
+            'description': mcf['identification']['abstract']
+        }
+
+    LOGGER.info(f'MCF: {mcf}')
+
+    return mcf

registration_api/processes/registration.py

@@ -26,9 +26,13 @@
 
 from jsonschema.validators import Draft202012Validator
 from owslib.ogcapi.records import Records
+from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError
+from pygeometa.schemas.ogcapi_records import OGCAPIRecordOutputSchema
 import requests
+import yaml
+
+from registration_api.parsers.cwl import from_cwl
 
-from pygeoapi.process.base import BaseProcessor, ProcessorExecuteError
 
 LOGGER = logging.getLogger(__name__)
 
@@ -248,6 +252,7 @@ def execute(self, data: dict, outputs: dict = None) -> tuple:
 
         self.auth = None
         mimetype = 'application/json'
+        resource_mimetype = 'application/geo+json'
 
         LOGGER.debug('Validating request payload against schema')
         validation_errors = validate_json(REGISTER_SCHEMA, data)
@@ -266,7 +271,13 @@ def execute(self, data: dict, outputs: dict = None) -> tuple:
 
         if 'href' in source:
             LOGGER.debug('Source is a URL')
-            content = requests.get(source['href']).json()
+            try:
+                content = requests.get(source['href']).json()
+            except json.decoder.JSONDecodeError as err:
+                LOGGER.debug(f'Error parsing JSON: {err}')
+                LOGGER.debug('Attempting YAML parsing')
+                content = yaml.safe_load(requests.get(source['href']).text)
+
             LOGGER.debug('Performing source validation')
             validation_errors = validate_json(
                 REGISTER_SCHEMA['properties']['source']['properties']['oneOf'][0]['content'],  # noqa
@@ -282,6 +293,12 @@ def execute(self, data: dict, outputs: dict = None) -> tuple:
             LOGGER.error(f'{msg}: {content}')
             raise ProcessorExecuteError(msg)
 
+        if 'cwlVersion' in content:
+            LOGGER.debug('Parsing CWL')
+            content = from_cwl(content, url=source.get('href'))
+            content = OGCAPIRecordOutputSchema().write(
+                content, stringify=False)
+
         for stac_extension in content.get('stac_extensions', []):
             LOGGER.debug(f'Validating against STAC Extension {stac_extension}')
             stac_extension_schema = requests.get(stac_extension)
@@ -373,12 +390,13 @@ def execute(self, data: dict, outputs: dict = None) -> tuple:
                 url = f"{target['href']}/collections/{collection}/items/{id_}"
             elif source['rel'] == 'collection':
                 url = f"{target['href']}/collections/{collection}"
+                resource_mimetype = 'application/json'
             produced_outputs = {
                 'id': PROCESS_REGISTER_METADATA['id'],
                 'resource-and-data-catalogue-link': {
                     'href': url,
                     'rel': 'item',
-                    'type': 'application/geo+json'
+                    'type': resource_mimetype
                 }
             }
 
@@ -504,6 +522,8 @@ def validate_json(schema: dict, instance: dict) -> list:
 
     validation_errors = []
     LOGGER.debug('Validating input against schema')
+    LOGGER.debug(f'Input: {instance}')
+    LOGGER.debug(f'Schema: {schema}')
     validator = Draft202012Validator(schema)
 
     for error in validator.iter_errors(instance):

requirements.txt

@@ -1,5 +1,7 @@
 gunicorn==23.0.0
 jsonschema==4.25.1
 pygeoapi==0.21.0
+pygeometa==0.18.0
 OWSLib==0.34.1
 requests==2.32.5
+PyYAML==6.0.3

schemas/process-register.json

@@ -41,6 +41,9 @@
                                 },
                                 {
                                     "$ref": "https://raw.githubusercontent.com/EOEPCA/metadata-profile/refs/heads/master/schemas/resource.json"
+                                },
+                                {
+                                    "$ref": "https://gist.githubusercontent.com/tomkralidis/a6713e2bc2f9fef5f39b52e0893674bc/raw/cf23172f02df807f717a267390d09f9efa32919f/cwl-1.2.json"
                                 }
                             ]
                         },