diff --git a/rocrate/model/metadata.py b/rocrate/model/metadata.py index 94d026a8..28f94225 100644 --- a/rocrate/model/metadata.py +++ b/rocrate/model/metadata.py @@ -44,7 +44,34 @@ LEGACY_BASENAME = "ro-crate-metadata.jsonld" DETACHED_MD_NAME = re.compile(r".*-ro-crate-metadata.json$") -WORKFLOW_PROFILE = "https://w3id.org/workflowhub/workflow-ro-crate/1.0" +# Mapping from RO-Crate version to Workflow RO-Crate version +WROC_VERSION_MAP = { + "1.0": "1.0", + "1.0-DRAFT": "1.0", + "1.1": "1.0", + "1.1-DRAFT": "1.0", + "1.2": None, + "1.2-DRAFT": None, + "1.3": "1.1", + "1.3-DRAFT": "1.1", +} +DEFAULT_WROC_VERSION = "1.1" +WROC_PROFILE_BASE = "https://w3id.org/workflowhub/workflow-ro-crate" + +# Mapping from RO-Crate version to Workflow Testing RO-Crate version +WTROC_VERSION_MAP = { + "1.0": "0.1", + "1.0-DRAFT": "0.1", + "1.1": "0.1", + "1.1-DRAFT": "0.1", + "1.2": None, + "1.2-DRAFT": None, + "1.3": "0.2", + "1.3-DRAFT": "0.2", +} +DEFAULT_WTROC_VERSION = "0.2" +WTROC_PROFILE_BASE = "https://w3id.org/ro/wftest" +WTROC_CONTEXT = "https://w3id.org/ro/terms/test" class Metadata(File): @@ -112,21 +139,3 @@ def write_detached(self, path): @property def root(self) -> Dataset: return self.crate.root_dataset - - -# https://github.com/ResearchObject/ro-terms/tree/master/test -TESTING_EXTRA_TERMS = { - "TestSuite": "https://w3id.org/ro/terms/test#TestSuite", - "TestInstance": "https://w3id.org/ro/terms/test#TestInstance", - "TestService": "https://w3id.org/ro/terms/test#TestService", - "TestDefinition": "https://w3id.org/ro/terms/test#TestDefinition", - "PlanemoEngine": "https://w3id.org/ro/terms/test#PlanemoEngine", - "JenkinsService": "https://w3id.org/ro/terms/test#JenkinsService", - "TravisService": "https://w3id.org/ro/terms/test#TravisService", - "GithubService": "https://w3id.org/ro/terms/test#GithubService", - "instance": "https://w3id.org/ro/terms/test#instance", - "runsOn": "https://w3id.org/ro/terms/test#runsOn", - "resource": "https://w3id.org/ro/terms/test#resource", - "definition": "https://w3id.org/ro/terms/test#definition", - "engineVersion": "https://w3id.org/ro/terms/test#engineVersion" -} diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index 2944bd5d..3cc17d2c 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -57,7 +57,18 @@ TestSuite, WorkflowDescription, ) -from .model.metadata import WORKFLOW_PROFILE, TESTING_EXTRA_TERMS, DEFAULT_VERSION, BASENAME, LEGACY_BASENAME +from .model.metadata import ( + WROC_VERSION_MAP, + DEFAULT_WROC_VERSION, + WROC_PROFILE_BASE, + WTROC_VERSION_MAP, + DEFAULT_WTROC_VERSION, + WTROC_PROFILE_BASE, + WTROC_CONTEXT, + DEFAULT_VERSION, + BASENAME, + LEGACY_BASENAME, +) from .model.computationalworkflow import galaxy_to_abstract_cwl from .model.computerlanguage import get_lang from .model.testservice import get_service @@ -682,6 +693,25 @@ def _all_streams(self, chunk_size=8192): for writeable_entity in self.data_entities + self.default_entities: yield from writeable_entity.stream(chunk_size=chunk_size) + def _add_wroc_profile(self): + wroc_version = WROC_VERSION_MAP.get(self.version) + if wroc_version is None: + warnings.warn(f"RO-Crate version '{self.version}' is not compatible with Workflow RO-Crate") + wroc_version = DEFAULT_WROC_VERSION + profile_id = f"{WROC_PROFILE_BASE}/{wroc_version}" + if wroc_version == "1.0": + profiles = set(_.rstrip("/") for _ in get_norm_value(self.metadata, "conformsTo")) + profiles.add(profile_id) + self.metadata["conformsTo"] = [{"@id": _} for _ in sorted(profiles)] + else: + if profile_id not in self: + profile_entity = self.add(ContextEntity(self, profile_id, properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Workflow RO-Crate", + "version": wroc_version, + })) + self.root_dataset.append_to("conformsTo", profile_entity) + def add_workflow( self, source=None, dest_path=None, fetch_remote=False, validate_url=False, properties=None, main=False, lang="cwl", lang_version=None, gen_cwl=False, cls=ComputationalWorkflow, @@ -701,9 +731,7 @@ def add_workflow( workflow.lang = lang if main: self.mainEntity = workflow - profiles = set(_.rstrip("/") for _ in get_norm_value(self.metadata, "conformsTo")) - profiles.add(WORKFLOW_PROFILE) - self.metadata["conformsTo"] = [{"@id": _} for _ in sorted(profiles)] + self._add_wroc_profile() if gen_cwl and lang_str != "cwl": if lang_str != "galaxy": raise ValueError(f"conversion from {lang.name} to abstract CWL not supported") @@ -716,6 +744,25 @@ def add_workflow( workflow.subjectOf = cwl_workflow return workflow + def _add_wtroc_profile(self): + wtroc_version = WTROC_VERSION_MAP.get(self.version) + if wtroc_version is None: + warnings.warn(f"RO-Crate version '{self.version}' is not compatible with Workflow Testing RO-Crate") + wtroc_version = DEFAULT_WTROC_VERSION + profile_id = f"{WTROC_PROFILE_BASE}/{wtroc_version}" + if wtroc_version == "0.1": + pass + else: + if profile_id not in self: + profile_entity = self.add(ContextEntity(self, profile_id, properties={ + "@type": ["CreativeWork", "Profile"], + "name": "Workflow Testing RO-Crate", + "version": wtroc_version, + })) + self.root_dataset.append_to("conformsTo", profile_entity) + if WTROC_CONTEXT not in self.metadata.extra_contexts: + self.metadata.extra_contexts.append(WTROC_CONTEXT) + def add_test_suite(self, identifier=None, name=None, main_entity=None, properties=None): test_ref_prop = "mentions" if not main_entity: @@ -728,7 +775,7 @@ def add_test_suite(self, identifier=None, name=None, main_entity=None, propertie if main_entity: suite["mainEntity"] = main_entity self.root_dataset.append_to(test_ref_prop, suite) - self.metadata.extra_terms.update(TESTING_EXTRA_TERMS) + self._add_wtroc_profile() return suite def add_test_instance(self, suite, url, resource="", service="jenkins", identifier=None, name=None, properties=None): @@ -745,7 +792,7 @@ def add_test_instance(self, suite, url, resource="", service="jenkins", identifi if not properties or "name" not in properties: instance.name = name or instance.id.lstrip("#") suite.append_to("instance", instance) - self.metadata.extra_terms.update(TESTING_EXTRA_TERMS) + self._add_wtroc_profile() return instance def add_test_definition( @@ -766,7 +813,7 @@ def add_test_definition( if engine_version is not None: definition.engineVersion = engine_version suite.definition = definition - self.metadata.extra_terms.update(TESTING_EXTRA_TERMS) + self._add_wtroc_profile() return definition def add_action(self, instrument, identifier=None, object=None, result=None, properties=None): @@ -967,8 +1014,9 @@ def stream(self, chunk_size=8192): def make_workflow_rocrate(workflow_path, wf_type, include_files=[], - fetch_remote=False, cwl=None, diagram=None): - wf_crate = ROCrate() + fetch_remote=False, cwl=None, diagram=None, + version=DEFAULT_VERSION): + wf_crate = ROCrate(version=version) workflow_path = Path(workflow_path) wf_crate.add_workflow( workflow_path, workflow_path.name, fetch_remote=fetch_remote, diff --git a/test/conftest.py b/test/conftest.py index ac9d02cc..45d90802 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -35,11 +35,12 @@ BASE_URL = 'https://w3id.org/ro/crate' DEFAULT_VERSION = '1.3' LEGACY_VERSION = '1.0' +DEFAULT_WROC_VERSION = "1.1" class Helpers: - WORKFLOW_PROFILE = "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + WROC_PROFILE_BASE = "https://w3id.org/workflowhub/workflow-ro-crate" METADATA_FILE_NAME = 'ro-crate-metadata.json' LEGACY_METADATA_FILE_NAME = 'ro-crate-metadata.jsonld' WORKFLOW_TYPES = {"File", "SoftwareSourceCode", "ComputationalWorkflow"} @@ -74,8 +75,11 @@ def check_crate(cls, json_entities, root_id="./", data_entity_ids=None, version= assert data_entity_ids.issubset([_["@id"] for _ in root["hasPart"]]) @classmethod - def check_wf_crate(cls, json_entities, wf_file_name, root_id="./"): - cls.check_crate(json_entities, root_id=root_id) + def check_wf_crate(cls, json_entities, wf_file_name, root_id="./", wroc_version=DEFAULT_WROC_VERSION): + if wroc_version == "1.0": + cls.check_crate(json_entities, root_id=root_id, version="1.1") + else: + cls.check_crate(json_entities, root_id=root_id) assert json_entities[root_id]["mainEntity"]["@id"] == wf_file_name assert wf_file_name in json_entities wf_entity = json_entities[wf_file_name] @@ -83,7 +87,11 @@ def check_wf_crate(cls, json_entities, wf_file_name, root_id="./"): assert cls.WORKFLOW_TYPES.issubset(wf_entity["@type"]) assert "programmingLanguage" in wf_entity metadata = json_entities[cls.METADATA_FILE_NAME] - assert cls.WORKFLOW_PROFILE in get_norm_value(metadata, "conformsTo") + profile_id = f"{cls.WROC_PROFILE_BASE}/{wroc_version}" + if wroc_version == "1.0": + assert profile_id in get_norm_value(metadata, "conformsTo") + else: + assert profile_id in get_norm_value(json_entities[root_id], "conformsTo") @pytest.fixture diff --git a/test/test-data/crate_with_subcrates/ro-crate-metadata.json b/test/test-data/crate_with_subcrates/ro-crate-metadata.json index 0cd761f3..367b3fa7 100644 --- a/test/test-data/crate_with_subcrates/ro-crate-metadata.json +++ b/test/test-data/crate_with_subcrates/ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.1/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "./", @@ -27,7 +27,7 @@ "@id": "./" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.1" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/crate_with_subcrates/subcrate/ro-crate-metadata.json b/test/test-data/crate_with_subcrates/subcrate/ro-crate-metadata.json index eca9026b..657929d8 100644 --- a/test/test-data/crate_with_subcrates/subcrate/ro-crate-metadata.json +++ b/test/test-data/crate_with_subcrates/subcrate/ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.1/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "./", @@ -21,7 +21,7 @@ "@id": "./" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.1" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/crate_with_subcrates/subcrate/subsubcrate/ro-crate-metadata.json b/test/test-data/crate_with_subcrates/subcrate/subsubcrate/ro-crate-metadata.json index b552c795..dc40cf6b 100644 --- a/test/test-data/crate_with_subcrates/subcrate/subsubcrate/ro-crate-metadata.json +++ b/test/test-data/crate_with_subcrates/subcrate/subsubcrate/ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.1/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "./", @@ -18,7 +18,7 @@ "@id": "./" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.1" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/crate_with_subcrates/subcrate2/ro-crate-metadata.json b/test/test-data/crate_with_subcrates/subcrate2/ro-crate-metadata.json index 817ffe58..70ac1909 100644 --- a/test/test-data/crate_with_subcrates/subcrate2/ro-crate-metadata.json +++ b/test/test-data/crate_with_subcrates/subcrate2/ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.1/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "./", @@ -18,7 +18,7 @@ "@id": "./" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.1" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/detached-ro-crate-metadata.json b/test/test-data/detached-ro-crate-metadata.json index 63aefb96..a858285a 100644 --- a/test/test-data/detached-ro-crate-metadata.json +++ b/test/test-data/detached-ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.2/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "ro-crate-metadata.json", @@ -8,7 +8,7 @@ "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.2" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json b/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json index ae514025..dcda3092 100644 --- a/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json +++ b/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.2/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached_crate_with_subcrates/", @@ -24,7 +24,7 @@ "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached_crate_with_subcrates/" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.2" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/read_crate/ro-crate-metadata.json b/test/test-data/read_crate/ro-crate-metadata.json index f047760d..46d3fe11 100644 --- a/test/test-data/read_crate/ro-crate-metadata.json +++ b/test/test-data/read_crate/ro-crate-metadata.json @@ -14,6 +14,9 @@ { "@id": "./", "@type": "Dataset", + "conformsTo": { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.1" + }, "datePublished": "2020-06-25 17:03:04.098286", "hasPart": [ { @@ -45,6 +48,15 @@ "@id": "test_galaxy_wf.ga" } }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.1", + "@type": [ + "CreativeWork", + "Profile" + ], + "name": "Workflow RO-Crate", + "version": "1.1" + }, { "@id": "ro-crate-preview.html", "@type": "CreativeWork", diff --git a/test/test-data/read_extra/ro-crate-metadata.json b/test/test-data/read_extra/ro-crate-metadata.json index 62b88242..783923fe 100644 --- a/test/test-data/read_extra/ro-crate-metadata.json +++ b/test/test-data/read_extra/ro-crate-metadata.json @@ -1,5 +1,5 @@ { - "@context": "https://w3id.org/ro/crate/1.2/context", + "@context": "https://w3id.org/ro/crate/1.3/context", "@graph": [ { "@id": "./", @@ -24,7 +24,7 @@ "@id": "./" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.2" + "@id": "https://w3id.org/ro/crate/1.3" } }, { diff --git a/test/test-data/ro-crate-galaxy-sortchangecase/ro-crate-metadata.json b/test/test-data/ro-crate-galaxy-sortchangecase/ro-crate-metadata.json index aeddd770..a8c3ec74 100644 --- a/test/test-data/ro-crate-galaxy-sortchangecase/ro-crate-metadata.json +++ b/test/test-data/ro-crate-galaxy-sortchangecase/ro-crate-metadata.json @@ -1,21 +1,7 @@ { "@context": [ - "https://w3id.org/ro/crate/1.2/context", - { - "TestSuite": "https://w3id.org/ro/terms/test#TestSuite", - "TestInstance": "https://w3id.org/ro/terms/test#TestInstance", - "TestService": "https://w3id.org/ro/terms/test#TestService", - "TestDefinition": "https://w3id.org/ro/terms/test#TestDefinition", - "PlanemoEngine": "https://w3id.org/ro/terms/test#PlanemoEngine", - "JenkinsService": "https://w3id.org/ro/terms/test#JenkinsService", - "TravisService": "https://w3id.org/ro/terms/test#TravisService", - "GithubService": "https://w3id.org/ro/terms/test#GithubService", - "instance": "https://w3id.org/ro/terms/test#instance", - "runsOn": "https://w3id.org/ro/terms/test#runsOn", - "resource": "https://w3id.org/ro/terms/test#resource", - "definition": "https://w3id.org/ro/terms/test#definition", - "engineVersion": "https://w3id.org/ro/terms/test#engineVersion" - } + "https://w3id.org/ro/crate/1.3/context", + "https://w3id.org/ro/terms/test" ], "@graph": [ { @@ -25,12 +11,20 @@ "@id": "./" }, "conformsTo": { - "@id": "https://w3id.org/ro/crate/1.2" + "@id": "https://w3id.org/ro/crate/1.3" } }, { "@id": "./", "@type": "Dataset", + "conformsTo": [ + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.1" + }, + { + "@id": "https://w3id.org/ro/wftest/0.2" + } + ], "name": "sort-and-change-case", "description": "sort lines and change text to upper case", "license": "Apache-2.0", @@ -57,6 +51,24 @@ } ] }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.1", + "@type": [ + "CreativeWork", + "Profile" + ], + "name": "Workflow RO-Crate", + "version": "1.1" + }, + { + "@id": "https://w3id.org/ro/wftest/0.2", + "@type": [ + "CreativeWork", + "Profile" + ], + "name": "Workflow Testing RO-Crate", + "version": "0.2" + }, { "@id": "sort-and-change-case.ga", "@type": [ diff --git a/test/test_cli.py b/test/test_cli.py index c94b393e..ef33d341 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -30,8 +30,15 @@ from rocrate.cli import cli from rocrate.model import File -from rocrate.model.metadata import TESTING_EXTRA_TERMS +from rocrate.model.metadata import ( + WROC_PROFILE_BASE, + DEFAULT_WROC_VERSION, + WTROC_CONTEXT, + WTROC_PROFILE_BASE, + DEFAULT_WTROC_VERSION, +) from rocrate.rocrate import ROCrate +from rocrate.utils import get_norm_value def get_command_paths(command): @@ -289,7 +296,7 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd): assert set(json_entities[def_id]["@type"]) == {"File", "TestDefinition"} assert json_entities[def_id]["name"] == "foo" assert json_entities[def_id]["description"] == "foo bar" - # check extra terms + # check extra context metadata_path = crate_dir / helpers.METADATA_FILE_NAME with open(metadata_path, "rt") as f: json_data = json.load(f) @@ -297,8 +304,15 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd): context = json_data["@context"] assert isinstance(context, list) assert len(context) > 1 - extra_terms = context[1] - assert set(TESTING_EXTRA_TERMS.items()).issubset(extra_terms.items()) + extra_context = context[1] + assert extra_context == WTROC_CONTEXT + # check profiles + json_entities = helpers.read_json_entities(crate_dir) + rde = json_entities.get("./") + assert "conformsTo" in rde + profiles = set(get_norm_value(rde, "conformsTo")) + assert f"{WROC_PROFILE_BASE}/{DEFAULT_WROC_VERSION}" in profiles + assert f"{WTROC_PROFILE_BASE}/{DEFAULT_WTROC_VERSION}" in profiles def test_cli_add_test_metadata_explicit_ids(test_data_dir, helpers, monkeypatch): diff --git a/test/test_read.py b/test/test_read.py index 4d2c7ab0..e6902dcb 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -62,7 +62,10 @@ def test_crate_dir_loading(test_data_dir, tmpdir, helpers, gen_preview, from_zip "with%20space.txt", "a%20b/", } - assert set(_["@id"] for _ in crate.contextual_entities) == {"#joe"} + assert set(_["@id"] for _ in crate.contextual_entities) == { + "#joe", + "https://w3id.org/workflowhub/workflow-ro-crate/1.1", + } root = crate.dereference('./') assert crate.root_dataset is root diff --git a/test/test_workflow_ro_crate.py b/test/test_workflow_ro_crate.py index b2b09db2..103d0cf9 100644 --- a/test/test_workflow_ro_crate.py +++ b/test/test_workflow_ro_crate.py @@ -73,10 +73,11 @@ def test_galaxy_wf_crate(test_data_dir, tmpdir, helpers): assert abstract_wf_out_path.exists() -def test_cwl_wf_crate(test_data_dir, tmpdir, helpers): +@pytest.mark.parametrize("crate_version", ["1.1", "1.3"]) +def test_cwl_wf_crate(test_data_dir, tmpdir, helpers, crate_version): wf_id = 'sample_cwl_wf.cwl' wf_path = test_data_dir / wf_id - wf_crate = make_workflow_rocrate(wf_path, wf_type='CWL') + wf_crate = make_workflow_rocrate(wf_path, wf_type='CWL', version=crate_version) assert isinstance(wf_crate, ROCrate) wf = wf_crate.dereference(wf_id) @@ -91,7 +92,8 @@ def test_cwl_wf_crate(test_data_dir, tmpdir, helpers): out_path.mkdir() wf_crate.write(out_path) json_entities = helpers.read_json_entities(out_path) - helpers.check_wf_crate(json_entities, wf_id) + kwargs = {"wroc_version": "1.0"} if crate_version == "1.1" else {} + helpers.check_wf_crate(json_entities, wf_id, **kwargs) wf_out_path = out_path / wf_id assert wf_out_path.exists()