Support for detached crates

README.md

@@ -293,6 +293,95 @@ article = crate.dereference("paper.pdf")
 
 ## Advanced features
 
+### Detached crates
+
+[RO-Crate 1.2](https://www.researchobject.org/ro-crate/whats-changed-in-1-2) introduces the concept of _detached_ RO-Crates, which have no defined root directory: in detached crates, the metadata is accessed independently, for instance via an API or from a standalone metadata file. By contrast, "traditional" crates that describe a payload of files and directories contained in a root directory are called _attached_.
+
+Both detached and attached crates can have a root data entity with an absolute URI as `@id`. To create an RO-Crate whose root data entity `@id` is different from the default `./`, use the `root_dataset_id` argument in the constructor:
+
+```python
+from rocrate.rocrate import ROCrate
+
+url = "http://example.com/crate/"
+crate = ROCrate(root_dataset_id=url)
+```
+
+In detached crates, _all_ data entities must be web-based, i.e., have an absolute URI as `@id`:
+
+```python
+f1 = crate.add_file(f"{url}f1")
+```
+
+The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`:
+
+```python
+crate.write_detached("/tmp/example-ro-crate-metadata.json")
+```
+
+One of the ways to consume a detached crate is to read the metadata from a local file:
+
+```python
+rcrate = ROCrate("/tmp/example-ro-crate-metadata.json")
+rf1 = rcrate.dereference(f"{url}f1")
+```
+
+This also works with a local `file://` URI:
+
+```python
+rcrate = ROCrate("file:///tmp/example-ro-crate-metadata.json")
+```
+
+and with a remote URI:
+
+```python
+base = "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/"
+rcrate = ROCrate(f"{base}detached-ro-crate-metadata.json")
+assert rcrate.root_dataset.id == base
+sample_file = rcrate.dereference(f"{base}sample_file.txt")
+test_file_galaxy = rcrate.dereference(f"{base}test_file_galaxy.txt")
+```
+
+Suppose you now want to save the crate to the local file system. You could use `write_detached` as shown above:
+
+```python
+rcrate.write_detached("/tmp/detached-ro-crate-metadata.json")
+```
+
+but you could also write the crate as attached, after tweaking the data entities a bit:
+
+```python
+sample_file.fetch_remote = True
+sample_file["localPath"] = "sample_file.txt"
+test_file_galaxy.fetch_remote = True
+rcrate.write("/tmp/crate")
+```
+
+This leads to the following structure on the file system:
+
+```
+/tmp/crate/
+|-- ro-crate-metadata.json
+|-- sample_file.txt
+`-- test-data
+    `-- test_file_galaxy.txt
+```
+
+Note that we did not have to set `localPath` for `test_file_galaxy` because it was already set in the original crate that we read from the remote url.
+
+Another way to read a detached crate is to pass a JSON dictionary with the RO-Crate metadata directly to `ROCrate`. For instance:
+
+```python
+import json
+from rocrate.rocrate import ROCrate
+
+with open("/tmp/example-ro-crate-metadata.json") as f:
+    metadata = json.load(f)
+crate = ROCrate(metadata)
+```
+
+In the above example we read the metadata from a local file, but you could get it from an API endpoint or any other source.
+
+
 ### Subcrates
 
 An RO-Crate can contain one or more nested RO-Crates. For instance, consider the following layout:

rocrate/metadata.py

@@ -21,9 +21,20 @@
 # limitations under the License.
 
 import json
+import re
 import warnings
+import urllib.request
+
+import requests
 
 from .model.metadata import BASENAME, LEGACY_BASENAME
+from .utils import is_url
+
+# https://www.researchobject.org/ro-crate/specification/1.2/structure
+#   "If stored in a file... the filename SHOULD be..."
+# https://www.researchobject.org/ro-crate/specification/1.2/data-entities
+#   "It is NOT RECOMMENDED to resolve a relative root identifier..."
+MD_PATTERN = re.compile(r".*[/-]ro-crate-metadata.json(ld)?$")
 
 
 def read_metadata(metadata_path):
@@ -35,6 +46,16 @@ def read_metadata(metadata_path):
     """
     if isinstance(metadata_path, dict):
         metadata = metadata_path
+    elif is_url(str(metadata_path)):
+        if not MD_PATTERN.match(metadata_path):
+            warnings.warn(f"URI {metadata_path} should follow the pattern {MD_PATTERN.pattern!r}")
+        if metadata_path.startswith("file:"):
+            with urllib.request.urlopen(metadata_path) as resp:
+                metadata = json.load(resp)
+        else:
+            with requests.get(metadata_path) as resp:
+                resp.raise_for_status()
+                metadata = resp.json()
     else:
         with open(metadata_path, 'r', encoding='utf-8') as f:
             metadata = json.load(f)
@@ -68,48 +89,8 @@ def find_root_entity_id(entities):
     Return a tuple of the corresponding identifiers (descriptor, root).
     If the entities are not found, raise KeyError. If they are found,
     but they don't satisfy the required constraints, raise ValueError.
-
-    In the general case, the metadata file descriptor id can be an
-    absolute URI whose last path segment is "ro-crate-metadata.json[ld]".
-    Since there can be more than one such id in the crate, we need to
-    choose among the corresponding (descriptor, root) entity pairs. First, we
-    exclude those that don't satisfy other constraints, such as the
-    descriptor entity being of type CreativeWork, etc.; if this doesn't
-    leave us with a single pair, we try to pick one with a
-    heuristic. Suppose we are left with the (m1, r1) and (m2, r2) pairs:
-    if r1 is the actual root of this crate, then m2 and r2 are regular
-    files in it, and as such they must appear in r1's hasPart; r2,
-    however, is not required to have a hasPart property listing other
-    files. Thus, we look for a pair whose root entity "contains" all
-    descriptor entities from other pairs. If there is no such pair, or there
-    is more than one, we just return an arbitrary pair.
-
     """
     descriptor = entities.get(BASENAME, entities.get(LEGACY_BASENAME))
-    if descriptor:
-        return _check_descriptor(descriptor, entities)
-    candidates = []
-    for id_, e in entities.items():
-        basename = id_.rsplit("/", 1)[-1]
-        if basename == BASENAME or basename == LEGACY_BASENAME:
-            try:
-                candidates.append(_check_descriptor(e, entities))
-            except ValueError:
-                pass
-    if not candidates:
+    if not descriptor:
         raise KeyError("Metadata file descriptor not found")
-    elif len(candidates) == 1:
-        return candidates[0]
-    else:
-        warnings.warn("Multiple metadata file descriptors, will pick one with a heuristic")
-        descriptor_ids = set(_[0] for _ in candidates)
-        for m_id, r_id in candidates:
-            try:
-                root = entities[r_id]
-                part_ids = set(_["@id"] for _ in root["hasPart"])
-            except KeyError:
-                continue
-            if part_ids >= descriptor_ids - {m_id}:
-                # if True for more than one candidate, this pick is arbitrary
-                return m_id, r_id
-        return candidates[0]  # fall back to arbitrary pick
+    return _check_descriptor(descriptor, entities)

rocrate/model/file.py

@@ -71,7 +71,8 @@ def _copy_file(self, path, out_file_path):
             self._jsonld['contentSize'] = str(out_file_path.stat().st_size)
 
     def write(self, base_path):
-        out_file_path = Path(base_path) / unquote(self.id)
+        local_path = self.get("localPath")
+        out_file_path = Path(base_path) / unquote(local_path or self.id)
         if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)):
             self._write_from_stream(out_file_path)
         elif self.source is None:

rocrate/model/metadata.py

@@ -24,6 +24,8 @@
 
 import json
 from pathlib import Path
+import re
+import warnings
 
 from .file import File
 from .dataset import Dataset
@@ -37,6 +39,7 @@
 DEFAULT_VERSION = "1.2"
 BASENAME = "ro-crate-metadata.json"
 LEGACY_BASENAME = "ro-crate-metadata.jsonld"
+DETACHED_MD_NAME = re.compile(r".*-ro-crate-metadata.json$")
 
 WORKFLOW_PROFILE = "https://w3id.org/workflowhub/workflow-ro-crate/1.0"
 
@@ -94,9 +97,15 @@ def _has_writeable_stream(self):
         return True
 
     def write(self, dest_base):
-        write_path = Path(dest_base) / self.id
+        write_path = Path(dest_base) / self.id.rsplit("/", 1)[-1]
         super()._write_from_stream(write_path)
 
+    def write_detached(self, path):
+        if not DETACHED_MD_NAME.match(str(path)):
+            warnings.warn(f"{path} should follow the pattern {DETACHED_MD_NAME.pattern!r}")
+        path = Path(path)
+        super()._write_from_stream(path)
+
     @property
     def root(self) -> Dataset:
         return self.crate.root_dataset

rocrate/model/preview.py

@@ -27,6 +27,7 @@
 
 from jinja2 import Template
 from .file import File
+from ..utils import is_url
 
 
 class Preview(File):
@@ -98,6 +99,8 @@ def stream(self, chunk_size=8192):
             yield self.id, str.encode(self.generate_html(), encoding='utf-8')
 
     def _has_writeable_stream(self):
+        if is_url(str(self.source)):
+            return self.fetch_remote
         return True
 
     def write(self, dest_base):

rocrate/rocrate.py

@@ -123,7 +123,8 @@ def __init__(self,
                  gen_preview=False,
                  init=False, exclude=None,
                  version=DEFAULT_VERSION,
-                 load_subcrates=False):
+                 load_subcrates=False,
+                 root_dataset_id=None):
         self.mode = None
         self.source = source
         self.exclude = exclude
@@ -138,7 +139,12 @@ def __init__(self,
             self.add(Preview(self))
         if not source:
             self.mode = Mode.CREATE
-            self.add(RootDataset(self), Metadata(self, version=version))
+            if root_dataset_id is not None:
+                if root_dataset_id != "./" and not is_url(root_dataset_id):
+                    raise ValueError("the root dataset id must be either ./ or an absolute URI")
+            rde = RootDataset(self, root_dataset_id)
+            md = Metadata(self, properties={"about": rde}, version=version)
+            self.add(rde, md)
         elif init:
             self.mode = Mode.INIT
             if isinstance(source, dict):
@@ -170,7 +176,7 @@ def __init_from_tree(self, top_dir, gen_preview=False, version=DEFAULT_VERSION):
                     self.add(Preview(self, source))
 
     def __read(self, source, gen_preview=False):
-        if isinstance(source, dict):
+        if isinstance(source, dict) or is_url(str(source)):
             metadata_path = source
         else:
             source = Path(source)
@@ -182,11 +188,14 @@ def __read(self, source, gen_preview=False):
                 with zipfile.ZipFile(source, "r") as zf:
                     zf.extractall(zip_path)
                 source = Path(zip_path)
-            metadata_path = source / BASENAME
-            if not metadata_path.is_file():
-                metadata_path = source / LEGACY_BASENAME
-            if not metadata_path.is_file():
-                raise ValueError(f"Not a valid RO-Crate: missing {BASENAME}")
+            if source.is_file():
+                metadata_path = source
+            else:
+                metadata_path = source / BASENAME
+                if not metadata_path.is_file():
+                    metadata_path = source / LEGACY_BASENAME
+                if not metadata_path.is_file():
+                    raise ValueError(f"Not a valid RO-Crate: missing {BASENAME}")
         _, entities = read_metadata(metadata_path)
         self.__read_data_entities(entities, source, gen_preview)
         self.__read_contextual_entities(entities)
@@ -195,6 +204,8 @@ def __read(self, source, gen_preview=False):
     def __read_data_entities(self, entities, source, gen_preview):
         if isinstance(source, dict):
             source = Path("")
+        elif is_url(str(source)):
+            source = source.rsplit("/", 1)[0] + "/"
         metadata_id, root_id = find_root_entity_id(entities)
         root_entity = entities.pop(root_id)
         assert root_id == root_entity.pop('@id')
@@ -206,7 +217,13 @@ def __read_data_entities(self, entities, source, gen_preview):
 
         preview_entity = entities.pop(Preview.BASENAME, None)
         if preview_entity and not gen_preview:
-            self.add(Preview(self, source / Preview.BASENAME, properties=preview_entity))
+            if is_url(str(source)):
+                preview_source = source + Preview.BASENAME
+            elif source.is_file():
+                preview_source = source.parent / Preview.BASENAME
+            else:
+                preview_source = source / Preview.BASENAME
+            self.add(Preview(self, preview_source, properties=preview_entity))
         self.__add_parts(parts, entities, source)
 
     def __add_parts(self, parts, entities, source):
@@ -234,6 +251,10 @@ def __add_parts(self, parts, entities, source):
 
                 if is_url(id_):
                     instance = Subcrate(self, source=id_, properties=entity)
+                elif is_url(str(source)):
+                    instance = Subcrate(self, source + id_, id_, properties=entity)
+                elif source.is_file():
+                    instance = Subcrate(self, source.parent / unquote(id_), id_, properties=entity)
                 else:
                     instance = Subcrate(self, source=source / unquote(id_), properties=entity)
 
@@ -244,6 +265,10 @@ def __add_parts(self, parts, entities, source):
                 # cls is either a File or a Dataset (Directory)
                 if is_url(id_):
                     instance = cls(self, id_, properties=entity)
+                elif is_url(str(source)):
+                    instance = cls(self, source + id_, id_, properties=entity)
+                elif source.is_file():
+                    instance = cls(self, source.parent / unquote(id_), id_, properties=entity)
                 else:
                     instance = cls(self, source / unquote(id_), id_, properties=entity)
             self.add(instance)
@@ -588,7 +613,7 @@ def _copy_unlisted(self, top, base_path):
     def write(self, base_path):
         base_path = Path(base_path)
         base_path.mkdir(parents=True, exist_ok=True)
-        if self.source and not isinstance(self.source, dict):
+        if self.source and not isinstance(self.source, dict) and Path(self.source).is_dir():
             self._copy_unlisted(self.source, base_path)
         for writable_entity in self.data_entities + self.default_entities:
             writable_entity.write(base_path)
@@ -602,6 +627,9 @@ def write_zip(self, out_path):
                 f.write(chunk)
         return out_path
 
+    def write_detached(self, metadata_path):
+        self.metadata.write_detached(metadata_path)
+
     def stream_zip(self, chunk_size=8192):
         """ Create a stream of bytes representing the RO-Crate as a ZIP file. """
         yield from self._stream_zip(chunk_size=chunk_size)
@@ -917,7 +945,11 @@ def _load_subcrate(self):
         """
         if self._crate is None:
             # load_subcrates=True to load further nested RO-Crate (on-demand / lazily too)
-            self._crate = ROCrate(self.source, load_subcrates=True)
+            if subject_of := self.get("subjectOf"):
+                subcrate_uri = subject_of.id if isinstance(subject_of, Entity) else subject_of
+                self._crate = ROCrate(subcrate_uri, load_subcrates=True)
+            else:
+                self._crate = ROCrate(self.source, load_subcrates=True)
 
     def write(self, base_path):
         super().write(base_path)