Add missing parts to be conformant with PRR specs

deep_code/constants.py

@@ -42,6 +42,7 @@
 )
 ZARR_MEDIA_TYPE = "application/vnd+zarr"
 DATACUBE_SCHEMA_URI = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json"
+FILE_SCHEMA_URI = "https://stac-extensions.github.io/file/v2.1.0/schema.json"
 PROCESSING_SCHEMA_URI = (
     "https://stac-extensions.github.io/processing/v1.2.0/schema.json"
 )

deep_code/tests/utils/test_dataset_stac_generator.py

@@ -46,7 +46,12 @@ def setUp(self):
                     ],
                 ),
             },
-            attrs={"description": "Mock dataset for testing.", "title": "Mock Dataset"},
+            attrs={
+                "description": "Mock dataset for testing.",
+                "title": "Mock Dataset",
+                "size": 123456,
+                "metadata_size": 123,
+            },
             data_vars={
                 "var1": (
                     ("time", "lat", "lon"),
@@ -85,7 +90,6 @@ def setUp(self):
             ],
             workflow_id="dummy",
             workflow_title="test",
-            access_link_root="s3://mock-bucket/",
             documentation_link="https://example.com/docs",
             license_type="proprietary",
             osc_project="deep-earth-system-data-lab",
@@ -469,14 +473,14 @@ def test_build_zarr_stac_item_structure(self):
         self.assertIn("zarr-consolidated-metadata", item.assets)
 
         zarr_asset = item.assets["zarr-data"]
-        self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset-id")
+        self.assertEqual(zarr_asset.href, "./mock-dataset-id")
         self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE)
         self.assertIn("data", zarr_asset.roles)
 
         meta_asset = item.assets["zarr-consolidated-metadata"]
         self.assertEqual(
             meta_asset.href,
-            "s3://mock-bucket/mock-dataset-id/.zmetadata",
+            "./mock-dataset-id/.zmetadata",
         )
         self.assertIn("metadata", meta_asset.roles)
 
@@ -895,7 +899,11 @@ def setUp(self):
                 # A CRS var that must be excluded from cube:variables and drive EPSG.
                 "spatial_ref": ((), 0, {"spatial_epsg": 3035}),
             },
-            attrs={"description": "PRR test cube"},
+            attrs={
+                "description": "PRR test cube",
+                "size": 123456,
+                "metadata_size": 123,
+            },
         )
         self.open_dataset_patcher = patch(
             "deep_code.utils.dataset_stac_generator.open_dataset",
@@ -912,7 +920,6 @@ def setUp(self):
             workflow_title="WF",
             license_type="CC-BY-4.0",
             osc_project="deep-earth-system-data-lab",
-            access_link_root="s3://bucket",
             osc_status="ongoing",
             osc_region="Global",
             osc_themes=["oceans"],
@@ -1012,11 +1019,11 @@ def test_build_prr_stac_item(self):
         self.assertTrue(item.properties["end_datetime"].endswith("+00:00"))
 
         self.assertEqual(set(item.assets), {"zarr-data", "zarr-consolidated-metadata"})
-        self.assertEqual(item.assets["zarr-data"].href, "s3://bucket/test.zarr")
+        self.assertEqual(item.assets["zarr-data"].href, "./test.zarr")
         self.assertEqual(item.assets["zarr-data"].media_type, ZARR_MEDIA_TYPE)
         self.assertEqual(
             item.assets["zarr-consolidated-metadata"].href,
-            "s3://bucket/test.zarr/.zmetadata",
+            "./test.zarr/.zmetadata",
         )
 
     # ---- collection ----
@@ -1067,7 +1074,6 @@ def test_build_prr_collection_conformant_fields(self):
                 workflow_id="wf",
                 workflow_title="WF",
                 license_type="CC-BY-4.0",
-                access_link_root="s3://bucket/",
                 osc_status="ongoing",
                 osc_region="Global",
                 osc_themes=["oceans"],
@@ -1140,7 +1146,6 @@ def test_build_prr_collection_no_themes(self, mock_open_ds):
             workflow_title="WF",
             license_type="CC-BY-4.0",
             osc_project="deep-earth-system-data-lab",
-            access_link_root="s3://bucket",
         )
         coll = gen.build_prr_collection()
         self.assertNotIn("themes", coll.extra_fields)
@@ -1174,9 +1179,7 @@ def test_save_prr_collection_writes_tree(self):
             self.assertTrue(item_link["href"].endswith(".json"))
 
             # Asset hrefs stay absolute (the data lives on S3).
-            self.assertEqual(
-                item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr"
-            )
+            self.assertEqual(item_dict["assets"]["zarr-data"]["href"], "./test.zarr")
 
     def test_save_prr_collection_readable_by_pystac(self):
         with tempfile.TemporaryDirectory() as tmp:

deep_code/tools/new.py

@@ -79,12 +79,11 @@ def generate_dataset_template(output_path: str | None = None) -> str:
                 "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]"
             ],
             "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]",
-            "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]",
+            "osc_status": "[OPTIONAL: ongoing | completed | planned (default: completed)]",
             "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]",
             "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]",
             "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]",
             "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]",
-            "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]",
             "collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]",
             "coord_position": "[OPTIONAL: Position of the coordinates within each grid cell. 'center' assumes coordinates represent cell centers, 'left' assumes they represent the left/bottom edge, and 'right' assumes they represent the right/top edge.]",
             "cf_parameter": [

deep_code/tools/prr.py

@@ -73,8 +73,7 @@ def generate_prr_collection(
         license_type=license_type,
         documentation_link=config.get("documentation_link"),
         collection_title=config.get("collection_title"),
-        access_link_root=config.get("access_link_root"),
-        osc_status=config.get("dataset_status") or "ongoing",
+        osc_status=config.get("osc_status") or "completed",
         osc_region=config.get("osc_region") or "Global",
         osc_themes=config.get("osc_themes"),
         osc_missions=config.get("osc_missions"),

deep_code/tools/publish.py

@@ -288,8 +288,7 @@ def publish_dataset(
             )
         self.collection_id = self.dataset_config.get("collection_id")
         documentation_link = self.dataset_config.get("documentation_link")
-        access_link_root = self.dataset_config.get("access_link_root")
-        dataset_status = self.dataset_config.get("dataset_status") or "ongoing"
+        osc_status = self.dataset_config.get("osc_status") or "completed"
         osc_region = self.dataset_config.get("osc_region")
         osc_themes = self.dataset_config.get("osc_themes")
         cf_params = self.dataset_config.get("cf_parameter")
@@ -330,8 +329,7 @@ def publish_dataset(
             workflow_title=self.workflow_title,
             license_type=license_type,
             documentation_link=documentation_link,
-            access_link_root=access_link_root,
-            osc_status=dataset_status,
+            osc_status=osc_status,
             osc_region=osc_region,
             osc_themes=osc_themes,
             cf_params=cf_params,

deep_code/utils/dataset_stac_generator.py

@@ -28,6 +28,7 @@
 from deep_code.constants import (
     CONTACTS_SCHEMA_URI,
     DATACUBE_SCHEMA_URI,
+    FILE_SCHEMA_URI,
     OSC_SCHEMA_URI,
     OSC_THEME_SCHEME,
     PROCESSING_SCHEMA_URI,
@@ -54,9 +55,8 @@ class OscDatasetStacGenerator:
         items_config: List of item configuration entries. Each item maps one
             dataset_id to one item_id
         collection_title: Title present in the collection and in the STAC browser
-        access_link_root: Public access link to the root of the datasets.
         documentation_link: Link to dataset documentation.
-        osc_status: Status of the dataset (e.g., "ongoing").
+        osc_status: Status of the dataset (one of "planned", "ongoing", "completed").
         osc_region: Geographical region associated with the dataset.
         osc_themes: List of themes related to the dataset (e.g., ["climate"]).
         osc_missions: List of satellite missions associated with the dataset.
@@ -77,10 +77,9 @@ def __init__(
         license_type: str,
         osc_project: str,
         collection_title: str | None = None,
-        access_link_root: str | None = None,
         documentation_link: str | None = None,
-        osc_status: str = "ongoing",
-        osc_region: str = "Global",
+        osc_status: str | None = "completed",
+        osc_region: str | None = "Global",
         osc_themes: list[str] | None = None,
         osc_missions: list[str] | None = None,
         cf_params: list[dict[str, Any]] | None = None,
@@ -111,7 +110,6 @@ def __init__(
         self.osc_project = osc_project
         self.osc_project_title = osc_project_title or osc_project
         self.osc_project_url = osc_project_url
-        self.access_link_root = access_link_root or "s3://deep-esdl-public/"
         self.collection_title = collection_title or collection_id
         self.documentation_link = documentation_link
         self.osc_status = osc_status
@@ -223,13 +221,6 @@ def _normalize_name(name: str | None) -> str | None:
             return name.replace(" ", "-").replace("_", "-").lower()
         return None
 
-    def _build_access_link(self, item_config: ItemConfig) -> str:
-        """Return the asset href for an item, supporting prefix and full URLs."""
-        root = self.access_link_root
-        if root.endswith("/"):
-            root = root.rstrip("/")
-        return f"{root}/{item_config.dataset_id}"
-
     @staticmethod
     def _union_spatial_extent(items: list[Item]) -> SpatialExtent:
         """Merge multiple dataset spatial extents into a single bounding box."""
@@ -271,8 +262,10 @@ def _get_general_metadata(self, dataset: xr.Dataset) -> dict:
     def extract_metadata_for_variable(self, variable_data) -> dict:
         """Extract metadata for a single variable."""
         long_name = variable_data.attrs.get("long_name")
-        standard_name = variable_data.attrs.get("standard_name")
-        variable_id = standard_name or variable_data.name
+        standard_name = variable_data.attrs.get("standard_name", "unknown")
+        variable_id = (
+            variable_data.name if standard_name == "unknown" else standard_name
+        )
         description = variable_data.attrs.get("description", long_name)
         gcmd_keyword_url = variable_data.attrs.get("gcmd_keyword_url")
         return {
@@ -730,14 +723,15 @@ def build_zarr_stac_item(
                 title=self.collection_id,
             )
         )
-        access_link = self._build_access_link(item_config)
+        access_link = f"./{item_config.dataset_id}"
         item.add_asset(
             "zarr-data",
             Asset(
                 href=access_link,
                 media_type=ZARR_MEDIA_TYPE,
                 title="Zarr Data Store",
                 roles=["data"],
+                extra_fields={"file:size": dataset.attrs["size"]},
             ),
         )
         item.add_asset(
@@ -747,6 +741,7 @@ def build_zarr_stac_item(
                 media_type="application/json",
                 title="Consolidated Zarr Metadata",
                 roles=["metadata"],
+                extra_fields={"file:size": dataset.attrs["metadata_size"]},
             ),
         )
         self.logger.info(f"STAC Item built: {item_href}")
@@ -999,17 +994,19 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item:
                 "cube:variables": self._get_cube_variables(dataset),
             },
         )
+        item.stac_extensions.append(FILE_SCHEMA_URI)
         item.stac_extensions.append(DATACUBE_SCHEMA_URI)
         # Asset hrefs stay absolute (the Zarr lives on S3); only the structural
         # links become relative when the tree is normalised locally.
-        access_link = self._build_access_link(item_config)
+        access_link = f"./{item_config.dataset_id}"
         item.add_asset(
             "zarr-data",
             Asset(
                 href=access_link,
                 media_type=ZARR_MEDIA_TYPE,
                 title="Zarr Data Store",
                 roles=["data"],
+                extra_fields={"file:size": dataset.attrs["size"]},
             ),
         )
         item.add_asset(
@@ -1019,6 +1016,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item:
                 media_type="application/json",
                 title="Consolidated Zarr Metadata",
                 roles=["metadata"],
+                extra_fields={"file:size": dataset.attrs["metadata_size"]},
             ),
         )
 
@@ -1041,7 +1039,9 @@ def build_prr_collection(self) -> Collection:
         ]
         spatial_extent = self._union_spatial_extent(items)
         temporal_extent = self._union_temporal_extent(items)
-        dataset_ref = open_dataset(self.items_config[0].dataset_id, logger=self.logger)
+        dataset_ref = open_dataset(
+            self.items_config[0].dataset_id, logger=self.logger, calc_filesizes=False
+        )
         variables = self.get_variable_ids(dataset_ref)
 
         collection = Collection(
@@ -1236,7 +1236,11 @@ def build_dataset_stac_collection(
         """
         try:
             assert len(self.items_config) == 1
-            dataset = open_dataset(self.items_config[0].dataset_id, logger=self.logger)
+            dataset = open_dataset(
+                self.items_config[0].dataset_id,
+                logger=self.logger,
+                calc_filesizes=False,
+            )
             spatial_extent = self._get_spatial_extent(dataset, self.coord_position)
             temporal_extent = self._get_temporal_extent(dataset)
             variables = self.get_variable_ids(dataset)

deep_code/utils/helper.py

@@ -26,6 +26,7 @@ def open_dataset(
     root: str = "deep-esdl-public",
     storage_configs: list[dict] | None = None,
     logger: logging.Logger | None = None,
+    calc_filesizes: bool = True,
 ) -> xr.Dataset:
     """Open an xarray dataset from a specified store.
 
@@ -35,6 +36,8 @@ def open_dataset(
         root: Root path or bucket for the store. Defaults to 'deep-esdl-public'.
         storage_configs: List of storage configurations. If None, uses default S3 configs.
         logger: Optional logger for logging messages. If None, uses default logger.
+        calc_filesizes: Boolean for size calculation of the Zarr store and the Zarr
+            metadata. Defaults to True.
 
     Returns:
         xarray.Dataset: The opened dataset.
@@ -95,6 +98,13 @@ def open_dataset(
                 storage_options=config["params"]["storage_options"],
             )
             dataset = store.open_data(dataset_id)
+            if calc_filesizes:
+                files = store.fs.find(f"{store.root}/{dataset_id}")
+                dataset.attrs["size"] = sum(
+                    store.fs.info(file)["size"] for file in files
+                )
+                metadata_path = f"{store.root}/{dataset_id}/.zmetadata"
+                dataset.attrs["metadata_size"] = store.fs.info(metadata_path)["size"]
             logger.info(
                 f"Successfully opened dataset '{dataset_id}' with configuration: "
                 f"{config['description']}"