From a52712fd39ec8ec675ff44912949df0b83f1024d Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 13:27:26 +0200 Subject: [PATCH 01/10] backup --- a | 1 + b | 1 + deep_code/constants.py | 1 + .../utils/test_dataset_stac_generator.py | 4 ---- deep_code/tools/new.py | 1 - deep_code/tools/prr.py | 1 - deep_code/tools/publish.py | 2 -- deep_code/utils/dataset_stac_generator.py | 24 +++++++++---------- deep_code/utils/helper.py | 2 ++ 9 files changed, 17 insertions(+), 20 deletions(-) create mode 100644 a create mode 100644 b diff --git a/a b/a new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/a @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/b b/b new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/b @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/deep_code/constants.py b/deep_code/constants.py index e1fee57..4849fac 100644 --- a/deep_code/constants.py +++ b/deep_code/constants.py @@ -42,6 +42,7 @@ ) ZARR_MEDIA_TYPE = "application/vnd+zarr" DATACUBE_SCHEMA_URI = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json" +FILE_SCHEMA_URI = "https://stac-extensions.github.io/file/v2.1.0/schema.json" PROCESSING_SCHEMA_URI = ( "https://stac-extensions.github.io/processing/v1.2.0/schema.json" ) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index ca9f83a..8867a8a 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -85,7 +85,6 @@ def setUp(self): ], workflow_id="dummy", workflow_title="test", - access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", osc_project="deep-earth-system-data-lab", @@ -912,7 +911,6 @@ def setUp(self): workflow_title="WF", license_type="CC-BY-4.0", osc_project="deep-earth-system-data-lab", - access_link_root="s3://bucket", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -1067,7 +1065,6 @@ def test_build_prr_collection_conformant_fields(self): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link_root="s3://bucket/", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -1140,7 +1137,6 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): workflow_title="WF", license_type="CC-BY-4.0", osc_project="deep-earth-system-data-lab", - access_link_root="s3://bucket", ) coll = gen.build_prr_collection() self.assertNotIn("themes", coll.extra_fields) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 7e42a73..b813a6d 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -84,7 +84,6 @@ def generate_dataset_template(output_path: str | None = None) -> str: "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", - "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", "collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]", "coord_position": "[OPTIONAL: Position of the coordinates within each grid cell. 'center' assumes coordinates represent cell centers, 'left' assumes they represent the left/bottom edge, and 'right' assumes they represent the right/top edge.]", "cf_parameter": [ diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 805825e..dc79056 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -73,7 +73,6 @@ def generate_prr_collection( license_type=license_type, documentation_link=config.get("documentation_link"), collection_title=config.get("collection_title"), - access_link_root=config.get("access_link_root"), osc_status=config.get("dataset_status") or "ongoing", osc_region=config.get("osc_region") or "Global", osc_themes=config.get("osc_themes"), diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 399f102..141f4f7 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -288,7 +288,6 @@ def publish_dataset( ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") - access_link_root = self.dataset_config.get("access_link_root") dataset_status = self.dataset_config.get("dataset_status") or "ongoing" osc_region = self.dataset_config.get("osc_region") osc_themes = self.dataset_config.get("osc_themes") @@ -330,7 +329,6 @@ def publish_dataset( workflow_title=self.workflow_title, license_type=license_type, documentation_link=documentation_link, - access_link_root=access_link_root, osc_status=dataset_status, osc_region=osc_region, osc_themes=osc_themes, diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index a1baa2a..ee76430 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -28,6 +28,7 @@ from deep_code.constants import ( CONTACTS_SCHEMA_URI, DATACUBE_SCHEMA_URI, + FILE_SCHEMA_URI, OSC_SCHEMA_URI, OSC_THEME_SCHEME, PROCESSING_SCHEMA_URI, @@ -54,7 +55,6 @@ class OscDatasetStacGenerator: items_config: List of item configuration entries. Each item maps one dataset_id to one item_id collection_title: Title present in the collection and in the STAC browser - access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -77,7 +77,6 @@ def __init__( license_type: str, osc_project: str, collection_title: str | None = None, - access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", osc_region: str = "Global", @@ -111,7 +110,6 @@ def __init__( self.osc_project = osc_project self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url - self.access_link_root = access_link_root or "s3://deep-esdl-public/" self.collection_title = collection_title or collection_id self.documentation_link = documentation_link self.osc_status = osc_status @@ -223,13 +221,6 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - def _build_access_link(self, item_config: ItemConfig) -> str: - """Return the asset href for an item, supporting prefix and full URLs.""" - root = self.access_link_root - if root.endswith("/"): - root = root.rstrip("/") - return f"{root}/{item_config.dataset_id}" - @staticmethod def _union_spatial_extent(items: list[Item]) -> SpatialExtent: """Merge multiple dataset spatial extents into a single bounding box.""" @@ -730,7 +721,7 @@ def build_zarr_stac_item( title=self.collection_id, ) ) - access_link = self._build_access_link(item_config) + access_link = f"../{item_config.dataset_id}" item.add_asset( "zarr-data", Asset( @@ -999,10 +990,11 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: "cube:variables": self._get_cube_variables(dataset), }, ) + item.stac_extensions.append(FILE_SCHEMA_URI) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. - access_link = self._build_access_link(item_config) + access_link = f"../{item_config.dataset_id}" item.add_asset( "zarr-data", Asset( @@ -1010,6 +1002,10 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: media_type=ZARR_MEDIA_TYPE, title="Zarr Data Store", roles=["data"], + extra_fields={ + "file:size": file_size, + "file:checksum": checksum, + }, ), ) item.add_asset( @@ -1019,6 +1015,10 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: media_type="application/json", title="Consolidated Zarr Metadata", roles=["metadata"], + extra_fields={ + "file:size": file_size, + "file:checksum": checksum, + }, ), ) diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index a27a786..fc622ff 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -95,6 +95,8 @@ def open_dataset( storage_options=config["params"]["storage_options"], ) dataset = store.open_data(dataset_id) + files = store.fs.find(dataset_id) + dataset.attrs["size"] = sum(store.fs.info(file)["size"] for file in files) logger.info( f"Successfully opened dataset '{dataset_id}' with configuration: " f"{config['description']}" From e898b0f90468bd9153d3c205ad5821f17a437ddc Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 15:28:40 +0200 Subject: [PATCH 02/10] add file size and extensions --- deep_code/tools/new.py | 1 + deep_code/utils/dataset_stac_generator.py | 16 ++++++---------- deep_code/utils/helper.py | 2 ++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index b813a6d..d8f0a39 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -79,6 +79,7 @@ def generate_dataset_template(output_path: str | None = None) -> str: "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" ], "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]", + "osc_status": "[OPTIONAL: status of the dataset, can be one of 'planned', 'ongoing', 'completed', defaults to 'completed']", "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]", "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index ee76430..f8c8567 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -56,7 +56,7 @@ class OscDatasetStacGenerator: dataset_id to one item_id collection_title: Title present in the collection and in the STAC browser documentation_link: Link to dataset documentation. - osc_status: Status of the dataset (e.g., "ongoing"). + osc_status: Status of the dataset (one of "planned", "ongoing", "completed"). osc_region: Geographical region associated with the dataset. osc_themes: List of themes related to the dataset (e.g., ["climate"]). osc_missions: List of satellite missions associated with the dataset. @@ -78,7 +78,7 @@ def __init__( osc_project: str, collection_title: str | None = None, documentation_link: str | None = None, - osc_status: str = "ongoing", + osc_status: str = "completed", osc_region: str = "Global", osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, @@ -729,6 +729,7 @@ def build_zarr_stac_item( media_type=ZARR_MEDIA_TYPE, title="Zarr Data Store", roles=["data"], + extra_fields={"file:size": dataset.attrs["size"]}, ), ) item.add_asset( @@ -738,6 +739,7 @@ def build_zarr_stac_item( media_type="application/json", title="Consolidated Zarr Metadata", roles=["metadata"], + extra_fields={"file:size": dataset.attrs["metadata_size"]}, ), ) self.logger.info(f"STAC Item built: {item_href}") @@ -1002,10 +1004,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: media_type=ZARR_MEDIA_TYPE, title="Zarr Data Store", roles=["data"], - extra_fields={ - "file:size": file_size, - "file:checksum": checksum, - }, + extra_fields={"file:size": dataset.attrs["size"]}, ), ) item.add_asset( @@ -1015,10 +1014,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: media_type="application/json", title="Consolidated Zarr Metadata", roles=["metadata"], - extra_fields={ - "file:size": file_size, - "file:checksum": checksum, - }, + extra_fields={"file:size": dataset.attrs["metadata_size"]}, ), ) diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index fc622ff..2434c74 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -97,6 +97,8 @@ def open_dataset( dataset = store.open_data(dataset_id) files = store.fs.find(dataset_id) dataset.attrs["size"] = sum(store.fs.info(file)["size"] for file in files) + metadata_path = f"{dataset_id}/.zmetadata" + dataset.attrs["metadata_size"] = store.fs.info(metadata_path)["size"] logger.info( f"Successfully opened dataset '{dataset_id}' with configuration: " f"{config['description']}" From 8146dd8c1c60bbf368130109e42a1f9d9baec923 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 16:11:04 +0200 Subject: [PATCH 03/10] debug --- deep_code/utils/helper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 2434c74..7c3f1eb 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -95,10 +95,14 @@ def open_dataset( storage_options=config["params"]["storage_options"], ) dataset = store.open_data(dataset_id) + print(dataset) files = store.fs.find(dataset_id) + print(files) dataset.attrs["size"] = sum(store.fs.info(file)["size"] for file in files) + print(dataset.attrs["size"]) metadata_path = f"{dataset_id}/.zmetadata" dataset.attrs["metadata_size"] = store.fs.info(metadata_path)["size"] + print(dataset.attrs["metadata_size"]) logger.info( f"Successfully opened dataset '{dataset_id}' with configuration: " f"{config['description']}" From 1fa680c3f20b912b2e5d55b80f5bea3359ef1194 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 16:20:34 +0200 Subject: [PATCH 04/10] file size added --- deep_code/utils/helper.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 7c3f1eb..24a8a28 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -95,14 +95,10 @@ def open_dataset( storage_options=config["params"]["storage_options"], ) dataset = store.open_data(dataset_id) - print(dataset) - files = store.fs.find(dataset_id) - print(files) + files = store.fs.find(f"{store.root}/{dataset_id}") dataset.attrs["size"] = sum(store.fs.info(file)["size"] for file in files) - print(dataset.attrs["size"]) - metadata_path = f"{dataset_id}/.zmetadata" + metadata_path = f"{store.root}/{dataset_id}/.zmetadata" dataset.attrs["metadata_size"] = store.fs.info(metadata_path)["size"] - print(dataset.attrs["metadata_size"]) logger.info( f"Successfully opened dataset '{dataset_id}' with configuration: " f"{config['description']}" From 496fad94af8becf84017b21944bb10a06ad5c331 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 16:41:23 +0200 Subject: [PATCH 05/10] ready for review --- .../utils/test_dataset_stac_generator.py | 25 ++++++++++++------- deep_code/utils/dataset_stac_generator.py | 10 ++++++-- deep_code/utils/helper.py | 14 ++++++++--- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 8867a8a..4b93276 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -46,7 +46,12 @@ def setUp(self): ], ), }, - attrs={"description": "Mock dataset for testing.", "title": "Mock Dataset"}, + attrs={ + "description": "Mock dataset for testing.", + "title": "Mock Dataset", + "size": 123456, + "metadata_size": 123, + }, data_vars={ "var1": ( ("time", "lat", "lon"), @@ -468,14 +473,14 @@ def test_build_zarr_stac_item_structure(self): self.assertIn("zarr-consolidated-metadata", item.assets) zarr_asset = item.assets["zarr-data"] - self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset-id") + self.assertEqual(zarr_asset.href, "../mock-dataset-id") self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE) self.assertIn("data", zarr_asset.roles) meta_asset = item.assets["zarr-consolidated-metadata"] self.assertEqual( meta_asset.href, - "s3://mock-bucket/mock-dataset-id/.zmetadata", + "../mock-dataset-id/.zmetadata", ) self.assertIn("metadata", meta_asset.roles) @@ -894,7 +899,11 @@ def setUp(self): # A CRS var that must be excluded from cube:variables and drive EPSG. "spatial_ref": ((), 0, {"spatial_epsg": 3035}), }, - attrs={"description": "PRR test cube"}, + attrs={ + "description": "PRR test cube", + "size": 123456, + "metadata_size": 123, + }, ) self.open_dataset_patcher = patch( "deep_code.utils.dataset_stac_generator.open_dataset", @@ -1010,11 +1019,11 @@ def test_build_prr_stac_item(self): self.assertTrue(item.properties["end_datetime"].endswith("+00:00")) self.assertEqual(set(item.assets), {"zarr-data", "zarr-consolidated-metadata"}) - self.assertEqual(item.assets["zarr-data"].href, "s3://bucket/test.zarr") + self.assertEqual(item.assets["zarr-data"].href, "../test.zarr") self.assertEqual(item.assets["zarr-data"].media_type, ZARR_MEDIA_TYPE) self.assertEqual( item.assets["zarr-consolidated-metadata"].href, - "s3://bucket/test.zarr/.zmetadata", + "../test.zarr/.zmetadata", ) # ---- collection ---- @@ -1170,9 +1179,7 @@ def test_save_prr_collection_writes_tree(self): self.assertTrue(item_link["href"].endswith(".json")) # Asset hrefs stay absolute (the data lives on S3). - self.assertEqual( - item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" - ) + self.assertEqual(item_dict["assets"]["zarr-data"]["href"], "../test.zarr") def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index f8c8567..8ecbcf3 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -1037,7 +1037,9 @@ def build_prr_collection(self) -> Collection: ] spatial_extent = self._union_spatial_extent(items) temporal_extent = self._union_temporal_extent(items) - dataset_ref = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + dataset_ref = open_dataset( + self.items_config[0].dataset_id, logger=self.logger, calc_filesizes=False + ) variables = self.get_variable_ids(dataset_ref) collection = Collection( @@ -1232,7 +1234,11 @@ def build_dataset_stac_collection( """ try: assert len(self.items_config) == 1 - dataset = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + dataset = open_dataset( + self.items_config[0].dataset_id, + logger=self.logger, + calc_filesizes=False, + ) spatial_extent = self._get_spatial_extent(dataset, self.coord_position) temporal_extent = self._get_temporal_extent(dataset) variables = self.get_variable_ids(dataset) diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 24a8a28..98bbe05 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -26,6 +26,7 @@ def open_dataset( root: str = "deep-esdl-public", storage_configs: list[dict] | None = None, logger: logging.Logger | None = None, + calc_filesizes: bool = True, ) -> xr.Dataset: """Open an xarray dataset from a specified store. @@ -35,6 +36,8 @@ def open_dataset( root: Root path or bucket for the store. Defaults to 'deep-esdl-public'. storage_configs: List of storage configurations. If None, uses default S3 configs. logger: Optional logger for logging messages. If None, uses default logger. + calc_filesizes: Boolean for size calculation of the Zarr store and the Zarr + metadata. Defaults to True. Returns: xarray.Dataset: The opened dataset. @@ -95,10 +98,13 @@ def open_dataset( storage_options=config["params"]["storage_options"], ) dataset = store.open_data(dataset_id) - files = store.fs.find(f"{store.root}/{dataset_id}") - dataset.attrs["size"] = sum(store.fs.info(file)["size"] for file in files) - metadata_path = f"{store.root}/{dataset_id}/.zmetadata" - dataset.attrs["metadata_size"] = store.fs.info(metadata_path)["size"] + if calc_filesizes: + files = store.fs.find(f"{store.root}/{dataset_id}") + dataset.attrs["size"] = sum( + store.fs.info(file)["size"] for file in files + ) + metadata_path = f"{store.root}/{dataset_id}/.zmetadata" + dataset.attrs["metadata_size"] = store.fs.info(metadata_path)["size"] logger.info( f"Successfully opened dataset '{dataset_id}' with configuration: " f"{config['description']}" From 199374970389b345f978b40cdf8632c93e61e650 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 16:45:46 +0200 Subject: [PATCH 06/10] remove test files --- a | 1 - b | 1 - 2 files changed, 2 deletions(-) delete mode 100644 a delete mode 100644 b diff --git a/a b/a deleted file mode 100644 index 9e26dfe..0000000 --- a/a +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/b b/b deleted file mode 100644 index 9e26dfe..0000000 --- a/b +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file From b1c94adb332a91942b56a9d26905d8f1c874992d Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 16 Sep 2026 16:59:29 +0200 Subject: [PATCH 07/10] adjusted relate path --- deep_code/tests/utils/test_dataset_stac_generator.py | 10 +++++----- deep_code/utils/dataset_stac_generator.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 4b93276..915ff24 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -473,14 +473,14 @@ def test_build_zarr_stac_item_structure(self): self.assertIn("zarr-consolidated-metadata", item.assets) zarr_asset = item.assets["zarr-data"] - self.assertEqual(zarr_asset.href, "../mock-dataset-id") + self.assertEqual(zarr_asset.href, "./mock-dataset-id") self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE) self.assertIn("data", zarr_asset.roles) meta_asset = item.assets["zarr-consolidated-metadata"] self.assertEqual( meta_asset.href, - "../mock-dataset-id/.zmetadata", + "./mock-dataset-id/.zmetadata", ) self.assertIn("metadata", meta_asset.roles) @@ -1019,11 +1019,11 @@ def test_build_prr_stac_item(self): self.assertTrue(item.properties["end_datetime"].endswith("+00:00")) self.assertEqual(set(item.assets), {"zarr-data", "zarr-consolidated-metadata"}) - self.assertEqual(item.assets["zarr-data"].href, "../test.zarr") + self.assertEqual(item.assets["zarr-data"].href, "./test.zarr") self.assertEqual(item.assets["zarr-data"].media_type, ZARR_MEDIA_TYPE) self.assertEqual( item.assets["zarr-consolidated-metadata"].href, - "../test.zarr/.zmetadata", + "./test.zarr/.zmetadata", ) # ---- collection ---- @@ -1179,7 +1179,7 @@ def test_save_prr_collection_writes_tree(self): self.assertTrue(item_link["href"].endswith(".json")) # Asset hrefs stay absolute (the data lives on S3). - self.assertEqual(item_dict["assets"]["zarr-data"]["href"], "../test.zarr") + self.assertEqual(item_dict["assets"]["zarr-data"]["href"], "./test.zarr") def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 8ecbcf3..c72cfe3 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -721,7 +721,7 @@ def build_zarr_stac_item( title=self.collection_id, ) ) - access_link = f"../{item_config.dataset_id}" + access_link = f"./{item_config.dataset_id}" item.add_asset( "zarr-data", Asset( @@ -996,7 +996,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. - access_link = f"../{item_config.dataset_id}" + access_link = f"./{item_config.dataset_id}" item.add_asset( "zarr-data", Asset( From 45dda78786be9faadcbdf6988ce0ab43d49191c4 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 17 Sep 2026 13:42:06 +0200 Subject: [PATCH 08/10] extract data var names correctly --- deep_code/utils/dataset_stac_generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index c72cfe3..7a528d3 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -262,8 +262,10 @@ def _get_general_metadata(self, dataset: xr.Dataset) -> dict: def extract_metadata_for_variable(self, variable_data) -> dict: """Extract metadata for a single variable.""" long_name = variable_data.attrs.get("long_name") - standard_name = variable_data.attrs.get("standard_name") - variable_id = standard_name or variable_data.name + standard_name = variable_data.attrs.get("standard_name", "unknown") + variable_id = ( + variable_data.name if standard_name == "unknown" else standard_name + ) description = variable_data.attrs.get("description", long_name) gcmd_keyword_url = variable_data.attrs.get("gcmd_keyword_url") return { @@ -1056,6 +1058,7 @@ def build_prr_collection(self) -> Collection: osc_extension.osc_type = "product" osc_extension.osc_status = self.osc_status osc_extension.osc_region = self.osc_region + print(variables) osc_extension.osc_variables = variables osc_extension.osc_missions = self.osc_missions osc_extension.cf_parameter = self.cf_params or [{"name": self.collection_id}] From 7ef83f4624093c2df26ffcd54b43ed5f2b50d939 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 17 Sep 2026 14:13:41 +0200 Subject: [PATCH 09/10] rename dataset_status to osc_status --- deep_code/tools/new.py | 3 +-- deep_code/tools/prr.py | 2 +- deep_code/tools/publish.py | 4 ++-- deep_code/utils/dataset_stac_generator.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index d8f0a39..9e54a3d 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -79,8 +79,7 @@ def generate_dataset_template(output_path: str | None = None) -> str: "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" ], "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]", - "osc_status": "[OPTIONAL: status of the dataset, can be one of 'planned', 'ongoing', 'completed', defaults to 'completed']", - "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]", + "osc_status": "[OPTIONAL: ongoing | completed | planned (default: completed)]", "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index dc79056..d551a52 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -73,7 +73,7 @@ def generate_prr_collection( license_type=license_type, documentation_link=config.get("documentation_link"), collection_title=config.get("collection_title"), - osc_status=config.get("dataset_status") or "ongoing", + osc_status=config.get("osc_status") or "completed", osc_region=config.get("osc_region") or "Global", osc_themes=config.get("osc_themes"), osc_missions=config.get("osc_missions"), diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 141f4f7..ba870a7 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -288,7 +288,7 @@ def publish_dataset( ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") - dataset_status = self.dataset_config.get("dataset_status") or "ongoing" + osc_status = self.dataset_config.get("osc_status") or "completed" osc_region = self.dataset_config.get("osc_region") osc_themes = self.dataset_config.get("osc_themes") cf_params = self.dataset_config.get("cf_parameter") @@ -329,7 +329,7 @@ def publish_dataset( workflow_title=self.workflow_title, license_type=license_type, documentation_link=documentation_link, - osc_status=dataset_status, + osc_status=osc_status, osc_region=osc_region, osc_themes=osc_themes, cf_params=cf_params, diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 7a528d3..dc4a9d7 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -78,8 +78,8 @@ def __init__( osc_project: str, collection_title: str | None = None, documentation_link: str | None = None, - osc_status: str = "completed", - osc_region: str = "Global", + osc_status: str | None = "completed", + osc_region: str | None = "Global", osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, cf_params: list[dict[str, Any]] | None = None, From 7c453104e19c8d42adfa63823550d34320e50968 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 17 Sep 2026 14:17:16 +0200 Subject: [PATCH 10/10] remove printing --- deep_code/utils/dataset_stac_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index dc4a9d7..74570a9 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -1058,7 +1058,6 @@ def build_prr_collection(self) -> Collection: osc_extension.osc_type = "product" osc_extension.osc_status = self.osc_status osc_extension.osc_region = self.osc_region - print(variables) osc_extension.osc_variables = variables osc_extension.osc_missions = self.osc_missions osc_extension.cf_parameter = self.cf_params or [{"name": self.collection_id}]