diff --git a/README.md b/README.md index 3f5fa109..cdea70e3 100755 --- a/README.md +++ b/README.md @@ -57,6 +57,31 @@ The following projects have adopted Neuroshapes: # Formats and standards All schemas in this repository conform to the [W3C SHACL recommendation](https://www.w3.org/TR/shacl) and are serialized using [JSON-LD](https://www.w3.org/TR/2014/REC-json-ld-20140116/). +## Using YAML templates for manual data entry + +For researchers who need to manually enter experimental metadata, we provide human-friendly YAML templates in the `templates/yaml/` directory. These templates are easier to edit than JSON-LD and can be converted automatically. + +### Quick start + +```bash +# Copy a template +cp templates/yaml/subject.yaml my_experiment.yaml + +# Fill in your data with any text editor +nano my_experiment.yaml + +# Convert to JSON-LD +python scripts/yaml_to_jsonld.py my_experiment.yaml my_experiment.json +``` + +Available templates: +- `subject.yaml` - Animal/subject information +- `slice.yaml` - Brain slice preparation +- `patched_slice.yaml` - Electrophysiology recording +- `reconstructed_cell.yaml` - Complete reconstruction workflow + +See [templates/yaml/README.md](templates/yaml/README.md) for detailed documentation. + ## Testing shapes with examples Two different tests are executed in the unittest. The first test validates that schemas conform with the SHACL specifications. @@ -86,7 +111,7 @@ Tests require python > 3.6, and pytest. To run them follow next: python3 -m venv env source env/bin/activate # install requirements - pip install pytest pyshacl + pip install -r requirements.txt # run tests pytest diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..534e0a4e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pytest>=7.0.0 +rdflib>=6.0.0 +pyshacl>=0.17.0 +PyYAML>=6.0 diff --git a/scripts/yaml_to_jsonld.py b/scripts/yaml_to_jsonld.py new file mode 100755 index 00000000..8cf2cc35 --- /dev/null +++ b/scripts/yaml_to_jsonld.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +""" +Convert YAML templates to JSON-LD format compatible with Neuroshapes schemas. + +Usage: + python yaml_to_jsonld.py input.yaml output.json + python yaml_to_jsonld.py input.yaml output.json --validate +""" + +import yaml +import json +import sys +from pathlib import Path +from typing import Dict, Any + +CONTEXT = "https://incf.github.io/neuroshapes/contexts/data.json" + +TYPE_MAPPINGS = { + "Subject": "nsg:Subject", + "Slice": "nsg:Slice", + "PatchedSlice": "nsg:PatchedSlice", + "FixedStainedSlice": "nsg:FixedStainedSlice", + "ImagedSlice": "nsg:ImagedSlice", + "LabeledCell": "nsg:LabeledCell", + "ReconstructedCell": "nsg:ReconstructedCell" +} + + +def clean_entity(entity_data: Dict[str, Any]) -> Dict[str, Any]: + """Remove empty fields from entity data.""" + cleaned = {} + for key, value in entity_data.items(): + if value == "" or value is None: + continue + if isinstance(value, dict): + nested = clean_entity(value) + if nested: + cleaned[key] = nested + else: + cleaned[key] = value + return cleaned + + +def yaml_to_jsonld(yaml_data: Dict[str, Any]) -> Dict[str, Any]: + """Convert YAML structure to JSON-LD with proper @type and @context.""" + jsonld = { + "@context": CONTEXT, + "@graph": [] + } + + for entity_type, entity_data in yaml_data.items(): + if entity_type not in TYPE_MAPPINGS: + continue + + if not entity_data: + continue + + cleaned_data = clean_entity(entity_data) + if not cleaned_data: + continue + + entity = { + "@type": TYPE_MAPPINGS[entity_type], + **cleaned_data + } + jsonld["@graph"].append(entity) + + return jsonld + + +def main(): + if len(sys.argv) < 3: + print(__doc__) + sys.exit(1) + + input_path = Path(sys.argv[1]) + output_path = Path(sys.argv[2]) + + if not input_path.exists(): + print(f"Error: Input file {input_path} not found") + sys.exit(1) + + try: + with open(input_path) as f: + yaml_data = yaml.safe_load(f) + except yaml.YAMLError as e: + print(f"Error parsing YAML: {e}") + sys.exit(1) + + jsonld_data = yaml_to_jsonld(yaml_data) + + with open(output_path, 'w') as f: + json.dump(jsonld_data, f, indent=2) + + print(f"Converted {input_path} to {output_path}") + + if "--validate" in sys.argv: + print("\nNote: Validation against SHACL schemas not yet implemented") + print("Please use existing validation tools in tests/") + + +if __name__ == "__main__": + main() diff --git a/templates/yaml/README.md b/templates/yaml/README.md new file mode 100644 index 00000000..07e90dcc --- /dev/null +++ b/templates/yaml/README.md @@ -0,0 +1,49 @@ +# YAML Templates for Neuroshapes + +Human-readable templates for manual data entry. These templates can be filled out and converted to JSON-LD for validation. + +## Usage + +1. Copy the relevant template file +2. Fill in your experimental data +3. Convert to JSON-LD using `scripts/yaml_to_jsonld.py` +4. Validate against Neuroshapes schemas + +## Available Templates + +- `subject.yaml` - Subject/animal information +- `slice.yaml` - Brain slice preparation +- `patched_slice.yaml` - Electrophysiology recording +- `reconstructed_cell.yaml` - Complete neuron reconstruction workflow + +## Example + +```bash +# Copy a template +cp templates/yaml/subject.yaml my_experiment.yaml + +# Edit with your data +nano my_experiment.yaml + +# Convert to JSON-LD +python scripts/yaml_to_jsonld.py my_experiment.yaml my_experiment.json + +# Validate (optional) +python scripts/yaml_to_jsonld.py my_experiment.yaml my_experiment.json --validate +``` + +## Why YAML? + +- **Human-readable**: Cleaner syntax than JSON +- **Less error-prone**: No bracket matching issues +- **Compact**: Uses indentation instead of separators +- **JSON-compatible**: YAML is a superset of JSON +- **Better for manual editing**: Easier to see hierarchy + +## Contributing + +When creating new templates: +1. Follow the existing naming convention +2. Include helpful comments for each field +3. Provide example values where appropriate +4. Test conversion to JSON-LD diff --git a/templates/yaml/example_subject.yaml b/templates/yaml/example_subject.yaml new file mode 100644 index 00000000..de1a61de --- /dev/null +++ b/templates/yaml/example_subject.yaml @@ -0,0 +1,11 @@ +# Example: Mouse Hippocampus Experiment + +Subject: + id: "M001" + species: "Mus musculus" + strain: "C57BL/6" + sex: "Male" + age: "P21" + animal_weight: "25g" + date: "2024-03-15" + comment: "Healthy juvenile mouse" diff --git a/templates/yaml/patched_slice.yaml b/templates/yaml/patched_slice.yaml new file mode 100644 index 00000000..d2e91964 --- /dev/null +++ b/templates/yaml/patched_slice.yaml @@ -0,0 +1,14 @@ +# Electrophysiology Recording Template + +PatchedSlice: + protocol: "" + person: "" + date: "" + name: "" + brainLocation: "" + bathSolution: "" + temperature: "" # e.g., "32C" + recordingType: "" # e.g., "whole-cell patch clamp" + intracellularSolution: "" + generated: "" + comment: "" diff --git a/templates/yaml/reconstructed_cell.yaml b/templates/yaml/reconstructed_cell.yaml new file mode 100644 index 00000000..86a4a518 --- /dev/null +++ b/templates/yaml/reconstructed_cell.yaml @@ -0,0 +1,77 @@ +# Complete Neuron Reconstruction Workflow Template +# This template covers the full experimental pipeline from animal to reconstructed cell + +Subject: + id: "" + species: "" + strain: "" + sex: "" + age: "" + animal_weight: "" + date: "" + comment: "" + +Slice: + protocol: "" + person: "" + date: "" + solution: "" + brainLocation: "" + cuttingThickness: "" # e.g., "300um" + generated: "" + hemisphere: "" # "left" or "right" + slicingPlane: "" # e.g., "sagittal", "coronal" + slicingAngle: "" + comment: "" + +PatchedSlice: + protocol: "" + person: "" + date: "" + name: "" + brainLocation: "" + bathSolution: "" + temperature: "" # e.g., "32C" + recordingType: "" + intracellularSolution: "" + generated: "" + comment: "" + +FixedStainedSlice: + protocol: "" + person: "" + date: "" + name: "" + comment: "" + +ImagedSlice: + protocol: "" + person: "" + date: "" + name: "" + generated: "" + comment: "" + +LabeledCell: + name: "" + brainLocation: "" + coordinatesInBrainAtlas: + rostrocaudal: "" + lateral: "" + dorsal: "" + locationInSlice: "" + putativeMType: "" + generated: "" + comment: "" + +ReconstructedCell: + protocol: "" + person: "" + date: "" + name: "" + mType: "" + axonProjection: "" + compressionCorrection: "" + shrinkageCorrection: "" + geometryCorrected: "" + comment: "" diff --git a/templates/yaml/slice.yaml b/templates/yaml/slice.yaml new file mode 100644 index 00000000..5099717b --- /dev/null +++ b/templates/yaml/slice.yaml @@ -0,0 +1,14 @@ +# Brain Slice Preparation Template + +Slice: + protocol: "" + person: "" + date: "" + solution: "" + brainLocation: "" + cuttingThickness: "" # e.g., "300um" + generated: "" + hemisphere: "" # "left" or "right" + slicingPlane: "" # e.g., "sagittal", "coronal", "horizontal" + slicingAngle: "" + comment: "" diff --git a/templates/yaml/subject.yaml b/templates/yaml/subject.yaml new file mode 100644 index 00000000..aa7b97ee --- /dev/null +++ b/templates/yaml/subject.yaml @@ -0,0 +1,28 @@ +# Subject/Animal Metadata Template +# Fill in the fields below with your experimental data +# Remove or leave blank any fields that don't apply to your experiment + +Subject: + # Unique identifier for this subject (required) + id: "" + + # Species name (e.g., "Mus musculus") or taxonomy ID (e.g., "NCBITaxon:10090") + species: "" + + # Genetic strain (e.g., "C57BL/6", "Wistar") + strain: "" + + # Biological sex ("Male" or "Female") + sex: "" + + # Age of subject (e.g., "P21", "8 weeks", "3 months") + age: "" + + # Weight with unit (e.g., "25g", "250g") + animal_weight: "" + + # Date of experiment in ISO format (YYYY-MM-DD) + date: "" + + # Additional notes or comments + comment: "" diff --git a/tests/test_yaml_templates.py b/tests/test_yaml_templates.py new file mode 100644 index 00000000..83931f0b --- /dev/null +++ b/tests/test_yaml_templates.py @@ -0,0 +1,131 @@ +"""Tests for YAML templates and conversion""" +import pytest +import yaml +import json +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent)) +from scripts.yaml_to_jsonld import yaml_to_jsonld, clean_entity, TYPE_MAPPINGS + +TEMPLATES_DIR = Path(__file__).parent.parent / "templates" / "yaml" + + +def test_yaml_templates_exist(): + """Ensure all expected YAML templates are present.""" + expected_templates = [ + "subject.yaml", + "slice.yaml", + "patched_slice.yaml", + "reconstructed_cell.yaml" + ] + + for template in expected_templates: + template_path = TEMPLATES_DIR / template + assert template_path.exists(), f"Missing template: {template}" + + +def test_yaml_templates_valid_syntax(): + """Ensure all YAML templates have valid syntax.""" + for yaml_file in TEMPLATES_DIR.glob("*.yaml"): + if yaml_file.name == "README.md": + continue + with open(yaml_file) as f: + data = yaml.safe_load(f) + assert data is not None, f"Invalid YAML in {yaml_file}" + + +def test_yaml_to_jsonld_conversion(): + """Test conversion from YAML to JSON-LD.""" + sample_yaml = { + "Subject": { + "id": "subject001", + "species": "Mus musculus", + "strain": "C57BL/6" + } + } + + result = yaml_to_jsonld(sample_yaml) + + assert "@context" in result + assert "@graph" in result + assert len(result["@graph"]) == 1 + assert result["@graph"][0]["@type"] == TYPE_MAPPINGS["Subject"] + assert result["@graph"][0]["id"] == "subject001" + + +def test_empty_fields_removed(): + """Test that empty fields are removed during conversion.""" + sample_yaml = { + "Subject": { + "id": "subject001", + "species": "", + "strain": None + } + } + + result = yaml_to_jsonld(sample_yaml) + subject = result["@graph"][0] + + assert "id" in subject + assert "species" not in subject + assert "strain" not in subject + + +def test_clean_entity(): + """Test entity cleaning function.""" + dirty = { + "id": "test", + "empty": "", + "none": None, + "valid": "data", + "nested": { + "kept": "value", + "removed": "" + } + } + + cleaned = clean_entity(dirty) + + assert "id" in cleaned + assert "empty" not in cleaned + assert "none" not in cleaned + assert "valid" in cleaned + assert "nested" in cleaned + assert "kept" in cleaned["nested"] + assert "removed" not in cleaned["nested"] + + +def test_multiple_entities(): + """Test conversion with multiple entity types.""" + sample_yaml = { + "Subject": {"id": "S1", "species": "Mouse"}, + "Slice": {"protocol": "P1", "person": "Researcher"} + } + + result = yaml_to_jsonld(sample_yaml) + + assert len(result["@graph"]) == 2 + types = [entity["@type"] for entity in result["@graph"]] + assert TYPE_MAPPINGS["Subject"] in types + assert TYPE_MAPPINGS["Slice"] in types + + +def test_nested_coordinates(): + """Test conversion with nested structures like coordinates.""" + sample_yaml = { + "LabeledCell": { + "name": "cell01", + "coordinatesInBrainAtlas": { + "rostrocaudal": "1.2", + "lateral": "2.3", + "dorsal": "3.4" + } + } + } + + result = yaml_to_jsonld(sample_yaml) + cell = result["@graph"][0] + + assert "coordinatesInBrainAtlas" in cell + assert cell["coordinatesInBrainAtlas"]["rostrocaudal"] == "1.2"