Skip to content

Manage iodef core no_del tweak_iodef #447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mo-marqh wants to merge 18 commits into
base: main
Choose a base branch
from
Open

Manage iodef core no_del tweak_iodef #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
cad0368
remove tweak-iodef
mo-marqh Jul 9, 2026
b878852
sym-links for metadata and etc
mo-marqh Jul 9, 2026
48f0d56
XML consolidation
mo-marqh Jul 9, 2026
4ef7d4a
xml example consistency
mo-marqh Jul 9, 2026
f627be2
set up symlinks using rose-app.conf
mo-marqh Jul 13, 2026
a9b2218
restore relative paths and add example etc symlinks
mo-marqh Jul 13, 2026
2375504
pytest iodef.xml
mo-marqh Jul 13, 2026
be7dd52
cylc test configuration & code quality linting
mo-marqh Jul 14, 2026
45815bc
config and linting
mo-marqh Jul 14, 2026
39d63ad
test configurations
mo-marqh Jul 14, 2026
8362d36
f
mo-marqh Jul 14, 2026
d97caaf
b0rked :(
mo-marqh Jul 14, 2026
205129b
refactor shared xios contexts and rose confs
mo-marqh Jul 15, 2026
190a60e
XIOS .xml docs
mo-marqh Jul 15, 2026
296eccb
restore tweak_iodef temporarily, to decouple PRs
mo-marqh Aug 13, 2026
3ba37cf
return example iodef files to using explicit xios context configurati…
mo-marqh Aug 19, 2026
eef6855
improve pytest code for review
mo-marqh Aug 19, 2026
d7ab9c7
docs linting
mo-marqh Aug 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions applications/io_demo/example/iodef.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

<context id = "io_demo">

<axis_definition src="../metadata/axis_dictionary.xml"/>
<grid_definition src="../metadata/grid_dictionary.xml"/>
<field_definition src="../metadata/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition
type="one_file"
Expand Down
1 change: 1 addition & 0 deletions applications/io_demo/example/metadata
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../metadata/
2 changes: 1 addition & 1 deletion applications/io_demo/metadata/field_dictionary.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
<field_group id="io_demo_fields" prec="8" operation="instant">
<field id="diffusion_field" name="diffusion_field" long_name="diffusion_field" unit="1" grid_ref="full_level_face_grid" />
</field_group>
</field_definiton>
</field_definition>
12 changes: 6 additions & 6 deletions applications/lbc_demo/example/iodef.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
<simulation>

<context id = "lbc_demo:diags">
<axis_definition src="../metadata/axis_dictionary.xml"/>
<grid_definition src="../metadata/grid_dictionary.xml"/>
<field_definition src="../metadata/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition type="one_file" time_counter="none">
<file id="diagnostics" name="diagnostics" output_freq="1ts" convention="UGRID" enabled=".TRUE.">
Expand All @@ -24,9 +24,9 @@


<context id = "lbc_demo:lbcs">
<axis_definition src="../metadata/axis_dictionary.xml"/>
<grid_definition src="../metadata/grid_dictionary.xml"/>
<field_definition src="../metadata/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition type="one_file" time_counter="none">
<file id="lbcs_out" name="lbcs_out" output_freq="1ts" convention="UGRID" enabled=".TRUE.">
Expand Down
1 change: 1 addition & 0 deletions applications/lbc_demo/example/metadata
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../metadata/
2 changes: 1 addition & 1 deletion applications/lbc_demo/metadata/field_dictionary.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@
<field id="w3_2d_field:lbc" name="w3_2d_field" long_name="w3_2d_field" unit="1" domain_ref="face" />
<field id="multi_data_field:lbc" name="multi_data_field" long_name="multi_data_field" unit="1" domain_ref="face" axis_ref="multi_data_axis" />
</field_group>
</field_definiton>
</field_definition>
6 changes: 3 additions & 3 deletions applications/simple_diffusion/example/iodef.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

<context id = "simple_diffusion">

<axis_definition src="../metadata/axis_dictionary.xml"/>
<grid_definition src="../metadata/grid_dictionary.xml"/>
<field_definition src="../metadata/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition
type="one_file"
Expand Down
1 change: 1 addition & 0 deletions applications/simple_diffusion/example/metadata
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../metadata/
2 changes: 1 addition & 1 deletion applications/simple_diffusion/metadata/field_dictionary.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
<field_group id="simple_diffusion_fields" prec="8" operation="instant">
<field id="diffusion_field" name="diffusion_field" long_name="diffusion_field" unit="1" grid_ref="full_level_face_grid" />
</field_group>
</field_definiton>
</field_definition>
14 changes: 10 additions & 4 deletions documentation/source/how_to_use_it/io/io_standards.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,22 @@ Standards for XIOS Iodef Files
section of every top-level iodef file.
#. Non-global file settings are to be specified at the level of
individual files.
#. ``id`` values shall be unique for a given ``XIOS`` ``XML`` element
type.

**Rationale**

.. warning::

When XIOS aggregates file definitions - possibly spread across
several XML files -, the last set of attributes processed "wins",
in the sense that all the others are silently ignored.
When XIOS aggregates definitions - possibly spread across
several XML files -, then attributes are overriden by defined elements
that use a ``_ref``.

This can create considerable confusion and frustration for users.
The overriding operates in a chain, so users may need to act cautiously
to understand attribute inheritance and overrides.

This is why reuse of ``id`` values for different elements is problematic,
not allowed, and tested by a code inspection rule for ``field`` elements.

With the current layout of the LFRic top-level iodef files, the last
set of attributes to be processed are the ones in the local file
Expand Down
9 changes: 9 additions & 0 deletions documentation/source/how_to_use_it/io/prognostics.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,3 +286,12 @@ to other model requirements including models that do not use XIOS.
checkpoint flag setting, to the ``checkpoint_fields`` field collection.
Additionally, like all other prognostic fields, it is added to the
``depository`` field collection.

``iodef.xml`` files may reference further ``.xml`` files using the
``src=""`` attribute. File paths in the source tree are maintained
using symbolic links, which are dereferenced by ``rose-app.conf``
to enable running tests. All links must resolve within the source,
and validation is implemented in a test called ``validate_iodef``.
Only the shared XIOS runtime configuration files within ``/etc``
are updated to use environment variables, these should not be used
within other ``.xml`` fragments within the ``lfric_core`` code base.
14 changes: 14 additions & 0 deletions etc/xios.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<context id = "xios">
<variable_definition>
<variable_group id = "buffer">
<variable id = "optimal_buffer_size" type = "string" >performance</variable>
</variable_group>

<variable_group id = "parameters" >
<variable id = "using_server" type = "bool" >$XIOS_SERVER_MODE_XML</variable>
<variable id = "info_level" type = "int" >50</variable>
<variable id = "print_file" type="bool">true</variable>
</variable_group>
</variable_definition>
</context>

16 changes: 16 additions & 0 deletions etc/xios_coupled.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<context id = "xios">
<variable_definition>
<variable_group id = "buffer">
<variable id = "optimal_buffer_size" type = "string" >performance</variable>
</variable_group>

<variable_group id = "parameters" >
<variable id = "using_server" type = "bool" >$XIOS_SERVER_MODE_XML</variable>
<variable id = "info_level" type = "int" >50</variable>
<variable id = "print_file" type="bool">true</variable>
<variable id="using_oasis" type="bool">true</variable>
<variable id="oasis_codes_id" type="string" >lfric</variable>
</variable_group>
</variable_definition>
</context>

8 changes: 4 additions & 4 deletions rose-stem/app/canned_test/rose-app.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[command]
aprun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
aprun=eval $CANNED_PRESCRIPT ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
default=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
mpiexec=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
mpiexec_cpl=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i
srun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
mpiexec=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
mpiexec_cpl=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i
srun=eval $CANNED_PRESCRIPT ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE}
1 change: 1 addition & 0 deletions rose-stem/app/coupled/file/etc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../etc/
16 changes: 1 addition & 15 deletions rose-stem/app/coupled/file/iodef.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,20 +55,6 @@

</context>

<context id = "xios">
<variable_definition>
<variable_group id = "buffer">
<variable id = "optimal_buffer_size" type = "string" >performance</variable>
</variable_group>

<variable_group id = "parameters" >
<variable id = "using_server" type = "bool" >false</variable>
<variable id = "info_level" type = "int" >50</variable>
<variable id = "print_file" type="bool">true</variable>
<variable id="using_oasis" type="bool">true</variable>
<variable id="oasis_codes_id" type="string" >lfric</variable>
</variable_group>
</variable_definition>
</context>
<context src="etc/xios_coupled.xml"/>

</simulation>
7 changes: 6 additions & 1 deletion rose-stem/app/coupled/rose-app.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
meta=lfric-coupled/vn3.2

[command]
default=rose app-run -C $CYLC_WORKFLOW_RUN_DIR/app/coupled -O lam -O LAM50x50-2x2 --install-only
default=rose env-cat -o etc/updated_xios.xml etc/xios_coupled.xml
=mv etc/updated_xios.xml etc/xios_coupled.xml
=rose app-run -C $CYLC_WORKFLOW_RUN_DIR/app/coupled -O lam -O LAM50x50-2x2 --install-only
=rose app-run -C $CYLC_WORKFLOW_RUN_DIR/app/coupled -O glo -O C12 --install-only
=$LAUNCH_SCRIPT/cpl-launch-exe

Expand All @@ -27,6 +29,9 @@ source=namelist:base_mesh
= namelist:time
= namelist:timestepping

[file:etc]
source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc

[namelist:base_mesh]
f_lat_deg=45.0
file_prefix='mesh'
Expand Down
1 change: 1 addition & 0 deletions rose-stem/app/io_demo/file/etc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../etc/
20 changes: 4 additions & 16 deletions rose-stem/app/io_demo/file/iodef.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,11 @@

<context id = "io_demo">

<axis_definition src="$METADATA/axis_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>

<grid_definition src="$METADATA/grid_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>

<field_definition src="$METADATA/field_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition
type="one_file"
Expand All @@ -21,18 +21,6 @@

</context>

<context id = "xios">
<variable_definition>
<variable_group id = "buffer">
<variable id = "optimal_buffer_size" type = "string" >performance</variable>
</variable_group>

<variable_group id = "parameters" >
<variable id = "using_server" type = "bool" >true</variable>
<variable id = "info_level" type = "int" >50</variable>
<variable id = "print_file" type="bool">true</variable>
</variable_group>
</variable_definition>
</context>
<context src="etc/xios.xml"/>

</simulation>
20 changes: 4 additions & 16 deletions rose-stem/app/io_demo/file/iodef_temporal.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

<context id = "io_demo">

<axis_definition src="$METADATA/axis_dictionary.xml"/>
<grid_definition src="$METADATA/grid_dictionary.xml"/>
<field_definition src="$METADATA/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition
type="one_file"
Expand All @@ -22,18 +22,6 @@
<!-- Pull in partial context definition for temporal I/O from XML fragment -->
<context id="temporal_context" src="./temporal_context.xml"/>

<context id = "xios">
<variable_definition>
<variable_group id = "buffer">
<variable id = "optimal_buffer_size" type = "string" >performance</variable>
</variable_group>

<variable_group id = "parameters" >
<variable id = "using_server" type = "bool" >true</variable>
<variable id = "info_level" type = "int" >50</variable>
<variable id = "print_file" type="bool">true</variable>
</variable_group>
</variable_definition>
</context>
<context src="etc/xios.xml"/>

</simulation>
1 change: 1 addition & 0 deletions rose-stem/app/io_demo/file/metadata
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../applications/io_demo/metadata/
4 changes: 2 additions & 2 deletions rose-stem/app/io_demo/file/temporal_context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@

<context id="temporal_context">

<axis_definition src="$METADATA/axis_dictionary.xml"/>
<grid_definition src="$METADATA/grid_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>

<file_definition
type="one_file"
Expand Down
5 changes: 0 additions & 5 deletions rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
[command]
default=$CORE_ROOT_DIR/bin/tweak_iodef ; \
=$CORE_ROOT_DIR/bin/tweak_iodef temporal_context.xml; \
=$LAUNCH_SCRIPT/launch-exe

[file:iodef.xml]
mode=auto
source=$ROSE_SUITE_DIR/app/io_demo/file/iodef_temporal.xml
Expand Down
9 changes: 8 additions & 1 deletion rose-stem/app/io_demo/rose-app.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
meta=lfric-io_demo/vn3.2

[command]
default=$CORE_ROOT_DIR/bin/tweak_iodef ; \
default=rose env-cat -o etc/updated_xios.xml etc/xios.xml
=mv etc/updated_xios.xml etc/xios.xml
=$LAUNCH_SCRIPT/launch-exe

[env]
Expand Down Expand Up @@ -29,6 +30,12 @@ source=namelist:base_mesh
= namelist:time
= namelist:timestepping

[file:etc]
source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc

[file:metadata]
source=$METADATA

[namelist:base_mesh]
f_lat_deg=45.0
file_prefix=''
Expand Down
1 change: 1 addition & 0 deletions rose-stem/app/lbc_demo/file/etc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../etc/
26 changes: 7 additions & 19 deletions rose-stem/app/lbc_demo/file/iodef.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
<simulation>

<context id = "lbc_demo:diags">
<axis_definition src="$METADATA/axis_dictionary.xml"/>
<grid_definition src="$METADATA/grid_dictionary.xml"/>
<field_definition src="$METADATA/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition type="one_file" time_counter="none">
<file id="diagnostics" name="diagnostics" output_freq="1ts" convention="UGRID" enabled=".TRUE.">
Expand All @@ -23,9 +23,9 @@
</context>

<context id = "lbc_demo:lbcs">
<axis_definition src="$METADATA/axis_dictionary.xml"/>
<grid_definition src="$METADATA/grid_dictionary.xml"/>
<field_definition src="$METADATA/field_dictionary.xml"/>
<axis_definition src="metadata/axis_dictionary.xml"/>
<grid_definition src="metadata/grid_dictionary.xml"/>
<field_definition src="metadata/field_dictionary.xml"/>

<file_definition type="one_file" time_counter="none">
<file id="lbcs_out" name="lbcs_out" output_freq="1ts" convention="UGRID" enabled=".TRUE.">
Expand All @@ -43,18 +43,6 @@
</file_definition>
</context>

<context id = "xios">
<variable_definition>
<variable_group id = "buffer">
<variable id = "optimal_buffer_size" type = "string" >performance</variable>
</variable_group>

<variable_group id = "parameters" >
<variable id = "using_server" type = "bool" >true</variable>
<variable id = "info_level" type = "int" >50</variable>
<variable id = "print_file" type="bool">true</variable>
</variable_group>
</variable_definition>
</context>
<context src="etc/xios.xml"/>

</simulation>
1 change: 1 addition & 0 deletions rose-stem/app/lbc_demo/file/metadata
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../../../applications/lbc_demo/metadata/
Loading
Loading