From cad0368782ea10f39234a090c6541bd1d7201860 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:05:43 +0000 Subject: [PATCH 01/18] remove tweak-iodef --- bin/tweak_iodef | 38 ------------------- rose-stem/app/canned_test/rose-app.conf | 10 ++--- .../opt/rose-app-temporal_reading.conf | 4 +- rose-stem/app/io_demo/rose-app.conf | 3 +- rose-stem/app/lbc_demo/rose-app.conf | 3 +- rose-stem/app/simple_diffusion/rose-app.conf | 3 +- 6 files changed, 9 insertions(+), 52 deletions(-) delete mode 100755 bin/tweak_iodef diff --git a/bin/tweak_iodef b/bin/tweak_iodef deleted file mode 100755 index aa846d67f..000000000 --- a/bin/tweak_iodef +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -############################################################################### -# (c) Crown copyright 2024 Met Office. All rights reserved. -# The file LICENCE, distributed with this code, contains details of the terms -# under which the code may be used. -############################################################################### -# tweak_iodef: -# replace relative path to LFRic dictionary with env symbol and -# resolve env symbols -# in iodef.xml before rose suite starts. - -iodef=iodef.xml -case $# in -0) ;; -1) iodef=$1 ;; -*) echo 'usage: tweak_iodef [iodef_file]' 1>&2; exit 2 -esac - -set -e -temp_file=$(mktemp) -trap 'rm -f $temp_file; exit 1' INT TERM EXIT -# examples substitutions -# ../../../lfric_atm/metadata used in gungho_model/example -# ../metadata -> $METADATA used in lfric_atm/example -if sed 's@\.\./\(\.\./\)*lfric_atm/metadata@$METADATA@' $iodef |\ - sed 's@\.\./\(\.\./\)*metadata@$METADATA@' |\ - rose env-cat - >$temp_file - echo "tweak_iodef: editing sucessful in $temp_file" -then - trap '' INT TERM EXIT - cp $temp_file $iodef - echo "tweak_iodef: $iodef saved with changes" -else - echo "tweak_iodef: editing failed, $iodef unchanged" 1>&2 - rm -f $temp_file - exit 1 -fi -rm -f $temp_file diff --git a/rose-stem/app/canned_test/rose-app.conf b/rose-stem/app/canned_test/rose-app.conf index 2934ac1b2..035465e98 100644 --- a/rose-stem/app/canned_test/rose-app.conf +++ b/rose-stem/app/canned_test/rose-app.conf @@ -1,6 +1,6 @@ [command] -aprun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -default=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -mpiexec=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -mpiexec_cpl=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i -srun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +aprun=eval $CANNED_PRESCRIPT ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +default=eval $CANNED_PRESCRIPT ; ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +mpiexec=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +mpiexec_cpl=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i +srun=eval $CANNED_PRESCRIPT ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} diff --git a/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf b/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf index f836f27d6..91aefcd57 100644 --- a/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf +++ b/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf @@ -1,7 +1,5 @@ [command] -default=$CORE_ROOT_DIR/bin/tweak_iodef ; \ - =$CORE_ROOT_DIR/bin/tweak_iodef temporal_context.xml; \ - =$LAUNCH_SCRIPT/launch-exe +default=$LAUNCH_SCRIPT/launch-exe [file:iodef.xml] mode=auto diff --git a/rose-stem/app/io_demo/rose-app.conf b/rose-stem/app/io_demo/rose-app.conf index afc0f3e80..63c8623a8 100644 --- a/rose-stem/app/io_demo/rose-app.conf +++ b/rose-stem/app/io_demo/rose-app.conf @@ -1,8 +1,7 @@ meta=lfric-io_demo/vn3.2 [command] -default=$CORE_ROOT_DIR/bin/tweak_iodef ; \ - =$LAUNCH_SCRIPT/launch-exe +default=$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=io_demo diff --git a/rose-stem/app/lbc_demo/rose-app.conf b/rose-stem/app/lbc_demo/rose-app.conf index 69ad1de7e..7daca2db7 100644 --- a/rose-stem/app/lbc_demo/rose-app.conf +++ b/rose-stem/app/lbc_demo/rose-app.conf @@ -1,8 +1,7 @@ meta=lfric-lbc_demo/vn3.2 [command] -default=$CORE_ROOT_DIR/bin/tweak_iodef ; \ - =$LAUNCH_SCRIPT/launch-exe +default=$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=lbc_demo diff --git a/rose-stem/app/simple_diffusion/rose-app.conf b/rose-stem/app/simple_diffusion/rose-app.conf index c4ad299fc..45ea26cf8 100644 --- a/rose-stem/app/simple_diffusion/rose-app.conf +++ b/rose-stem/app/simple_diffusion/rose-app.conf @@ -1,8 +1,7 @@ meta=lfric-simple_diffusion/vn3.2 [command] -default=$CORE_ROOT_DIR/bin/tweak_iodef ; \ - =$LAUNCH_SCRIPT/launch-exe +default=$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=simple_diffusion From b878852ad06edf382bc33fbeddca8ea0f21c51cf Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:16:56 +0000 Subject: [PATCH 02/18] sym-links for metadata and etc --- rose-stem/app/coupled/file/etc | 1 + rose-stem/app/io_demo/file/etc | 1 + rose-stem/app/io_demo/file/metadata | 1 + rose-stem/app/lbc_demo/file/etc | 1 + rose-stem/app/lbc_demo/file/metadata | 1 + rose-stem/app/simple_diffusion/file/etc | 1 + rose-stem/app/simple_diffusion/file/metadata | 1 + rose-stem/app/skeleton/file/etc | 1 + 8 files changed, 8 insertions(+) create mode 120000 rose-stem/app/coupled/file/etc create mode 120000 rose-stem/app/io_demo/file/etc create mode 120000 rose-stem/app/io_demo/file/metadata create mode 120000 rose-stem/app/lbc_demo/file/etc create mode 120000 rose-stem/app/lbc_demo/file/metadata create mode 120000 rose-stem/app/simple_diffusion/file/etc create mode 120000 rose-stem/app/simple_diffusion/file/metadata create mode 120000 rose-stem/app/skeleton/file/etc diff --git a/rose-stem/app/coupled/file/etc b/rose-stem/app/coupled/file/etc new file mode 120000 index 000000000..07008abc1 --- /dev/null +++ b/rose-stem/app/coupled/file/etc @@ -0,0 +1 @@ +../../../../etc/ \ No newline at end of file diff --git a/rose-stem/app/io_demo/file/etc b/rose-stem/app/io_demo/file/etc new file mode 120000 index 000000000..07008abc1 --- /dev/null +++ b/rose-stem/app/io_demo/file/etc @@ -0,0 +1 @@ +../../../../etc/ \ No newline at end of file diff --git a/rose-stem/app/io_demo/file/metadata b/rose-stem/app/io_demo/file/metadata new file mode 120000 index 000000000..9b32a1c6e --- /dev/null +++ b/rose-stem/app/io_demo/file/metadata @@ -0,0 +1 @@ +../../../../applications/io_demo/metadata/ \ No newline at end of file diff --git a/rose-stem/app/lbc_demo/file/etc b/rose-stem/app/lbc_demo/file/etc new file mode 120000 index 000000000..07008abc1 --- /dev/null +++ b/rose-stem/app/lbc_demo/file/etc @@ -0,0 +1 @@ +../../../../etc/ \ No newline at end of file diff --git a/rose-stem/app/lbc_demo/file/metadata b/rose-stem/app/lbc_demo/file/metadata new file mode 120000 index 000000000..648c201f7 --- /dev/null +++ b/rose-stem/app/lbc_demo/file/metadata @@ -0,0 +1 @@ +../../../../applications/lbc_demo/metadata/ \ No newline at end of file diff --git a/rose-stem/app/simple_diffusion/file/etc b/rose-stem/app/simple_diffusion/file/etc new file mode 120000 index 000000000..07008abc1 --- /dev/null +++ b/rose-stem/app/simple_diffusion/file/etc @@ -0,0 +1 @@ +../../../../etc/ \ No newline at end of file diff --git a/rose-stem/app/simple_diffusion/file/metadata b/rose-stem/app/simple_diffusion/file/metadata new file mode 120000 index 000000000..d151871be --- /dev/null +++ b/rose-stem/app/simple_diffusion/file/metadata @@ -0,0 +1 @@ +../../../../applications/simple_diffusion/metadata/ \ No newline at end of file diff --git a/rose-stem/app/skeleton/file/etc b/rose-stem/app/skeleton/file/etc new file mode 120000 index 000000000..07008abc1 --- /dev/null +++ b/rose-stem/app/skeleton/file/etc @@ -0,0 +1 @@ +../../../../etc/ \ No newline at end of file From 48f0d564fc98255b1b0a6acee1f3ad5e9fbdf2df Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:17:26 +0000 Subject: [PATCH 03/18] XML consolidation --- etc/xios.xml | 14 ++++++++++ rose-stem/app/coupled/file/iodef.xml | 16 +----------- rose-stem/app/io_demo/file/iodef.xml | 20 +++----------- rose-stem/app/io_demo/file/iodef_temporal.xml | 20 +++----------- .../app/io_demo/file/temporal_context.xml | 4 +-- rose-stem/app/lbc_demo/file/iodef.xml | 26 +++++-------------- rose-stem/app/simple_diffusion/file/iodef.xml | 20 +++----------- rose-stem/app/skeleton/file/iodef.xml | 14 +--------- .../runtime/generate_runtime_application.cylc | 2 ++ 9 files changed, 39 insertions(+), 97 deletions(-) create mode 100644 etc/xios.xml diff --git a/etc/xios.xml b/etc/xios.xml new file mode 100644 index 000000000..a970c070b --- /dev/null +++ b/etc/xios.xml @@ -0,0 +1,14 @@ + + + + performance + + + + true + 50 + true + + + + diff --git a/rose-stem/app/coupled/file/iodef.xml b/rose-stem/app/coupled/file/iodef.xml index d32b67fb6..fe7ebf7a1 100644 --- a/rose-stem/app/coupled/file/iodef.xml +++ b/rose-stem/app/coupled/file/iodef.xml @@ -55,20 +55,6 @@ - - - - performance - - - - false - 50 - true - true - lfric - - - + diff --git a/rose-stem/app/io_demo/file/iodef.xml b/rose-stem/app/io_demo/file/iodef.xml index 442c373ce..0f3626f95 100644 --- a/rose-stem/app/io_demo/file/iodef.xml +++ b/rose-stem/app/io_demo/file/iodef.xml @@ -3,11 +3,11 @@ - + - + - + - - - - performance - - - - true - 50 - true - - - + diff --git a/rose-stem/app/io_demo/file/iodef_temporal.xml b/rose-stem/app/io_demo/file/iodef_temporal.xml index 77e38b572..16dd8919e 100644 --- a/rose-stem/app/io_demo/file/iodef_temporal.xml +++ b/rose-stem/app/io_demo/file/iodef_temporal.xml @@ -3,9 +3,9 @@ - - - + + + - - - - performance - - - - true - 50 - true - - - + diff --git a/rose-stem/app/io_demo/file/temporal_context.xml b/rose-stem/app/io_demo/file/temporal_context.xml index af51348d0..fb7e2aee6 100644 --- a/rose-stem/app/io_demo/file/temporal_context.xml +++ b/rose-stem/app/io_demo/file/temporal_context.xml @@ -1,8 +1,8 @@ - - + + - - - + + + @@ -23,9 +23,9 @@ - - - + + + @@ -43,18 +43,6 @@ - - - - performance - - - - true - 50 - true - - - + diff --git a/rose-stem/app/simple_diffusion/file/iodef.xml b/rose-stem/app/simple_diffusion/file/iodef.xml index d3db83f7a..5192970cc 100644 --- a/rose-stem/app/simple_diffusion/file/iodef.xml +++ b/rose-stem/app/simple_diffusion/file/iodef.xml @@ -3,11 +3,11 @@ - + - + - + - - - - performance - - - - true - 50 - true - - - + diff --git a/rose-stem/app/skeleton/file/iodef.xml b/rose-stem/app/skeleton/file/iodef.xml index 786cfffa0..0c590d865 100644 --- a/rose-stem/app/skeleton/file/iodef.xml +++ b/rose-stem/app/skeleton/file/iodef.xml @@ -55,18 +55,6 @@ - - - - performance - - - - true - 50 - true - - - + diff --git a/rose-stem/templates/runtime/generate_runtime_application.cylc b/rose-stem/templates/runtime/generate_runtime_application.cylc index ce88cf3db..7bce39f13 100644 --- a/rose-stem/templates/runtime/generate_runtime_application.cylc +++ b/rose-stem/templates/runtime/generate_runtime_application.cylc @@ -22,6 +22,8 @@ {# Generate the script command #} {% set script_str = namespace(name="") %} {% set script_str.name = "$CYLC_WORKFLOW_RUN_DIR/bin/application_results_setup.sh ; " %} +{% set script_str.name = script_str.name~"ln -sf $METADATA $CYLC_TASK_WORK_DIR/metadata ; " %} +{% set script_str.name = script_str.name~"ln -sf $CORE_ROOT_DIR/etc $CYLC_TASK_WORK_DIR/etc ; " %} {% set script_str.name = script_str.name~"rose task-run --app-key="~task_values["app_name"] %} {% set env_application = task_ns.application %} From 4ef7d4ac2220a0bb05b623cd17998d894b3ebe99 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:37:52 +0000 Subject: [PATCH 04/18] xml example consistency --- applications/coupled/example/iodef.xml | 16 +----------- applications/io_demo/example/iodef.xml | 20 +++----------- applications/lbc_demo/example/iodef.xml | 26 +++++-------------- .../simple_diffusion/example/iodef.xml | 20 +++----------- applications/skeleton/example/iodef.xml | 14 +--------- 5 files changed, 17 insertions(+), 79 deletions(-) diff --git a/applications/coupled/example/iodef.xml b/applications/coupled/example/iodef.xml index d32b67fb6..fe7ebf7a1 100644 --- a/applications/coupled/example/iodef.xml +++ b/applications/coupled/example/iodef.xml @@ -55,20 +55,6 @@ - - - - performance - - - - false - 50 - true - true - lfric - - - + diff --git a/applications/io_demo/example/iodef.xml b/applications/io_demo/example/iodef.xml index c4deecb41..238a45351 100644 --- a/applications/io_demo/example/iodef.xml +++ b/applications/io_demo/example/iodef.xml @@ -3,9 +3,9 @@ - - - + + + - - - - performance - - - - true - 50 - true - - - + diff --git a/applications/lbc_demo/example/iodef.xml b/applications/lbc_demo/example/iodef.xml index 47855ec1f..4ee03ccc7 100644 --- a/applications/lbc_demo/example/iodef.xml +++ b/applications/lbc_demo/example/iodef.xml @@ -2,9 +2,9 @@ - - - + + + @@ -24,9 +24,9 @@ - - - + + + @@ -44,18 +44,6 @@ - - - - performance - - - - true - 50 - true - - - + diff --git a/applications/simple_diffusion/example/iodef.xml b/applications/simple_diffusion/example/iodef.xml index 39033c256..3193315e5 100644 --- a/applications/simple_diffusion/example/iodef.xml +++ b/applications/simple_diffusion/example/iodef.xml @@ -3,9 +3,9 @@ - - - + + + - - - - performance - - - - true - 50 - true - - - + diff --git a/applications/skeleton/example/iodef.xml b/applications/skeleton/example/iodef.xml index 5b2978a4d..ad8e08461 100644 --- a/applications/skeleton/example/iodef.xml +++ b/applications/skeleton/example/iodef.xml @@ -55,18 +55,6 @@ - - - - performance - - - - true - 50 - true - - - + From f627be2aa16aa8f1aa0e6802e088dde882857342 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:08:39 +0000 Subject: [PATCH 05/18] set up symlinks using rose-app.conf --- rose-stem/app/coupled/rose-app.conf | 4 ++++ rose-stem/app/io_demo/rose-app.conf | 8 ++++++++ rose-stem/app/lbc_demo/rose-app.conf | 8 ++++++++ rose-stem/app/simple_diffusion/rose-app.conf | 8 ++++++++ rose-stem/app/skeleton/rose-app.conf | 4 ++++ .../templates/runtime/generate_runtime_application.cylc | 2 -- 6 files changed, 32 insertions(+), 2 deletions(-) diff --git a/rose-stem/app/coupled/rose-app.conf b/rose-stem/app/coupled/rose-app.conf index 2b402b3c8..2d208e395 100644 --- a/rose-stem/app/coupled/rose-app.conf +++ b/rose-stem/app/coupled/rose-app.conf @@ -15,6 +15,10 @@ TOTAL_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [file:configuration_glo.nml] mode=auto source=namelist:base_mesh diff --git a/rose-stem/app/io_demo/rose-app.conf b/rose-stem/app/io_demo/rose-app.conf index 63c8623a8..493e05a67 100644 --- a/rose-stem/app/io_demo/rose-app.conf +++ b/rose-stem/app/io_demo/rose-app.conf @@ -13,6 +13,14 @@ XIOS_SERVER_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir +[file:metadata] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/io_demo/metadata + +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [file:configuration.nml] mode=auto source=namelist:base_mesh diff --git a/rose-stem/app/lbc_demo/rose-app.conf b/rose-stem/app/lbc_demo/rose-app.conf index 7daca2db7..413746421 100644 --- a/rose-stem/app/lbc_demo/rose-app.conf +++ b/rose-stem/app/lbc_demo/rose-app.conf @@ -13,6 +13,14 @@ XIOS_SERVER_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir +[file:metadata] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/lbc_demo/metadata + +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [file:configuration.nml] mode=auto source=namelist:base_mesh diff --git a/rose-stem/app/simple_diffusion/rose-app.conf b/rose-stem/app/simple_diffusion/rose-app.conf index 45ea26cf8..0093742ab 100644 --- a/rose-stem/app/simple_diffusion/rose-app.conf +++ b/rose-stem/app/simple_diffusion/rose-app.conf @@ -13,6 +13,14 @@ XIOS_SERVER_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir +[file:metadata] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/simple_diffusion/metadata + +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [file:configuration.nml] mode=auto source=namelist:base_mesh diff --git a/rose-stem/app/skeleton/rose-app.conf b/rose-stem/app/skeleton/rose-app.conf index 41784ce12..e74403fde 100644 --- a/rose-stem/app/skeleton/rose-app.conf +++ b/rose-stem/app/skeleton/rose-app.conf @@ -13,6 +13,10 @@ TOTAL_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [file:configuration.nml] mode=auto source=namelist:base_mesh diff --git a/rose-stem/templates/runtime/generate_runtime_application.cylc b/rose-stem/templates/runtime/generate_runtime_application.cylc index 7bce39f13..ce88cf3db 100644 --- a/rose-stem/templates/runtime/generate_runtime_application.cylc +++ b/rose-stem/templates/runtime/generate_runtime_application.cylc @@ -22,8 +22,6 @@ {# Generate the script command #} {% set script_str = namespace(name="") %} {% set script_str.name = "$CYLC_WORKFLOW_RUN_DIR/bin/application_results_setup.sh ; " %} -{% set script_str.name = script_str.name~"ln -sf $METADATA $CYLC_TASK_WORK_DIR/metadata ; " %} -{% set script_str.name = script_str.name~"ln -sf $CORE_ROOT_DIR/etc $CYLC_TASK_WORK_DIR/etc ; " %} {% set script_str.name = script_str.name~"rose task-run --app-key="~task_values["app_name"] %} {% set env_application = task_ns.application %} From a9b22189cbb3f6437b8ee176b6210b2dc03d6bd1 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:29:01 +0000 Subject: [PATCH 06/18] restore relative paths and add example etc symlinks --- applications/coupled/example/etc | 1 + applications/io_demo/example/etc | 1 + applications/io_demo/example/iodef.xml | 6 +++--- applications/io_demo/metadata/field_dictionary.xml | 2 +- applications/lbc_demo/example/etc | 1 + applications/lbc_demo/example/iodef.xml | 12 ++++++------ applications/lbc_demo/metadata/field_dictionary.xml | 2 +- applications/simple_diffusion/example/etc | 1 + applications/simple_diffusion/example/iodef.xml | 6 +++--- .../simple_diffusion/metadata/field_dictionary.xml | 2 +- applications/skeleton/example/etc | 1 + 11 files changed, 20 insertions(+), 15 deletions(-) create mode 120000 applications/coupled/example/etc create mode 120000 applications/io_demo/example/etc create mode 120000 applications/lbc_demo/example/etc create mode 120000 applications/simple_diffusion/example/etc create mode 120000 applications/skeleton/example/etc diff --git a/applications/coupled/example/etc b/applications/coupled/example/etc new file mode 120000 index 000000000..5bed95183 --- /dev/null +++ b/applications/coupled/example/etc @@ -0,0 +1 @@ +../../../etc/ \ No newline at end of file diff --git a/applications/io_demo/example/etc b/applications/io_demo/example/etc new file mode 120000 index 000000000..5bed95183 --- /dev/null +++ b/applications/io_demo/example/etc @@ -0,0 +1 @@ +../../../etc/ \ No newline at end of file diff --git a/applications/io_demo/example/iodef.xml b/applications/io_demo/example/iodef.xml index 238a45351..35deedcc6 100644 --- a/applications/io_demo/example/iodef.xml +++ b/applications/io_demo/example/iodef.xml @@ -3,9 +3,9 @@ - - - + + + - \ No newline at end of file + diff --git a/applications/lbc_demo/example/etc b/applications/lbc_demo/example/etc new file mode 120000 index 000000000..5bed95183 --- /dev/null +++ b/applications/lbc_demo/example/etc @@ -0,0 +1 @@ +../../../etc/ \ No newline at end of file diff --git a/applications/lbc_demo/example/iodef.xml b/applications/lbc_demo/example/iodef.xml index 4ee03ccc7..fb1266979 100644 --- a/applications/lbc_demo/example/iodef.xml +++ b/applications/lbc_demo/example/iodef.xml @@ -2,9 +2,9 @@ - - - + + + @@ -24,9 +24,9 @@ - - - + + + diff --git a/applications/lbc_demo/metadata/field_dictionary.xml b/applications/lbc_demo/metadata/field_dictionary.xml index 103f7e3f5..21a9c8d57 100644 --- a/applications/lbc_demo/metadata/field_dictionary.xml +++ b/applications/lbc_demo/metadata/field_dictionary.xml @@ -19,4 +19,4 @@ - + diff --git a/applications/simple_diffusion/example/etc b/applications/simple_diffusion/example/etc new file mode 120000 index 000000000..5bed95183 --- /dev/null +++ b/applications/simple_diffusion/example/etc @@ -0,0 +1 @@ +../../../etc/ \ No newline at end of file diff --git a/applications/simple_diffusion/example/iodef.xml b/applications/simple_diffusion/example/iodef.xml index 3193315e5..ebc380b46 100644 --- a/applications/simple_diffusion/example/iodef.xml +++ b/applications/simple_diffusion/example/iodef.xml @@ -3,9 +3,9 @@ - - - + + + - \ No newline at end of file + diff --git a/applications/skeleton/example/etc b/applications/skeleton/example/etc new file mode 120000 index 000000000..5bed95183 --- /dev/null +++ b/applications/skeleton/example/etc @@ -0,0 +1 @@ +../../../etc/ \ No newline at end of file From 2375504f0af2bac81bd87fa7526f9d3290a1da05 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:30:25 +0000 Subject: [PATCH 07/18] pytest iodef.xml --- .../bin/test_validate_iodef_xml.py | 76 +++++++++++++++++++ rose-stem/app/validate_iodef/rose-app.conf | 2 + 2 files changed, 78 insertions(+) create mode 100644 rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py create mode 100644 rose-stem/app/validate_iodef/rose-app.conf diff --git a/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py b/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py new file mode 100644 index 000000000..d30827d38 --- /dev/null +++ b/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py @@ -0,0 +1,76 @@ +import glob +import os +import xml.etree.ElementTree as ET + +import pytest + + +def srcReplace(parent, path): + """ + Recursively update input parent element, resolVing `src` content and replacing this + with parsed XML elements from referenced files. + """ + i = 0 + for elem in parent: + if elem.attrib.get('src'): + inf = os.path.join(path, elem.attrib['src']) + print(f'parsing: {inf}') + newelem = ET.parse(inf).getroot() + parent[i] = newelem + else: + srcReplace(elem, path) + i += 1 + + +def load_source_xml(fname): + path = os.path.dirname(fname) + tree = ET.parse(fname) + root = tree.getroot() + + # Validate that external elements, defined using `src=` are only accessing known internal paths + # to minimise rish from "XML external entity attack" + for elem in root.findall('.//*[@src]'): + if not (elem.attrib['src'].startswith('metadata/') or + elem.attrib['src'].startswith('etc/') or + elem.attrib['src'].startswith('./') or + elem.attrib['src'].startswith('../') or + elem.attrib['src'].startswith('$SOURCE_ROOT/')): + raise ValueError('only `src` attributes from local `metadata/` or ' + '`etc/` or `$SOURCE_ROOT/`paths are supported, not ' + f'{elem.attrib["src"]}') + srcReplace(root, path) + return tree + +root_dir = os.environ.get('$SOURCE_ROOT', '/home/users/mark.hedley/metofficegit/') + 'lfric_core' + +iodef_likes = [] +for iodef_like in glob.glob('**/iodef*.xml', + root_dir=root_dir, + recursive=True): + infile = os.path.join(root_dir, iodef_like) + tree = load_source_xml(infile) + root = tree.getroot() + iodef_likes.append((root, infile)) + +# tree = load_source_xml(os.path.join(hard_path, iodef)) +# root = tree.getroot() + +@pytest.mark.parametrize("aroot, infile", iodef_likes) +def test_unique_field_ids_within_context(aroot, infile): + print(f'validating XIOS XML for {aroot} ...') + + for context in aroot.findall('.//context'): + field_ids = set() + for elem in context.findall('.//field[@id]'): + # for elem in aroot.findall('.//field[@id]'): + eid = elem.attrib.get('id') + if eid is not None: + err_str = (f'Within {infile}\n:' + '\tThe `field` element:\n' + f'{elem.attrib}\n' + f'\thas an `id`: "{eid}" ' + 'which is already defined for an existing field.\n' + '\tXIOS will not distinguish betwen these fields, operations ' + 'using this id will affect all fields with this `id`') + assert eid not in field_ids, err_str + field_ids.add(eid) diff --git a/rose-stem/app/validate_iodef/rose-app.conf b/rose-stem/app/validate_iodef/rose-app.conf new file mode 100644 index 000000000..a36e08138 --- /dev/null +++ b/rose-stem/app/validate_iodef/rose-app.conf @@ -0,0 +1,2 @@ +[command] +default=pytest bin/validate_iodef_xml.py \ No newline at end of file From be7dd52c0ac0804c31f66b3be5069bb1a58a2393 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Tue, 14 Jul 2026 08:15:56 +0000 Subject: [PATCH 08/18] cylc test configuration & code quality linting --- .../bin/test_validate_iodef_xml.py | 80 ++++++++++++------- rose-stem/app/validate_iodef/rose-app.conf | 2 +- rose-stem/site/meto/groups.cylc | 1 + .../graph/populate_graph_scripts.cylc | 1 + .../runtime/generate_runtime_scripts.cylc | 8 ++ 5 files changed, 63 insertions(+), 29 deletions(-) diff --git a/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py b/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py index d30827d38..349b520d7 100644 --- a/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py +++ b/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py @@ -1,14 +1,33 @@ +#!/usr/bin/env python3 +############################################################################## +# (c) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +""" +XIOS Iodef.xml metadata consistency validation tests. +""" + import glob import os +import re import xml.etree.ElementTree as ET import pytest -def srcReplace(parent, path): +# security pattern to check whether `src` links are local and link to known +# controlled facets of the local source tree. Do not load from unknown sources. +# allows strings that start with: +# 'metadata/' etc/' './' '../metadata' '$SOURCE_ROOT/' +# followed by alphabetic characters only +security_re = re.compile(r'^(metadata/|etc/|\./|../metadata/|\$SOURCE_ROOT/)' + r'[a-zA-Z]+\w') + +def src_replace(parent, path): """ - Recursively update input parent element, resolVing `src` content and replacing this - with parsed XML elements from referenced files. + Recursively update input parent element, resolVing `src` content and + replacing this with parsed XML elements from referenced files. """ i = 0 for elem in parent: @@ -18,31 +37,32 @@ def srcReplace(parent, path): newelem = ET.parse(inf).getroot() parent[i] = newelem else: - srcReplace(elem, path) + src_replace(elem, path) i += 1 def load_source_xml(fname): + """ + Load a source XML file and recursively populated `src` links from + identified safe local paths. + """ path = os.path.dirname(fname) - tree = ET.parse(fname) - root = tree.getroot() + load_tree = ET.parse(fname) + load_root = load_tree.getroot() - # Validate that external elements, defined using `src=` are only accessing known internal paths - # to minimise rish from "XML external entity attack" - for elem in root.findall('.//*[@src]'): - if not (elem.attrib['src'].startswith('metadata/') or - elem.attrib['src'].startswith('etc/') or - elem.attrib['src'].startswith('./') or - elem.attrib['src'].startswith('../') or - elem.attrib['src'].startswith('$SOURCE_ROOT/')): - raise ValueError('only `src` attributes from local `metadata/` or ' + # Validate that external elements, defined using `src=` are only accessing + # known internal paths to minimise rish from "XML external entity attack" + for elem in load_root.findall('.//*[@src]'): + if security_re.match(elem.attrib['src']) is None: + raise ValueError('only `src` attributes from local, `metadata/`, ' '`etc/` or `$SOURCE_ROOT/`paths are supported, not ' f'{elem.attrib["src"]}') - srcReplace(root, path) - return tree - -root_dir = os.environ.get('$SOURCE_ROOT', '/home/users/mark.hedley/metofficegit/') + 'lfric_core' + src_replace(load_root, path) + return load_tree +# Generator for the pytest parametrize fixture. +root_dir = os.environ.get('SOURCE_PATH', '') +print(f'root_dir = {root_dir}') iodef_likes = [] for iodef_like in glob.glob('**/iodef*.xml', root_dir=root_dir, @@ -52,25 +72,29 @@ def load_source_xml(fname): root = tree.getroot() iodef_likes.append((root, infile)) -# tree = load_source_xml(os.path.join(hard_path, iodef)) -# root = tree.getroot() +if len(iodef_likes) == 0: + raise ValueError('No `iodef*.xml` files found.') -@pytest.mark.parametrize("aroot, infile", iodef_likes) -def test_unique_field_ids_within_context(aroot, infile): +@pytest.mark.parametrize("aroot, test_file", iodef_likes) +def test_unique_field_ids_within_context(aroot, test_file): + """ + Pytest test to verify that `field` element `id`s are unique within + a given XIOS context. + """ print(f'validating XIOS XML for {aroot} ...') for context in aroot.findall('.//context'): - field_ids = set() + field_ids = set() for elem in context.findall('.//field[@id]'): - # for elem in aroot.findall('.//field[@id]'): eid = elem.attrib.get('id') if eid is not None: - err_str = (f'Within {infile}\n:' + err_str = (f'Within {test_file}\n:' '\tThe `field` element:\n' f'{elem.attrib}\n' f'\thas an `id`: "{eid}" ' 'which is already defined for an existing field.\n' - '\tXIOS will not distinguish betwen these fields, operations ' - 'using this id will affect all fields with this `id`') + '\tXIOS will not distinguish betwen these fields, ' + 'operations using this id will affect all fields ' + 'with this `id` leading to indeterminate behaviour') assert eid not in field_ids, err_str field_ids.add(eid) diff --git a/rose-stem/app/validate_iodef/rose-app.conf b/rose-stem/app/validate_iodef/rose-app.conf index a36e08138..cb422f37c 100644 --- a/rose-stem/app/validate_iodef/rose-app.conf +++ b/rose-stem/app/validate_iodef/rose-app.conf @@ -1,2 +1,2 @@ [command] -default=pytest bin/validate_iodef_xml.py \ No newline at end of file +default=pytest $CYLC_WORKFLOW_RUN_DIR/app/validate_iodef/bin/test_validate_iodef_xml.py \ No newline at end of file diff --git a/rose-stem/site/meto/groups.cylc b/rose-stem/site/meto/groups.cylc index 62646ad80..fbfb9c2b1 100644 --- a/rose-stem/site/meto/groups.cylc +++ b/rose-stem/site/meto/groups.cylc @@ -14,6 +14,7 @@ "validate_rose_meta", "rose-stem_lint_checker", "global_variables_checker", + "validate_iodef", "python_unit_tests" ], "azspice_developer": [], diff --git a/rose-stem/templates/graph/populate_graph_scripts.cylc b/rose-stem/templates/graph/populate_graph_scripts.cylc index 2aa328729..10493f1d8 100644 --- a/rose-stem/templates/graph/populate_graph_scripts.cylc +++ b/rose-stem/templates/graph/populate_graph_scripts.cylc @@ -56,6 +56,7 @@ {# * validate_rose_meta #} {# * global_variables_checker #} {# * python_unit_tests #} + {# * validate_iodef #} {# * local_build_test #} {% do graph_sections.append([ diff --git a/rose-stem/templates/runtime/generate_runtime_scripts.cylc b/rose-stem/templates/runtime/generate_runtime_scripts.cylc index 3c501f06f..cc10850c7 100644 --- a/rose-stem/templates/runtime/generate_runtime_scripts.cylc +++ b/rose-stem/templates/runtime/generate_runtime_scripts.cylc @@ -56,6 +56,14 @@ inherit={{inherit.str|upper}} script="rose task-run --app-key=rose-stem_lint_check" +{% elif "validate_iodef" in task %} + + inherit={{inherit.str|upper}} + script="rose task-run --app-key=validate_iodef" + [[[environment]]] + SOURCE_PATH = $SOURCE_ROOT/lfric_core + + {% elif "validate_rose_meta" in task %} inherit={{inherit.str|upper}} From 45815bccd4067a77e57a334e1ce22eb480ebba3b Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Tue, 14 Jul 2026 08:41:59 +0000 Subject: [PATCH 09/18] config and linting --- rose-stem/app/coupled/rose-app.conf | 8 ++++---- rose-stem/app/io_demo/rose-app.conf | 16 ++++++++-------- rose-stem/app/lbc_demo/rose-app.conf | 16 ++++++++-------- rose-stem/app/simple_diffusion/rose-app.conf | 16 ++++++++-------- rose-stem/app/skeleton/rose-app.conf | 8 ++++---- rose-stem/app/validate_iodef/rose-app.conf | 2 +- .../runtime/generate_runtime_scripts.cylc | 1 - 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/rose-stem/app/coupled/rose-app.conf b/rose-stem/app/coupled/rose-app.conf index 2d208e395..f31afcf03 100644 --- a/rose-stem/app/coupled/rose-app.conf +++ b/rose-stem/app/coupled/rose-app.conf @@ -15,10 +15,6 @@ TOTAL_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir -[file:etc] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc - [file:configuration_glo.nml] mode=auto source=namelist:base_mesh @@ -31,6 +27,10 @@ source=namelist:base_mesh = namelist:time = namelist:timestepping +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [namelist:base_mesh] f_lat_deg=45.0 file_prefix='mesh' diff --git a/rose-stem/app/io_demo/rose-app.conf b/rose-stem/app/io_demo/rose-app.conf index 493e05a67..0cb4656e6 100644 --- a/rose-stem/app/io_demo/rose-app.conf +++ b/rose-stem/app/io_demo/rose-app.conf @@ -13,14 +13,6 @@ XIOS_SERVER_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir -[file:metadata] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/io_demo/metadata - -[file:etc] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc - [file:configuration.nml] mode=auto source=namelist:base_mesh @@ -36,6 +28,14 @@ source=namelist:base_mesh = namelist:time = namelist:timestepping +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + +[file:metadata] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/io_demo/metadata + [namelist:base_mesh] f_lat_deg=45.0 file_prefix='' diff --git a/rose-stem/app/lbc_demo/rose-app.conf b/rose-stem/app/lbc_demo/rose-app.conf index 413746421..50b266ad0 100644 --- a/rose-stem/app/lbc_demo/rose-app.conf +++ b/rose-stem/app/lbc_demo/rose-app.conf @@ -13,14 +13,6 @@ XIOS_SERVER_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir -[file:metadata] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/lbc_demo/metadata - -[file:etc] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc - [file:configuration.nml] mode=auto source=namelist:base_mesh @@ -33,6 +25,14 @@ source=namelist:base_mesh = namelist:time = namelist:timestepping +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + +[file:metadata] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/lbc_demo/metadata + [namelist:base_mesh] f_lat_deg=45.0 file_prefix='' diff --git a/rose-stem/app/simple_diffusion/rose-app.conf b/rose-stem/app/simple_diffusion/rose-app.conf index 0093742ab..f18d43f1f 100644 --- a/rose-stem/app/simple_diffusion/rose-app.conf +++ b/rose-stem/app/simple_diffusion/rose-app.conf @@ -13,14 +13,6 @@ XIOS_SERVER_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir -[file:metadata] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/simple_diffusion/metadata - -[file:etc] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc - [file:configuration.nml] mode=auto source=namelist:base_mesh @@ -33,6 +25,14 @@ source=namelist:base_mesh = namelist:time = namelist:timestepping +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + +[file:metadata] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/simple_diffusion/metadata + [namelist:base_mesh] f_lat_deg=45.0 file_prefix='' diff --git a/rose-stem/app/skeleton/rose-app.conf b/rose-stem/app/skeleton/rose-app.conf index e74403fde..0cdc5ad30 100644 --- a/rose-stem/app/skeleton/rose-app.conf +++ b/rose-stem/app/skeleton/rose-app.conf @@ -13,10 +13,6 @@ TOTAL_RANKS=1 [file:$DESTINATION_DIRECTORY] mode=mkdir -[file:etc] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc - [file:configuration.nml] mode=auto source=namelist:base_mesh @@ -29,6 +25,10 @@ source=namelist:base_mesh = namelist:time = namelist:timestepping +[file:etc] +mode=symlink +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc + [namelist:base_mesh] f_lat_deg=45.0 file_prefix='' diff --git a/rose-stem/app/validate_iodef/rose-app.conf b/rose-stem/app/validate_iodef/rose-app.conf index cb422f37c..44c82d8e0 100644 --- a/rose-stem/app/validate_iodef/rose-app.conf +++ b/rose-stem/app/validate_iodef/rose-app.conf @@ -1,2 +1,2 @@ [command] -default=pytest $CYLC_WORKFLOW_RUN_DIR/app/validate_iodef/bin/test_validate_iodef_xml.py \ No newline at end of file +default=pytest $CYLC_WORKFLOW_RUN_DIR/app/validate_iodef/bin/test_validate_iodef_xml.py diff --git a/rose-stem/templates/runtime/generate_runtime_scripts.cylc b/rose-stem/templates/runtime/generate_runtime_scripts.cylc index cc10850c7..0fb2081e9 100644 --- a/rose-stem/templates/runtime/generate_runtime_scripts.cylc +++ b/rose-stem/templates/runtime/generate_runtime_scripts.cylc @@ -62,7 +62,6 @@ script="rose task-run --app-key=validate_iodef" [[[environment]]] SOURCE_PATH = $SOURCE_ROOT/lfric_core - {% elif "validate_rose_meta" in task %} From 39d63adea91986b0109bd6917a2195552fc85a8a Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:20:19 +0000 Subject: [PATCH 10/18] test configurations --- applications/io_demo/example/iodef.xml | 6 +++--- applications/io_demo/example/metadata | 1 + applications/lbc_demo/example/iodef.xml | 12 ++++++------ applications/lbc_demo/example/metadata | 1 + applications/simple_diffusion/example/iodef.xml | 6 +++--- applications/simple_diffusion/example/metadata | 1 + .../validate_iodef/bin/test_validate_iodef_xml.py | 2 +- .../runtime/generate_runtime_application.cylc | 2 +- 8 files changed, 17 insertions(+), 14 deletions(-) create mode 120000 applications/io_demo/example/metadata create mode 120000 applications/lbc_demo/example/metadata create mode 120000 applications/simple_diffusion/example/metadata diff --git a/applications/io_demo/example/iodef.xml b/applications/io_demo/example/iodef.xml index 35deedcc6..238a45351 100644 --- a/applications/io_demo/example/iodef.xml +++ b/applications/io_demo/example/iodef.xml @@ -3,9 +3,9 @@ - - - + + + - - - + + + @@ -24,9 +24,9 @@ - - - + + + diff --git a/applications/lbc_demo/example/metadata b/applications/lbc_demo/example/metadata new file mode 120000 index 000000000..56e175b1c --- /dev/null +++ b/applications/lbc_demo/example/metadata @@ -0,0 +1 @@ +../metadata/ \ No newline at end of file diff --git a/applications/simple_diffusion/example/iodef.xml b/applications/simple_diffusion/example/iodef.xml index ebc380b46..3193315e5 100644 --- a/applications/simple_diffusion/example/iodef.xml +++ b/applications/simple_diffusion/example/iodef.xml @@ -3,9 +3,9 @@ - - - + + + Date: Tue, 14 Jul 2026 10:24:58 +0000 Subject: [PATCH 11/18] f --- rose-stem/app/canned_test/rose-app.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rose-stem/app/canned_test/rose-app.conf b/rose-stem/app/canned_test/rose-app.conf index 035465e98..2934ac1b2 100644 --- a/rose-stem/app/canned_test/rose-app.conf +++ b/rose-stem/app/canned_test/rose-app.conf @@ -1,6 +1,6 @@ [command] -aprun=eval $CANNED_PRESCRIPT ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -default=eval $CANNED_PRESCRIPT ; ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -mpiexec=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -mpiexec_cpl=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i -srun=eval $CANNED_PRESCRIPT ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +aprun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +default=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +mpiexec=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +mpiexec_cpl=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i +srun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} From d97caafbe9881281b861c79bb81cf928d77c529b Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:29:00 +0000 Subject: [PATCH 12/18] b0rked :( --- etc/xios.xml | 2 +- rose-stem/app/canned_test/rose-app.conf | 8 ++++---- rose-stem/app/io_demo/rose-app.conf | 9 ++++++--- .../templates/runtime/generate_runtime_application.cylc | 6 +++++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/etc/xios.xml b/etc/xios.xml index a970c070b..59c5413fa 100644 --- a/etc/xios.xml +++ b/etc/xios.xml @@ -5,7 +5,7 @@ - true + $XIOS_SERVER_MODE 50 true diff --git a/rose-stem/app/canned_test/rose-app.conf b/rose-stem/app/canned_test/rose-app.conf index 2934ac1b2..28058f0b0 100644 --- a/rose-stem/app/canned_test/rose-app.conf +++ b/rose-stem/app/canned_test/rose-app.conf @@ -1,6 +1,6 @@ [command] -aprun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +aprun=eval $CANNED_PRESCRIPT ; aprun -cc depth -n ${PROCESSES:-1} -d $OMP_NUM_THREADS -j $HYPERTHREADS ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} default=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -mpiexec=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} -mpiexec_cpl=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i -srun=eval $CANNED_PRESCRIPT ; $CORE_ROOT_DIR/bin/tweak_iodef ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +mpiexec=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} +mpiexec_cpl=eval $CANNED_PRESCRIPT ; mpiexec -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE1} -c lfric_o : -n ${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE2} -c lfric_i +srun=eval $CANNED_PRESCRIPT ; srun --ntasks=${PROCESSES:-1} ${BIN_DIR}/${APPLICATION} ${CONFIGURATION_FILE} diff --git a/rose-stem/app/io_demo/rose-app.conf b/rose-stem/app/io_demo/rose-app.conf index 0cb4656e6..0aac1d234 100644 --- a/rose-stem/app/io_demo/rose-app.conf +++ b/rose-stem/app/io_demo/rose-app.conf @@ -1,7 +1,8 @@ meta=lfric-io_demo/vn3.2 [command] -default=$LAUNCH_SCRIPT/launch-exe +default=rose env-cat ${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc/xios.xml -o etc/xios.xml ; + =$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=io_demo @@ -29,8 +30,10 @@ source=namelist:base_mesh = namelist:timestepping [file:etc] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc +mode=mkdir + +[file:etc/xios.xml] +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc/xios.xml [file:metadata] mode=symlink diff --git a/rose-stem/templates/runtime/generate_runtime_application.cylc b/rose-stem/templates/runtime/generate_runtime_application.cylc index 77832b2eb..4d81fa46b 100644 --- a/rose-stem/templates/runtime/generate_runtime_application.cylc +++ b/rose-stem/templates/runtime/generate_runtime_application.cylc @@ -35,7 +35,11 @@ {# Set the pre-script command for canned tests #} {% set canned_prescript = "cp -rL $SOURCE_DIRECTORY/"~task_values["example_dir"]~"/* "~ - "$CYLC_TASK_WORK_DIR" %} + "$CYLC_TASK_WORK_DIR ; "~ + "rose env-cat -o $CYLC_TASK_WORK_DIR/etc/tmp_xios.xml "~ + "$CYLC_TASK_WORK_DIR/etc/xios.xml ; "~ + "mv $CYLC_TASK_WORK_DIR/etc/tmp_xios.xml "~ + "$CYLC_TASK_WORK_DIR/etc/xios.xml " %} {# Set the post script command - should be a list of strings #} {% set post_script_commands = [ From 205129bf457f3eb0bf93d522e048f047e0ec5d04 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:33:58 +0000 Subject: [PATCH 13/18] refactor shared xios contexts and rose confs --- applications/coupled/example/iodef.xml | 2 +- etc/xios.xml | 2 +- etc/xios_coupled.xml | 16 ++++++++++++++++ rose-stem/app/coupled/file/iodef.xml | 2 +- rose-stem/app/coupled/rose-app.conf | 5 +++-- .../io_demo/opt/rose-app-temporal_reading.conf | 3 --- rose-stem/app/io_demo/rose-app.conf | 11 ++++------- rose-stem/app/lbc_demo/rose-app.conf | 8 ++++---- rose-stem/app/simple_diffusion/rose-app.conf | 6 +++--- rose-stem/app/skeleton/rose-app.conf | 5 +++-- .../runtime/generate_runtime_application.cylc | 13 ++++++++++--- 11 files changed, 46 insertions(+), 27 deletions(-) create mode 100644 etc/xios_coupled.xml diff --git a/applications/coupled/example/iodef.xml b/applications/coupled/example/iodef.xml index fe7ebf7a1..126cc239f 100644 --- a/applications/coupled/example/iodef.xml +++ b/applications/coupled/example/iodef.xml @@ -55,6 +55,6 @@ - + diff --git a/etc/xios.xml b/etc/xios.xml index 59c5413fa..07879f5e6 100644 --- a/etc/xios.xml +++ b/etc/xios.xml @@ -5,7 +5,7 @@ - $XIOS_SERVER_MODE + $XIOS_SERVER_MODE_XML 50 true diff --git a/etc/xios_coupled.xml b/etc/xios_coupled.xml new file mode 100644 index 000000000..4d6fd18d3 --- /dev/null +++ b/etc/xios_coupled.xml @@ -0,0 +1,16 @@ + + + + performance + + + + $XIOS_SERVER_MODE_XML + 50 + true + true + lfric + + + + diff --git a/rose-stem/app/coupled/file/iodef.xml b/rose-stem/app/coupled/file/iodef.xml index fe7ebf7a1..126cc239f 100644 --- a/rose-stem/app/coupled/file/iodef.xml +++ b/rose-stem/app/coupled/file/iodef.xml @@ -55,6 +55,6 @@ - + diff --git a/rose-stem/app/coupled/rose-app.conf b/rose-stem/app/coupled/rose-app.conf index f31afcf03..c97bf550c 100644 --- a/rose-stem/app/coupled/rose-app.conf +++ b/rose-stem/app/coupled/rose-app.conf @@ -1,7 +1,9 @@ meta=lfric-coupled/vn3.2 [command] -default=rose app-run -C $CYLC_WORKFLOW_RUN_DIR/app/coupled -O lam -O LAM50x50-2x2 --install-only +default=rose env-cat -o etc/updated_xios.xml etc/xios_coupled.xml + =mv etc/updated_xios.xml etc/xios_coupled.xml + =rose app-run -C $CYLC_WORKFLOW_RUN_DIR/app/coupled -O lam -O LAM50x50-2x2 --install-only =rose app-run -C $CYLC_WORKFLOW_RUN_DIR/app/coupled -O glo -O C12 --install-only =$LAUNCH_SCRIPT/cpl-launch-exe @@ -28,7 +30,6 @@ source=namelist:base_mesh = namelist:timestepping [file:etc] -mode=symlink source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc [namelist:base_mesh] diff --git a/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf b/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf index 91aefcd57..852ac5f7e 100644 --- a/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf +++ b/rose-stem/app/io_demo/opt/rose-app-temporal_reading.conf @@ -1,6 +1,3 @@ -[command] -default=$LAUNCH_SCRIPT/launch-exe - [file:iodef.xml] mode=auto source=$ROSE_SUITE_DIR/app/io_demo/file/iodef_temporal.xml diff --git a/rose-stem/app/io_demo/rose-app.conf b/rose-stem/app/io_demo/rose-app.conf index 0aac1d234..94861d574 100644 --- a/rose-stem/app/io_demo/rose-app.conf +++ b/rose-stem/app/io_demo/rose-app.conf @@ -1,7 +1,8 @@ meta=lfric-io_demo/vn3.2 [command] -default=rose env-cat ${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc/xios.xml -o etc/xios.xml ; +default=rose env-cat -o etc/updated_xios.xml etc/xios.xml + =mv etc/updated_xios.xml etc/xios.xml =$LAUNCH_SCRIPT/launch-exe [env] @@ -30,14 +31,10 @@ source=namelist:base_mesh = namelist:timestepping [file:etc] -mode=mkdir - -[file:etc/xios.xml] -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc/xios.xml +source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc [file:metadata] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/io_demo/metadata +source=$METADATA [namelist:base_mesh] f_lat_deg=45.0 diff --git a/rose-stem/app/lbc_demo/rose-app.conf b/rose-stem/app/lbc_demo/rose-app.conf index 50b266ad0..bd1c70138 100644 --- a/rose-stem/app/lbc_demo/rose-app.conf +++ b/rose-stem/app/lbc_demo/rose-app.conf @@ -1,7 +1,9 @@ meta=lfric-lbc_demo/vn3.2 [command] -default=$LAUNCH_SCRIPT/launch-exe +default=rose env-cat -o etc/updated_xios.xml etc/xios.xml + =mv etc/updated_xios.xml etc/xios.xml + =$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=lbc_demo @@ -26,12 +28,10 @@ source=namelist:base_mesh = namelist:timestepping [file:etc] -mode=symlink source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc [file:metadata] -mode=symlink -source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/lbc_demo/metadata +source=$METADATA [namelist:base_mesh] f_lat_deg=45.0 diff --git a/rose-stem/app/simple_diffusion/rose-app.conf b/rose-stem/app/simple_diffusion/rose-app.conf index f18d43f1f..863a1c73f 100644 --- a/rose-stem/app/simple_diffusion/rose-app.conf +++ b/rose-stem/app/simple_diffusion/rose-app.conf @@ -1,7 +1,9 @@ meta=lfric-simple_diffusion/vn3.2 [command] -default=$LAUNCH_SCRIPT/launch-exe +default=rose env-cat -o etc/updated_xios.xml etc/xios.xml + =mv etc/updated_xios.xml etc/xios.xml + =$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=simple_diffusion @@ -26,11 +28,9 @@ source=namelist:base_mesh = namelist:timestepping [file:etc] -mode=symlink source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc [file:metadata] -mode=symlink source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/applications/simple_diffusion/metadata [namelist:base_mesh] diff --git a/rose-stem/app/skeleton/rose-app.conf b/rose-stem/app/skeleton/rose-app.conf index 0cdc5ad30..6e854e6a0 100644 --- a/rose-stem/app/skeleton/rose-app.conf +++ b/rose-stem/app/skeleton/rose-app.conf @@ -1,7 +1,9 @@ meta=lfric-skeleton/vn3.2 [command] -default=$LAUNCH_SCRIPT/launch-exe +default=rose env-cat -o etc/updated_xios.xml etc/xios.xml + =mv etc/updated_xios.xml etc/xios.xml + =$LAUNCH_SCRIPT/launch-exe [env] EXEC_NAME=skeleton @@ -26,7 +28,6 @@ source=namelist:base_mesh = namelist:timestepping [file:etc] -mode=symlink source=${CYLC_WORKFLOW_SHARE_DIR}/source/lfric_core/etc [namelist:base_mesh] diff --git a/rose-stem/templates/runtime/generate_runtime_application.cylc b/rose-stem/templates/runtime/generate_runtime_application.cylc index 4d81fa46b..2c905bddc 100644 --- a/rose-stem/templates/runtime/generate_runtime_application.cylc +++ b/rose-stem/templates/runtime/generate_runtime_application.cylc @@ -34,12 +34,18 @@ {% set restart_stem_name = checkpoint_dir~"/restart_"~task_family~"-" %} {# Set the pre-script command for canned tests #} +{# Copy the source, dereferencing symbolic links,#} +{# then use `rose env-cat` for Env Vars on etc/xios.xml & etc/xios_coupled.xml #} {% set canned_prescript = "cp -rL $SOURCE_DIRECTORY/"~task_values["example_dir"]~"/* "~ "$CYLC_TASK_WORK_DIR ; "~ - "rose env-cat -o $CYLC_TASK_WORK_DIR/etc/tmp_xios.xml "~ + "rose env-cat -o $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ "$CYLC_TASK_WORK_DIR/etc/xios.xml ; "~ - "mv $CYLC_TASK_WORK_DIR/etc/tmp_xios.xml "~ - "$CYLC_TASK_WORK_DIR/etc/xios.xml " %} + "mv $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ + "$CYLC_TASK_WORK_DIR/etc/xios.xml "~ + "rose env-cat -o $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ + "$CYLC_TASK_WORK_DIR/etc/xios_coupled.xml ; "~ + "mv $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ + "$CYLC_TASK_WORK_DIR/etc/xios-coupled.xml " %} {# Set the post script command - should be a list of strings #} {% set post_script_commands = [ @@ -147,6 +153,7 @@ CHECKPOINT_STEM_FILE = {{restart_stem_name}} CANNED_PRESCRIPT = {{canned_prescript}} XIOS_SERVER_MODE = {{task_values["xios_server_mode"]}} + XIOS_SERVER_MODE_XML = {{(task_values["xios_server_mode"]).__str__().lower() }} XIOS_SERVER_RANKS = {{task_values["xios_server_ranks"]}} xios_nodes = {{task_values["xios_nodes"]}} mpi_parts_xios = {{task_values["mpi_parts_xios"]}} From 190a60e13af820c7149c9be5a1890a496f311569 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:04:05 +0000 Subject: [PATCH 14/18] XIOS .xml docs --- .../source/how_to_use_it/io/io_standards.rst | 14 ++++++++++---- .../source/how_to_use_it/io/prognostics.rst | 9 +++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/documentation/source/how_to_use_it/io/io_standards.rst b/documentation/source/how_to_use_it/io/io_standards.rst index 866490cc1..0b5f6bd9e 100644 --- a/documentation/source/how_to_use_it/io/io_standards.rst +++ b/documentation/source/how_to_use_it/io/io_standards.rst @@ -20,16 +20,22 @@ Standards for XIOS Iodef Files section of every top-level iodef file. #. Non-global file settings are to be specified at the level of individual files. + #. ``id`` values shall be unique for a given ``XIOS`` ``XML` element + type. **Rationale** .. warning:: - When XIOS aggregates file definitions - possibly spread across - several XML files -, the last set of attributes processed "wins", - in the sense that all the others are silently ignored. + When XIOS aggregates definitions - possibly spread across + several XML files -, then attributes are overriden by defined elements + that use a ``_ref``. - This can create considerable confusion and frustration for users. + The overriding operates in a chain, so users may need to act cautiously + to understand attribute inheritance and overrides. + + This is why reuse of ``id`` values for different elements is problematic, + not allowed, and tested by a code inspection rule for ``field`` elements. With the current layout of the LFRic top-level iodef files, the last set of attributes to be processed are the ones in the local file diff --git a/documentation/source/how_to_use_it/io/prognostics.rst b/documentation/source/how_to_use_it/io/prognostics.rst index 7d0a1de36..620af65ea 100644 --- a/documentation/source/how_to_use_it/io/prognostics.rst +++ b/documentation/source/how_to_use_it/io/prognostics.rst @@ -286,3 +286,12 @@ to other model requirements including models that do not use XIOS. checkpoint flag setting, to the ``checkpoint_fields`` field collection. Additionally, like all other prognostic fields, it is added to the ``depository`` field collection. + + ``iodef.xml`` files may reference further ``.xml`` files using the + ``src=""`` attribute. File paths in the source tree are maintained + using symbolic links, which are dereferenced by ``rose-app.conf`` + to enable running tests. All links must resolve within the source, + and validation is implemented in a test called ``validate_iodef``. + Only the shared XIOS runtime configuration files within ``/etc`` + are updated to use environment variables, these should not be used + within other ``.xml`` fragments within the ``lfric_core`` code base. From 296eccbc326248a93ad90009f68d27bcc90330c0 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:23:08 +0000 Subject: [PATCH 15/18] restore tweak_iodef temporarily, to decouple PRs --- bin/tweak_iodef | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 bin/tweak_iodef diff --git a/bin/tweak_iodef b/bin/tweak_iodef new file mode 100755 index 000000000..aa846d67f --- /dev/null +++ b/bin/tweak_iodef @@ -0,0 +1,38 @@ +#!/bin/sh +############################################################################### +# (c) Crown copyright 2024 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################### +# tweak_iodef: +# replace relative path to LFRic dictionary with env symbol and +# resolve env symbols +# in iodef.xml before rose suite starts. + +iodef=iodef.xml +case $# in +0) ;; +1) iodef=$1 ;; +*) echo 'usage: tweak_iodef [iodef_file]' 1>&2; exit 2 +esac + +set -e +temp_file=$(mktemp) +trap 'rm -f $temp_file; exit 1' INT TERM EXIT +# examples substitutions +# ../../../lfric_atm/metadata used in gungho_model/example +# ../metadata -> $METADATA used in lfric_atm/example +if sed 's@\.\./\(\.\./\)*lfric_atm/metadata@$METADATA@' $iodef |\ + sed 's@\.\./\(\.\./\)*metadata@$METADATA@' |\ + rose env-cat - >$temp_file + echo "tweak_iodef: editing sucessful in $temp_file" +then + trap '' INT TERM EXIT + cp $temp_file $iodef + echo "tweak_iodef: $iodef saved with changes" +else + echo "tweak_iodef: editing failed, $iodef unchanged" 1>&2 + rm -f $temp_file + exit 1 +fi +rm -f $temp_file From 3ba37cfeee02e07deea233546f44b8e113be13a6 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:36:50 +0000 Subject: [PATCH 16/18] return example iodef files to using explicit xios context configuration, to avoid need for rose env-cat --- applications/coupled/example/etc | 1 - applications/coupled/example/iodef.xml | 16 +++++++++++++++- applications/io_demo/example/etc | 1 - applications/io_demo/example/iodef.xml | 14 +++++++++++++- applications/lbc_demo/example/etc | 1 - applications/lbc_demo/example/iodef.xml | 14 +++++++++++++- applications/simple_diffusion/example/etc | 1 - applications/simple_diffusion/example/iodef.xml | 14 +++++++++++++- applications/skeleton/example/etc | 1 - applications/skeleton/example/iodef.xml | 14 +++++++++++++- .../runtime/generate_runtime_application.cylc | 13 ++----------- 11 files changed, 69 insertions(+), 21 deletions(-) delete mode 120000 applications/coupled/example/etc delete mode 120000 applications/io_demo/example/etc delete mode 120000 applications/lbc_demo/example/etc delete mode 120000 applications/simple_diffusion/example/etc delete mode 120000 applications/skeleton/example/etc diff --git a/applications/coupled/example/etc b/applications/coupled/example/etc deleted file mode 120000 index 5bed95183..000000000 --- a/applications/coupled/example/etc +++ /dev/null @@ -1 +0,0 @@ -../../../etc/ \ No newline at end of file diff --git a/applications/coupled/example/iodef.xml b/applications/coupled/example/iodef.xml index 126cc239f..d32b67fb6 100644 --- a/applications/coupled/example/iodef.xml +++ b/applications/coupled/example/iodef.xml @@ -55,6 +55,20 @@ - + + + + performance + + + + false + 50 + true + true + lfric + + + diff --git a/applications/io_demo/example/etc b/applications/io_demo/example/etc deleted file mode 120000 index 5bed95183..000000000 --- a/applications/io_demo/example/etc +++ /dev/null @@ -1 +0,0 @@ -../../../etc/ \ No newline at end of file diff --git a/applications/io_demo/example/iodef.xml b/applications/io_demo/example/iodef.xml index 238a45351..8f84f23f2 100644 --- a/applications/io_demo/example/iodef.xml +++ b/applications/io_demo/example/iodef.xml @@ -19,6 +19,18 @@ - + + + + performance + + + + true + 50 + true + + + diff --git a/applications/lbc_demo/example/etc b/applications/lbc_demo/example/etc deleted file mode 120000 index 5bed95183..000000000 --- a/applications/lbc_demo/example/etc +++ /dev/null @@ -1 +0,0 @@ -../../../etc/ \ No newline at end of file diff --git a/applications/lbc_demo/example/iodef.xml b/applications/lbc_demo/example/iodef.xml index 4ee03ccc7..a88aace68 100644 --- a/applications/lbc_demo/example/iodef.xml +++ b/applications/lbc_demo/example/iodef.xml @@ -44,6 +44,18 @@ - + + + + performance + + + + true + 50 + true + + + diff --git a/applications/simple_diffusion/example/etc b/applications/simple_diffusion/example/etc deleted file mode 120000 index 5bed95183..000000000 --- a/applications/simple_diffusion/example/etc +++ /dev/null @@ -1 +0,0 @@ -../../../etc/ \ No newline at end of file diff --git a/applications/simple_diffusion/example/iodef.xml b/applications/simple_diffusion/example/iodef.xml index 3193315e5..51ce82a2e 100644 --- a/applications/simple_diffusion/example/iodef.xml +++ b/applications/simple_diffusion/example/iodef.xml @@ -19,6 +19,18 @@ - + + + + performance + + + + true + 50 + true + + + diff --git a/applications/skeleton/example/etc b/applications/skeleton/example/etc deleted file mode 120000 index 5bed95183..000000000 --- a/applications/skeleton/example/etc +++ /dev/null @@ -1 +0,0 @@ -../../../etc/ \ No newline at end of file diff --git a/applications/skeleton/example/iodef.xml b/applications/skeleton/example/iodef.xml index ad8e08461..5b2978a4d 100644 --- a/applications/skeleton/example/iodef.xml +++ b/applications/skeleton/example/iodef.xml @@ -55,6 +55,18 @@ - + + + + performance + + + + true + 50 + true + + + diff --git a/rose-stem/templates/runtime/generate_runtime_application.cylc b/rose-stem/templates/runtime/generate_runtime_application.cylc index 2c905bddc..cfd90ad8a 100644 --- a/rose-stem/templates/runtime/generate_runtime_application.cylc +++ b/rose-stem/templates/runtime/generate_runtime_application.cylc @@ -34,18 +34,9 @@ {% set restart_stem_name = checkpoint_dir~"/restart_"~task_family~"-" %} {# Set the pre-script command for canned tests #} -{# Copy the source, dereferencing symbolic links,#} -{# then use `rose env-cat` for Env Vars on etc/xios.xml & etc/xios_coupled.xml #} +{# Copy the source, dereferencing symbolic links #} {% set canned_prescript = "cp -rL $SOURCE_DIRECTORY/"~task_values["example_dir"]~"/* "~ - "$CYLC_TASK_WORK_DIR ; "~ - "rose env-cat -o $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ - "$CYLC_TASK_WORK_DIR/etc/xios.xml ; "~ - "mv $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ - "$CYLC_TASK_WORK_DIR/etc/xios.xml "~ - "rose env-cat -o $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ - "$CYLC_TASK_WORK_DIR/etc/xios_coupled.xml ; "~ - "mv $CYLC_TASK_WORK_DIR/etc/updated_xios.xml "~ - "$CYLC_TASK_WORK_DIR/etc/xios-coupled.xml " %} + "$CYLC_TASK_WORK_DIR "%} {# Set the post script command - should be a list of strings #} {% set post_script_commands = [ From eef68553ed5f35c212c19f43a80e528db355df41 Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:23:23 +0000 Subject: [PATCH 17/18] improve pytest code for review --- rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py b/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py index e819def12..e2189d18a 100644 --- a/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py +++ b/rose-stem/app/validate_iodef/bin/test_validate_iodef_xml.py @@ -19,7 +19,7 @@ # security pattern to check whether `src` links are local and link to known # controlled facets of the local source tree. Do not load from unknown sources. # allows strings that start with: -# 'metadata/' etc/' './' '../metadata' '$SOURCE_ROOT/' +# 'metadata/' etc/' './' '$SOURCE_ROOT/' # followed by alphabetic characters only security_re = re.compile(r'^(metadata/|etc/|\./|\$SOURCE_ROOT/)' r'[a-zA-Z]+\w') @@ -98,3 +98,6 @@ def test_unique_field_ids_within_context(aroot, test_file): 'with this `id` leading to indeterminate behaviour') assert eid not in field_ids, err_str field_ids.add(eid) + +if __name__ == '__main__': + pytest.main() From d7ab9c7ac2de3b76d2060ed3995c3795c3fdb97a Mon Sep 17 00:00:00 2001 From: mo-marqh <87382490+mo-marqh@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:03:16 +0000 Subject: [PATCH 18/18] docs linting --- documentation/source/how_to_use_it/io/io_standards.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/source/how_to_use_it/io/io_standards.rst b/documentation/source/how_to_use_it/io/io_standards.rst index 0b5f6bd9e..9c6609acc 100644 --- a/documentation/source/how_to_use_it/io/io_standards.rst +++ b/documentation/source/how_to_use_it/io/io_standards.rst @@ -20,8 +20,8 @@ Standards for XIOS Iodef Files section of every top-level iodef file. #. Non-global file settings are to be specified at the level of individual files. - #. ``id`` values shall be unique for a given ``XIOS`` ``XML` element - type. + #. ``id`` values shall be unique for a given ``XIOS`` ``XML`` element + type. **Rationale**