(develop) Add a UI for fix file customization

.github/CODEOWNERS

@@ -20,6 +20,7 @@ dev/ush/de_template_com.sh @DavidHuber-NOAA
 
 # Specific workflow scripts
 dev/workflow/generate_workflows.sh @DavidHuber-NOAA
+dev/workflow/fix_overlay.py @AlexanderHrabski-NOAA
 
 # build scripts
 sorc/build_*.sh @DavidHuber-NOAA @TravisElless-NOAA

dev/ci/scripts/unittests/test_setup.py

@@ -113,3 +113,67 @@ def test_setup_workflow_fail_config_env_cornercase(tmp_path):
             rmtree(RUNDIR)
         except FileNotFoundError:
             pass
+
+
+# --- fix_files: end-to-end through setup_expt.py -----------------------------
+#
+# These use their own pslot and clean up after themselves so they do not
+# interact with the order-dependent tests above.  Only `add:` is exercised on
+# the build path: on the GitHub runner link_workflow.sh leaves ${HOMEglobal}/fix
+# populated with dangling links, so nothing there "exists" for `replace:`.
+# The base-tree rules themselves are covered by test_fix_overlay.py.
+
+FIX_PSLOT = "C48_ATM_fix"
+
+
+def _run_setup_expt(yaml_path, fail_on_error=True):
+    setup_expt_script = Executable(os.path.join(HOMEglobal, "dev/workflow/setup_expt.py"))
+    setup_expt_script.add_default_arg([
+        "gfs", "forecast-only",
+        "--pslot", FIX_PSLOT, "--app", "ATM", "--resdetatmos", "48",
+        "--comroot", RUNDIR, "--expdir", RUNDIR,
+        "--idate", "2021032312", "--edate", "2021032312", "--overwrite",
+        "--yaml", str(yaml_path),
+    ])
+    setup_expt_script(fail_on_error=fail_on_error, output=os.devnull, error=os.devnull)
+    return setup_expt_script.returncode
+
+
+def _exp_yaml(tmp_path, fix_files_block=""):
+    yaml_path = tmp_path / 'exp.yaml'
+    header = "defaults:\n  !INC {{ HOMEglobal }}/dev/parm/config/gfs/yaml/defaults.yaml\n"
+    yaml_path.write_text(header + fix_files_block)
+    return yaml_path
+
+
+@pytest.fixture
+def fix_expdir():
+    yield f"{RUNDIR}/{FIX_PSLOT}"
+    rmtree(f"{RUNDIR}/{FIX_PSLOT}", ignore_errors=True)
+    rmtree(f"{RUNDIR}/COMROOT/{FIX_PSLOT}", ignore_errors=True)
+
+
+def test_setup_expt_fix_files(tmp_path, fix_expdir):
+    src = tmp_path / 'my_fix' / 'MOM'
+    src.mkdir(parents=True)
+    (src / 'regional.mom6.nc').write_text('regional')
+    (src / 'MOM_input').write_text('mom_input')
+
+    yaml_path = _exp_yaml(tmp_path, f"""
+fix_files:
+  add:
+    mom6/008/regional.mom6.nc: {src}/regional.mom6.nc
+    mom6/008:
+      - {src}/MOM_input
+""")
+    assert _run_setup_expt(yaml_path) == 0
+
+    # FIXglobal is rendered into config.base and points at the overlay
+    base = Configuration(fix_expdir).parse_config('config.base')
+    assert base.FIXglobal == f"{fix_expdir}/fix"
+
+    # the overlay holds the requested links, a manifest, and single links for untouched components
+    assert os.path.isfile(f"{fix_expdir}/fix/.fix_overlay.yaml")
+    assert os.readlink(f"{fix_expdir}/fix/mom6/008/regional.mom6.nc") == f"{src}/regional.mom6.nc"
+    assert os.readlink(f"{fix_expdir}/fix/mom6/008/MOM_input") == f"{src}/MOM_input"
+    assert os.readlink(f"{fix_expdir}/fix/product") == f"{HOMEglobal}/fix/product"  # tracked in git, always present

dev/parm/config/gcafs/config.base.j2

@@ -33,7 +33,7 @@ export HPSS_PROJECT="{{ HPSS_PROJECT }}"
 # Directories relative to installation areas:
 export HOMEglobal="{{ HOMEglobal }}"
 export EXECglobal="${HOMEglobal}/exec"
-export FIXglobal="${HOMEglobal}/fix"
+export FIXglobal="{{ FIXglobal }}"
 export PARMglobal="${HOMEglobal}/parm"
 export SCRglobal="${HOMEglobal}/scripts"
 export USHglobal="${HOMEglobal}/ush"
@@ -106,7 +106,7 @@ export DO_TEST_MODE="{{ DO_TEST_MODE }}" # option to change configuration for au
 # CLEAR
 ####################################################
 # Build paths relative to $HOMEglobal
-export FIXgsi="${HOMEglobal}/fix/gsi"
+export FIXgsi="${FIXglobal}/gsi"
 export HOMEpost="${HOMEglobal}"
 export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-}"
 

dev/parm/config/gcafs/yaml/defaults.yaml

@@ -36,6 +36,7 @@ base:
   assim_freq: 6
   FHMAX: 9
   FHOUT_AERO: 3
+  FIXglobal: "${HOMEglobal}/fix"
 
 atmanl:
   STATICB_TYPE: "gsibec"

dev/parm/config/gefs/config.base.j2

@@ -33,7 +33,7 @@ export HPSS_PROJECT="{{ HPSS_PROJECT }}"
 # Directories relative to installation areas:
 export HOMEglobal="{{ HOMEglobal }}"
 export EXECglobal=${HOMEglobal}/exec
-export FIXglobal=${HOMEglobal}/fix
+export FIXglobal="{{ FIXglobal }}"
 export PARMglobal=${HOMEglobal}/parm
 export SCRglobal=${HOMEglobal}/scripts
 export USHglobal=${HOMEglobal}/ush

dev/parm/config/gefs/yaml/defaults.yaml

@@ -22,6 +22,7 @@ base:
   USE_ATM_ENS_PERTURB_FILES: "NO"
   DO_TEST_MODE: "NO"
   MOM6_INTERP_ICS: "NO"
+  FIXglobal: "${HOMEglobal}/fix"
 fcst:
   CHECK_LAND_RESTART_OROG: "YES"
   reforecast: "NO"

dev/parm/config/gfs/config.aero.j2

@@ -17,7 +17,7 @@ echo "BEGIN: config.aero"
 # Base directory for aerosol input data files (e.g., initial conditions, climatologies).
 # This path is mounted or staged in the workflow and referenced by the model for reading aerosol fields.
 #---------------------------------------------------------------------------------------------------
-export AERO_INPUTS_DIR="${HOMEglobal}/fix/chem/Emission_data"
+export AERO_INPUTS_DIR="${FIXglobal}/chem/Emission_data"
 
 #-------------------------------------------------
 # Diag Table and Field Table for GOCART aerosols
@@ -81,7 +81,7 @@ export AERO_EMIS_FIRE_HIST=1 # Use historical fire emissions | 1 = true 0 = fals
 
 #---------------------------------------------------------------------------------------------------
 export FIRE_EMIS_NRT_DIR="" #TODO: set to DCOM for WCOSS2 "${DCOMROOT}/YYYYMMDD/firewx" # Directory containing NRT fire emissions
-export FIRE_EMIS_DIR="${HOMEglobal}/fix/chem/Emission_data/fires_data/GBBEPx/v4" # Directory containing historical fire emissions
+export FIRE_EMIS_DIR="${FIXglobal}/chem/Emission_data/fires_data/GBBEPx/v4" # Directory containing historical fire emissions
 
 
 #===============================================================================
@@ -112,7 +112,7 @@ export NEXUS_CONFIG_DIR="${PARMglobal}/chem/nexus/${NEXUS_CONFIG}" # Directory c
 # Specific path for GCAFS external data on this filesystem.
 # Contains emission datasets (e.g., CEDS2019/2024, HTAPv2, CAMS) processed by NEXUS.
 #---------------------------------------------------------------------------------------------------
-export NEXUS_INPUT_DIR="${HOMEglobal}/fix/chem/Emission_data/nexus"
+export NEXUS_INPUT_DIR="${FIXglobal}/chem/Emission_data/nexus"
 
 
 

dev/parm/config/gfs/config.base.j2

@@ -36,7 +36,7 @@ export HPSS_PROJECT="{{ HPSS_PROJECT }}"
 # Directories relative to installation areas:
 export HOMEglobal="{{ HOMEglobal }}"
 export EXECglobal="${HOMEglobal}/exec"
-export FIXglobal="${HOMEglobal}/fix"
+export FIXglobal="{{ FIXglobal }}"
 export PARMglobal="${HOMEglobal}/parm"
 export SCRglobal="${HOMEglobal}/scripts"
 export USHglobal="${HOMEglobal}/ush"
@@ -115,7 +115,7 @@ fi
 # CLEAR
 ####################################################
 # Build paths relative to $HOMEglobal
-export FIXgsi="${HOMEglobal}/fix/gsi"
+export FIXgsi="${FIXglobal}/gsi"
 export HOMEpost="${HOMEglobal}"
 export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-}"
 

dev/parm/config/gfs/yaml/defaults.yaml

@@ -31,6 +31,7 @@ base:
   NMEM_ENS_GFS: 30
   NMEM_ENS_GFS_OFFSET: 20
   MOM6_INTERP_ICS: "NO"
+  FIXglobal: "${HOMEglobal}/fix"
 
 atmanl:
   STATICB_TYPE: "gsibec"

dev/parm/config/sfs/config.base.j2

@@ -33,7 +33,7 @@ export HPSS_PROJECT="{{ HPSS_PROJECT }}"
 # Directories relative to installation areas:
 export HOMEglobal="{{ HOMEglobal }}"
 export EXECglobal=${HOMEglobal}/exec
-export FIXglobal=${HOMEglobal}/fix
+export FIXglobal="{{ FIXglobal }}"
 export PARMglobal=${HOMEglobal}/parm
 export SCRglobal=${HOMEglobal}/scripts
 export USHglobal=${HOMEglobal}/ush

dev/parm/config/sfs/yaml/defaults.yaml

@@ -22,6 +22,7 @@ base:
   USE_ATM_ENS_PERTURB_FILES: "YES"
   DO_TEST_MODE: "NO"
   MOM6_INTERP_ICS: "NO"
+  FIXglobal: "${HOMEglobal}/fix"
   DOIAU_COLDSTART: "NO"
 fcst:
   CHECK_LAND_RESTART_OROG: "YES"