Skip to content

Load mesh limit open calls per rank #446

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mo-marqh wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Load mesh limit open calls per rank #446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
30e14b8
always open once only and using mpi - fragile
mo-marqh Jul 1, 2026
c8d793a
refactor file_handler_open|close pattern
mo-marqh Jul 21, 2026
1c03cef
extend access pattern to file_handler_read_map
mo-marqh Jul 21, 2026
20c93a7
works for lfric_atm test case
mo-marqh Jul 22, 2026
30faa14
f
mo-marqh Aug 10, 2026
66cbddc
fix test refactor mistakes
mo-marqh Aug 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 44 additions & 5 deletions components/driver/source/driver_mesh_mod.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,11 @@ module driver_mesh_mod
use finite_element_config_mod, only: cellshape_quadrilateral
use base_mesh_config_mod, only: geometry_spherical, &
topology_fully_periodic
use timing_mod, only: start_timing, stop_timing, &
tik, LPROF
use ugrid_2d_mod, only: ugrid_2d_type
use ugrid_file_mod, only: ugrid_file_type
use ncdf_quad_mod, only: ncdf_quad_type

implicit none

Expand Down Expand Up @@ -146,7 +151,12 @@ subroutine init_mesh( config, &
character(str_def) :: fmt_str, number_str

integer(i_def) :: i, n_digit
integer(tik) :: timing_id
type(ugrid_2d_type) :: ugrid_2d

class(ugrid_file_type), allocatable :: file_handler

if ( LPROF ) call start_timing(timing_id, 'driver.init_mesh')
!============================================================================
! Extract configuration variables
!============================================================================
Expand Down Expand Up @@ -212,7 +222,6 @@ subroutine init_mesh( config, &
allocate(names, source=mesh_names)
end if


!===========================================================================
! Create local mesh objects:
! Two code pathes presented, either:
Expand Down Expand Up @@ -245,6 +254,18 @@ subroutine init_mesh( config, &
write(number_str, fmt_str) local_rank
write(input_mesh_file, '(A, "_", A, "-", I0, ".nc")') &
trim(file_prefix), trim(number_str), total_ranks
!====
! Open mesh file
! Once!
!====
write(log_scratch_space, '(A)') &
'opening file: "'// trim(input_mesh_file) // &
'" with MPI-IO for shared access'
call log_event(log_scratch_space, LOG_LEVEL_debug)

allocate( ncdf_quad_type :: file_handler )
call ugrid_2d%set_file_handler( file_handler )
call ugrid_2d%file_handler_open(trim(input_mesh_file))

call log_event( 'Using pre-partitioned mesh file:', log_level_debug )
call log_event( ' '//trim(input_mesh_file), log_level_debug )
Expand All @@ -256,7 +277,7 @@ subroutine init_mesh( config, &
!===========================================================
! Each partitioned mesh file will contain meshes of the
! same name as all other partitions.
call load_local_mesh( input_mesh_file, mesh_names )
call load_local_mesh( input_mesh_file, mesh_names, ugrid_2d )

! Apply configuration related checks to ensure that these
! meshes are suitable for the supplied application
Expand All @@ -273,7 +294,7 @@ subroutine init_mesh( config, &
! need to be loaded after the relevant local meshes have
! been loaded.
tmp_mesh_names = local_mesh_collection%get_mesh_names()
call load_local_mesh_maps( input_mesh_file, tmp_mesh_names )
call load_local_mesh_maps( input_mesh_file, tmp_mesh_names, ugrid_2d )
if (allocated(tmp_mesh_names)) deallocate(tmp_mesh_names)

else
Expand All @@ -298,6 +319,19 @@ subroutine init_mesh( config, &
end if
write(input_mesh_file,'(A)') trim(file_prefix) // '.nc'

!====
! Open mesh file
! Once!
!====
write(log_scratch_space, '(A)') &
'opening file: "'// trim(input_mesh_file) // &
'" with MPI-IO for shared access'
call log_event(log_scratch_space, LOG_LEVEL_debug)

allocate( ncdf_quad_type :: file_handler )
call ugrid_2d%set_file_handler( file_handler )
call ugrid_2d%file_handler_open(trim(input_mesh_file))

! 2.2a Set constants that will control partitioning.
!===========================================================
call get_partition_parameters( config%partitioning, mesh_selection, &
Expand All @@ -306,7 +340,7 @@ subroutine init_mesh( config, &

! 2.2b Read in all global meshes from input file
!===========================================================
call load_global_mesh( input_mesh_file, mesh_names )
call load_global_mesh( input_mesh_file, mesh_names, ugrid_2d )

! 2.2c Apply configuration related checks to ensure that these
! meshes are suitable for the supplied application
Expand All @@ -328,13 +362,17 @@ subroutine init_mesh( config, &
! 2.2f Read in the global intergrid mesh mappings,
! then create the associated local mesh maps
!===========================================================
call create_local_mesh_maps( input_mesh_file )
call create_local_mesh_maps( input_mesh_file, ugrid_2d )

! Clear the global mesh
call global_mesh_collection%clear()

end if ! prepartitioned

call ugrid_2d%file_handler_close()
if (allocated(file_handler)) deallocate( file_handler )


!============================================================================
! 3.0 Extrude the specified meshes from local mesh objects into
! mesh objects on the given extrusion.
Expand All @@ -350,6 +388,7 @@ subroutine init_mesh( config, &
call assign_mesh_maps(mesh_names)

deallocate(stencil_depths)
if ( LPROF ) call stop_timing(timing_id, 'driver.init_mesh')

end subroutine init_mesh

Expand Down
12 changes: 8 additions & 4 deletions components/driver/source/mesh/load_global_mesh_mod.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ module load_global_mesh_mod


use global_mesh_collection_mod, only: global_mesh_collection
use ugrid_2d_mod, only: ugrid_2d_type

implicit none

Expand All @@ -37,18 +38,19 @@ module load_global_mesh_mod
!> @param[in] mesh_names The names of the global meshes to load
!> from the <input_mesh_file>.
subroutine load_global_mesh_multiple( input_mesh_file, &
mesh_names )
mesh_names, ugrid_2d )

implicit none

character(str_max_filename), intent(in) :: input_mesh_file
character(str_def), intent(in) :: mesh_names(:)

integer(i_def) :: i
type(ugrid_2d_type), intent(inout) :: ugrid_2d

do i=1, size(mesh_names)
call load_global_mesh_single( input_mesh_file, &
mesh_names(i) )
mesh_names(i), ugrid_2d )
end do

end subroutine load_global_mesh_multiple
Expand All @@ -61,7 +63,7 @@ end subroutine load_global_mesh_multiple
!> @param[in] mesh_name The name of the global mesh to load
!> from the <input_mesh_file>.
subroutine load_global_mesh_single( input_mesh_file, &
mesh_name )
mesh_name, ugrid_2d )

implicit none

Expand All @@ -71,6 +73,8 @@ subroutine load_global_mesh_single( input_mesh_file, &
type(ugrid_mesh_data_type) :: ugrid_mesh_data
type(global_mesh_type) :: global_mesh

type(ugrid_2d_type), intent(inout) :: ugrid_2d

if (.not. global_mesh_collection%check_for(mesh_name)) then

write(log_scratch_space,'(A)') &
Expand All @@ -79,7 +83,7 @@ subroutine load_global_mesh_single( input_mesh_file, &

! Load mesh data into global_mesh
call ugrid_mesh_data%read_from_file( trim(input_mesh_file), &
mesh_name )
mesh_name, ugrid_2d )

global_mesh = global_mesh_type( ugrid_mesh_data )
call ugrid_mesh_data%clear()
Expand Down
25 changes: 11 additions & 14 deletions components/driver/source/mesh/load_local_mesh_maps_mod.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ module load_local_mesh_maps_mod
log_scratch_space, &
LOG_LEVEL_ERROR
use ncdf_quad_mod, only: ncdf_quad_type
use ugrid_2d_mod, only: ugrid_2d_type


use local_mesh_collection_mod, only: local_mesh_collection
Expand Down Expand Up @@ -39,18 +40,19 @@ module load_local_mesh_maps_mod
!> @param[in] source_mesh_name The name of the local source mesh to
!> load maps from the <input_mesh_file>.
subroutine load_local_mesh_maps_multiple_source( input_mesh_file, &
source_mesh_names )
source_mesh_names, ugrid_2d )

implicit none

character(str_max_filename), intent(in) :: input_mesh_file
character(str_def), intent(in) :: source_mesh_names(:)
character(str_def), intent(inout) :: source_mesh_names(:)
type(ugrid_2d_type), intent(inout) :: ugrid_2d

integer(i_def) :: i

do i=1, size(source_mesh_names)
call load_local_mesh_maps_single_source( input_mesh_file, &
source_mesh_names(i) )
source_mesh_names(i), ugrid_2d )
end do

end subroutine load_local_mesh_maps_multiple_source
Expand All @@ -67,22 +69,21 @@ end subroutine load_local_mesh_maps_multiple_source
!> @param[in] source_mesh_name The name of the local source mesh to load
!> maps from the <input_mesh_file>.
subroutine load_local_mesh_maps_single_source( input_mesh_file, &
source_mesh_name )
source_mesh_name, ugrid_2d )

implicit none

character(str_max_filename), intent(in) :: input_mesh_file
character(str_def), intent(in) :: source_mesh_name
character(str_def), intent(inout) :: source_mesh_name

character(str_def), allocatable :: target_mesh_names(:)
integer(i_def), allocatable :: lid_mesh_map(:,:,:)

integer(i_def) :: i

type(ncdf_quad_type) :: file_handler

type(local_mesh_type), pointer :: source_mesh => null()
type(local_mesh_type), pointer :: target_mesh => null()
type(ugrid_2d_type), intent(inout) :: ugrid_2d

integer(i_def) :: target_mesh_id

Expand All @@ -100,15 +101,13 @@ subroutine load_local_mesh_maps_single_source( input_mesh_file, &

if (allocated(target_mesh_names)) then

call file_handler%file_open(trim(input_mesh_file))

do i=1, size(target_mesh_names)
if ( local_mesh_collection%check_for( target_mesh_names(i) ) ) then

! Read in the local mesh map.
call file_handler%read_map( source_mesh_name, &
target_mesh_names(i), &
lid_mesh_map )
call ugrid_2d%file_handler_read_map( source_mesh_name, &
target_mesh_names(i), &
lid_mesh_map )

target_mesh &
=> local_mesh_collection%get_local_mesh(target_mesh_names(i))
Expand All @@ -123,8 +122,6 @@ subroutine load_local_mesh_maps_single_source( input_mesh_file, &
end if
end do

call file_handler%file_close()

deallocate( target_mesh_names )

end if
Expand Down
11 changes: 7 additions & 4 deletions components/driver/source/mesh/load_local_mesh_mod.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ module load_local_mesh_mod
use sci_query_mod, only: is_lbc

use local_mesh_collection_mod, only: local_mesh_collection
use ugrid_2d_mod, only: ugrid_2d_type

implicit none

Expand All @@ -37,16 +38,17 @@ module load_local_mesh_mod
!> @param[in] mesh_names The names of the local mesh data to
!> load from the <input_mesh_file>.
subroutine load_local_mesh_multiple( input_mesh_file, &
mesh_names )
mesh_names, ugrid_2d )
implicit none

character(str_max_filename), intent(in) :: input_mesh_file
character(str_def), intent(in) :: mesh_names(:)

integer(i_def) :: i
type(ugrid_2d_type), intent(inout) :: ugrid_2d

do i=1, size(mesh_names)
call load_local_mesh_single( input_mesh_file, mesh_names(i) )
call load_local_mesh_single( input_mesh_file, mesh_names(i), ugrid_2d )
end do

end subroutine load_local_mesh_multiple
Expand All @@ -59,7 +61,7 @@ end subroutine load_local_mesh_multiple
!> @param[in] mesh_name The name of the local mesh data to
!> load from the <input_mesh_file>.
subroutine load_local_mesh_single( input_mesh_file, &
mesh_name )
mesh_name, ugrid_2d )

implicit none

Expand All @@ -70,11 +72,12 @@ subroutine load_local_mesh_single( input_mesh_file, &
type(local_mesh_type) :: local_mesh

integer(i_def) :: local_mesh_id
type(ugrid_2d_type), intent(inout) :: ugrid_2d

if (.not. local_mesh_collection%check_for(mesh_name)) then

! Load mesh data into local_mesh
call ugrid_mesh_data%read_from_file( input_mesh_file, mesh_name )
call ugrid_mesh_data%read_from_file( input_mesh_file, mesh_name, ugrid_2d )

if (ugrid_mesh_data%contains_mesh()) then

Expand Down
14 changes: 6 additions & 8 deletions components/driver/source/mesh/runtime_partition_mod.f90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ module runtime_partition_mod

use local_mesh_collection_mod, only: local_mesh_collection
use global_mesh_collection_mod, only: global_mesh_collection
use ugrid_2d_mod, only: ugrid_2d_type

implicit none

Expand Down Expand Up @@ -198,13 +199,13 @@ end subroutine create_local_mesh
!! mesh object.
!!
!> @param[in] input_mesh_file Input file to load mesh maps from.
subroutine create_local_mesh_maps( input_mesh_file )
subroutine create_local_mesh_maps( input_mesh_file, ugrid_2d )

implicit none

character(len=str_max_filename) :: input_mesh_file

type(ncdf_quad_type) :: file_handler
type(ugrid_2d_type), intent(inout) :: ugrid_2d

character(str_def), allocatable :: source_mesh_names(:)
character(str_def), allocatable :: target_mesh_names(:)
Expand All @@ -225,7 +226,6 @@ subroutine create_local_mesh_maps( input_mesh_file )

! Read in the maps for each global mesh
!=================================================================
call file_handler%file_open(trim(input_mesh_file))

allocate( source_mesh_names, &
source=global_mesh_collection%get_mesh_names() )
Expand All @@ -251,9 +251,9 @@ subroutine create_local_mesh_maps( input_mesh_file )
if ( associated(target_local_mesh) ) then

! Read in the global mesh map
call file_handler%read_map( source_mesh_names(i), &
target_mesh_names(j), &
gid_mesh_map )
call ugrid_2d%file_handler_read_map( source_mesh_names(i), &
target_mesh_names(j), &
gid_mesh_map )

! Create the local mesh map
ntarget_per_source_cell_x = size(gid_mesh_map, 1)
Expand Down Expand Up @@ -296,8 +296,6 @@ subroutine create_local_mesh_maps( input_mesh_file )
deallocate( source_mesh_names)
end if

call file_handler%file_close()

return
end subroutine create_local_mesh_maps

Expand Down
Loading
Loading