Skip to content

PEM/ERP fixes for conusx4v1pg2_r05_oECv3 #8554

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rljacob merged 3 commits into from
Jul 22, 2026
Merged

PEM/ERP fixes for conusx4v1pg2_r05_oECv3 #8554

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7d83c70
Fix atm-ocn flux scatter by aliasing mbofxid to mboxid
vijaysm Jul 11, 2026
b684d84
Fix land-fraction restart scramble for non-samegrid land
vijaysm Jul 11, 2026
f3a31fb
Remove dead locals in cplcomp_moab_init_ocn after mbofxid alias
vijaysm Jul 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 14 additions & 59 deletions driver-moab/main/cplcomp_exchange_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ module cplcomp_exchange_mod
character(*),parameter :: subName = '(seq_mctext_mct)'
real(r8),parameter :: c1 = 1.0_r8
integer, parameter :: ATM_PHYS_ID_OFFSET = 200
integer, parameter :: OCN_SECOND_COPY_ID_OFFSET = 1000

!=======================================================================
contains
Expand Down Expand Up @@ -414,7 +413,6 @@ subroutine cplcomp_moab_init_ocn(infodata, comp, id_old, id_join, mpicom_old, mp

use iMOAB, only: iMOAB_WriteMesh, iMOAB_GetMeshInfo
use seq_infodata_mod, only: seq_infodata_type, seq_infodata_GetData
use shr_moab_mod, only: mbGetnCells, mbGetCellTagVals, mbSetCellTagVals

type(seq_infodata_type), intent(in) :: infodata
type(component_type), intent(inout) :: comp
Expand All @@ -429,10 +427,7 @@ subroutine cplcomp_moab_init_ocn(infodata, comp, id_old, id_join, mpicom_old, mp
integer :: ierr
character*200 :: appname, outfile, wopts, ropts, infile
character(CL) :: ocn_domain
character(CXX) :: tagname
integer :: nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3)
real(r8), allocatable :: tagValues(:)
integer :: arrsize, nloc

call seq_comm_getinfo(cplid ,mpigrp=mpigrp_cplid) ! receiver group
call seq_comm_getinfo(id_old,mpigrp=mpigrp_old) ! component group pes
Expand Down Expand Up @@ -527,62 +522,22 @@ subroutine cplcomp_moab_init_ocn(infodata, comp, id_old, id_join, mpicom_old, mp
call moab_exchange_domain_tags(comp, mpoid, mboxid, 'lat:lon:area:frac:mask', 'domo')
endif

!!!!!!!!! OCEAN 2nd COPY
! start copy
! Do another ocean copy on the coupler so So_fswpen does not collide between
! xao states and o2x states. Preserve the explicit second-copy context offset.
id_join = id_join + OCN_SECOND_COPY_ID_OFFSET

!!!!!! OCEAN COMPONENT
if (MPI_COMM_NULL /= mpicom_old ) then ! it means we are on the component pes (ocean)
!!!! FULL OCEAN
if ( trim(ocn_domain) == 'none' ) then
! send mesh to coupler, the second time! a copy would be cheaper
call moab_send_mesh(mpoid, mpicom_join, mpigrp_cplid, id_join, partMethod, subname)
endif
endif

!!!!!! ON 2nd OCN ON CPL
!!!!!!!!! OCEAN 2nd COPY (mbofxid) -- ALIAS of mboxid (robust fix)
! mbofxid shares mboxid's MOAB app / mesh / local ordering instead of being an
! independent second load. Two independent LoadMesh (data ocn) or send/receive
! (full ocn) calls do NOT guarantee identical local (handle) ordering: each rank
! owns the same ocean cells but in a different order, which scattered every by-index
! mboxid<->mbofxid transfer (atm-ocn flux xao, ofrac, domain copy, albedo, budget
! diagnostics) whenever the PE layout changed (PEM/ERP). Sharing one mesh makes all
! by-index access correct by construction, for both data and active ocean.
!
! The single field that lived on BOTH o2x and xao states, So_fswpen, is renamed to
! So_fswpen_ao on the xao side (seq_flds_mod) so the two do not collide on the shared
! mesh. The xao->atm second-hop map reuses mboxid's o2x->atm comm graph (prep_atm_mod);
! no separate app id / context offset is needed for mbofxid any more.
if (MPI_COMM_NULL /= mpicom_new ) then ! we are on the coupler pes
appname = "COUPLE_MPASOF"
! migrated mesh gets another app id, moab ocean to coupler (mbox)
call moab_register_app(appname, mpicom_new, id_join, mbofxid, subname)
!!!!! FULL OCN
if ( trim(ocn_domain) == 'none' ) then
call moab_receive_mesh(mbofxid, mpicom_join, mpigrp_old, id_old, subname)
!!!!! DATA OCN
else
! we need to read the ocean mesh on coupler, from domain file
infile = trim(ocn_domain)
ropts = 'PARALLEL=READ_PART;PARTITION_METHOD=SQIJ;VARIABLE=;NO_CULLING;REPARTITION'
if (seq_comm_iamroot(CPLID)) then
write(logunit,'(A)') subname//' load ocn domain mesh from file for second ocn instance '//trim(ocn_domain) &
, ' with options '//trim(ropts)
endif
call moab_load_mesh(mbofxid, infile, ropts, 0, subname)
call moab_define_global_id_tag(mbofxid, subname)
endif

!! ON 2nd OCEAN ON COUPLER
call moab_define_double_tag(mbofxid, trim(seq_flds_dom_fields)//":norm8wt", subname)

! copy domain data to mbofxid
nloc = mbGetnCells(mbofxid)
arrsize=nloc*5
allocate(tagValues(arrsize))
call mbGetCellTagVals(mboxid,'lat:lon:area:frac:mask'//C_NULL_CHAR,tagValues,arrsize)
call mbSetCellTagVals(mbofxid,'lat:lon:area:frac:mask'//C_NULL_CHAR,tagValues,arrsize)
deallocate(tagValues)

endif

!!!!!! ON OCN COMPONENT
if (mpoid .ge. 0) then ! we are on component ocn pes again, release buffers
if ( trim(ocn_domain) == 'none' ) then
call moab_free_sender_buffers(mpoid, id_join, subname)
endif
mbofxid = mboxid
endif
! end copy
#ifdef MOABDEBUG
if (mbofxid >= 0) then
outfile = 'recMeshOcnF.h5m'//C_NULL_CHAR
Expand Down
53 changes: 12 additions & 41 deletions driver-moab/main/prep_atm_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,14 +492,16 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at
! OCN for the intersection of OCN-ATM context (coverage)
call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID)

! we identified the app mbofxid with !id_join = id_join + 1000! kind of random
! line 1267 in cplcomp_exchange_mod.F90
context_id = ocn(1)%cplcompid + 1000
mapper_Sof2a%src_mbid = mbofxid
! mbofxid is now an ALIAS of mboxid (same MOAB app / mesh; see cplcomp_moab_init_ocn).
! The xao->atm second hop therefore REUSES mboxid's o2x->atm comm graph exactly like
! mapper_Fo2a does (same src_mbid, src_context and intx_context as mapper_So2a). No
! separate app id, context offset (formerly ocn%cplcompid+1000), or ComputeCommGraph
! is needed -- the (mboxid -> mbintxoa) graph under (ocn%cplcompid, idintx) already exists.
mapper_Sof2a%src_mbid = mboxid
mapper_Sof2a%tgt_mbid = mbaxid
mapper_Sof2a%intx_mbid = mbintxoa
mapper_Sof2a%src_context = context_id
mapper_Sof2a%intx_context = mapper_So2a%intx_context ! basically will use the same intx as ocean on coupler
mapper_Sof2a%src_context = mapper_So2a%src_context ! ocn(1)%cplcompid
mapper_Sof2a%intx_context = mapper_So2a%intx_context ! same intx as ocean on coupler
mapper_Sof2a%weight_identifier = wgtIdSo2a
mapper_Sof2a%mbname = 'mapper_Sof2a'

Expand All @@ -508,45 +510,14 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at
write(logunit,F00) 'Initializing MOAB mapper_Fof2a with copy of mapper_Sof2a'
endif
! now take care of the mapper
mapper_Fof2a%src_mbid = mbofxid
mapper_Fof2a%src_mbid = mboxid
mapper_Fof2a%tgt_mbid = mbaxid
mapper_Fof2a%intx_mbid = mbintxoa
mapper_Fof2a%src_context = mapper_Sof2a%src_context ! we use the same source 1000 + ?
mapper_Fof2a%intx_context = mapper_Sof2a%intx_context ! depends on samegrid_ao
mapper_Fof2a%src_context = mapper_So2a%src_context ! ocn(1)%cplcompid
mapper_Fof2a%intx_context = mapper_So2a%intx_context
mapper_Fof2a%weight_identifier = wgtIdFo2a
mapper_Fof2a%mbname = 'mapper_Fof2a'

type1 = 3; ! fv for ocean and atm;
if (.not. samegrid_ao) then
! Coverage/intersection mesh always has FV cells with GLOBAL_IDs,
! so type2 must be 3 (element-based matching), regardless of
! atm_pg_active. Using type2=2 (vertex matching) here would cause
! MOAB to match against vertex GLOBAL_IDs of mbintxoa instead of
! element GLOBAL_IDs, leading to an incorrect comm graph and
! heap corruption in iMOAB_ReceiveElementTag.
ierr = iMOAB_ComputeCommGraph( mbofxid, mbintxoa, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, 3, &
context_id, idintx)
if (ierr .ne. 0) then
write(logunit,*) subname,' error in computing comm graph for Sof2a, mbofxid-mbintxoa'
call shr_sys_abort(subname//' ERROR in computing comm graph for Sof2a, mbofxid-mbintxoa')
endif
else
! samegrid: comm graph to ATM mesh directly
! type2 depends on ATM discretization (PC for spectral, FV for PG2)
if (atm_pg_active .or. dead_comps) then
type2 = 3
else
type2 = 2 ! PC cloud
endif
! this is a case appearing in the data ocean case --res ne4pg2_ne4pg2 --compset FAQP
! also in spectral case, monogrid --res ne4_ne4 --compset F2010-SCREAMv1 ( type2 is 2, point cloud )
ierr = iMOAB_ComputeCommGraph( mbofxid, mbaxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, &
context_id, atm(1)%cplcompid )
if (ierr .ne. 0) then
write(logunit,*) subname,' error in computing communication graph for second hop, ATM-OCN'
call shr_sys_abort(subname//' ERROR in computing communication graph for second hop, ATM-OCN')
endif
endif
! comm graph reused from mapper_So2a/Fo2a (aliased mbofxid == mboxid); nothing to compute
endif

endif ! endif (ocn_present) then
Expand Down
6 changes: 4 additions & 2 deletions driver-moab/main/seq_flux_mct.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1607,7 +1607,7 @@ subroutine seq_flux_atmocn_moab(infodata, tod, dt, a2x, o2x, xao, mbid, mbfid)
index_xao_Faox_lwup = mct_aVect_indexRA(xao,'Faox_lwup')
index_xao_Faox_swdn = mct_aVect_indexRA(xao,'Faox_swdn')
index_xao_Faox_swup = mct_aVect_indexRA(xao,'Faox_swup')
index_xao_So_fswpen = mct_aVect_indexRA(xao,'So_fswpen')
Comment thread
rljacob marked this conversation as resolved.
index_xao_So_fswpen = mct_aVect_indexRA(xao,'So_fswpen_ao')
index_xao_So_warm_diurn = mct_aVect_indexRA(xao,'So_warm_diurn')
index_xao_So_salt_diurn = mct_aVect_indexRA(xao,'So_salt_diurn')
index_xao_So_speed_diurn = mct_aVect_indexRA(xao,'So_speed_diurn')
Expand Down Expand Up @@ -1935,7 +1935,9 @@ subroutine seq_flux_atmocn_moab(infodata, tod, dt, a2x, o2x, xao, mbid, mbfid)
call mbSetCellTagVals(mbfid, 'So_u10', u10res, nloc)
u10gust = sqrt(duu10n)
call mbSetCellTagVals(mbfid, 'So_u10withgusts', u10gust, nloc)
call mbSetCellTagVals(mbfid, 'So_fswpen', fswpen, nloc)
! xao copy of fswpen: renamed So_fswpen_ao so it does not collide with the o2x So_fswpen
! tag on the shared (aliased) ocn coupler mesh. See seq_flds_mod / cplcomp_moab_init_ocn.
call mbSetCellTagVals(mbfid, 'So_fswpen_ao', fswpen, nloc)

#ifdef MOABDEBUG
! debug out file
Expand Down
12 changes: 10 additions & 2 deletions driver-moab/main/seq_rest_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, &

integer (in), pointer :: l2racc_lm_cnt
integer (in) :: nx_lnd ! will be used if land and atm are on same grid
integer (in) :: lnd_nx, lnd_ny ! global land grid dims (from infodata), for non-samegrid land restart I/O
integer (in) :: ngv, nge ! global num vertices / elements from iMOAB_GetGlobalInfo
integer (in) :: ierr

Expand Down Expand Up @@ -523,6 +524,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, &
iac_prognostic=iac_prognostic, &
ocn_c2_glcshelf=ocn_c2_glcshelf, &
do_bgc_budgets=do_bgc_budgets, &
lnd_nx=lnd_nx, lnd_ny=lnd_ny, &
case_name=case_name, &
model_doi_url=model_doi_url)

Expand Down Expand Up @@ -667,9 +669,14 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, &
'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin'
whead=whead, wdata=wdata, nx=nx_lnd)
else
! land on its own (masked) grid, distinct from atm and rof: size the
! PIO global dim from the land grid dims in infodata, exactly as the
! cpl.hi history write does (seq_hist_mod). mblxid's num_global_elems
! is the packed active-cell count and is smaller than the max land
! GLOBAL_ID, which would scramble lfrac/lfrin on restart.
call seq_io_write(rest_file, mblxid, 'fractions_lx', &
'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin'
whead=whead, wdata=wdata)
whead=whead, wdata=wdata, nx=lnd_nx, ny=lnd_ny)
endif
endif
if (lnd_present .and. rof_prognostic) then
Expand Down Expand Up @@ -698,9 +705,10 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, &
trim(tagname), &
whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=nx_lnd)
else
! land on its own (masked) grid: size PIO global dim from infodata land dims
call seq_io_write(rest_file, mblxid, 'l2racc_lx', &
trim(tagname), &
whead=whead, wdata=wdata, matrix = p_l2racc_lm )
whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=lnd_nx, ny=lnd_ny )
endif
call seq_io_write(rest_file, l2racc_lm_cnt, 'l2racc_lx_cnt', &
whead=whead, wdata=wdata)
Expand Down
10 changes: 9 additions & 1 deletion driver-moab/shr/seq_flds_mod.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1838,13 +1838,21 @@ subroutine seq_flds_set(nmlfile, ID, infodata)
attname = 'So_bldepth'
call metadata_set(attname, longname, stdname, units)

call seq_flds_add(xao_states,"So_fswpen")
! NOTE: So_fswpen is the one field carried by BOTH o2x and xao states. In the MOAB
! coupler the atm-ocn "flux" mesh (mbofxid) is an alias of the ocn coupler mesh (mboxid),
! so o2x and xao tags live on the SAME MOAB mesh. To avoid a tag collision, the xao copy
! is named So_fswpen_ao while o2x keeps So_fswpen. (So_fswpen is consumed only by the
! atm-ocn flux as an o2x input; the xao copy is not merged to x2a/x2o, so this rename is
! transparent.) See cplcomp_moab_init_ocn (alias) and seq_flux_atmocn_moab (write).
call seq_flds_add(xao_states,"So_fswpen_ao")
call seq_flds_add(o2x_states,"So_fswpen")
longname = 'Fraction of sw penetrating surface layer for diurnal cycle'
stdname = 'Fraction of sw penetrating surface layer for diurnal cycle'
units = '1'
attname = 'So_fswpen'
call metadata_set(attname, longname, stdname, units)
attname = 'So_fswpen_ao'
call metadata_set(attname, longname, stdname, units)

!------------------------------
! ice<->ocn only exchange - BGC
Expand Down