diff --git a/driver-moab/main/cplcomp_exchange_mod.F90 b/driver-moab/main/cplcomp_exchange_mod.F90 index e41d3ebe1d0f..0e0a9d9d8db2 100644 --- a/driver-moab/main/cplcomp_exchange_mod.F90 +++ b/driver-moab/main/cplcomp_exchange_mod.F90 @@ -76,7 +76,6 @@ module cplcomp_exchange_mod character(*),parameter :: subName = '(seq_mctext_mct)' real(r8),parameter :: c1 = 1.0_r8 integer, parameter :: ATM_PHYS_ID_OFFSET = 200 - integer, parameter :: OCN_SECOND_COPY_ID_OFFSET = 1000 !======================================================================= contains @@ -414,7 +413,6 @@ subroutine cplcomp_moab_init_ocn(infodata, comp, id_old, id_join, mpicom_old, mp use iMOAB, only: iMOAB_WriteMesh, iMOAB_GetMeshInfo use seq_infodata_mod, only: seq_infodata_type, seq_infodata_GetData - use shr_moab_mod, only: mbGetnCells, mbGetCellTagVals, mbSetCellTagVals type(seq_infodata_type), intent(in) :: infodata type(component_type), intent(inout) :: comp @@ -429,10 +427,7 @@ subroutine cplcomp_moab_init_ocn(infodata, comp, id_old, id_join, mpicom_old, mp integer :: ierr character*200 :: appname, outfile, wopts, ropts, infile character(CL) :: ocn_domain - character(CXX) :: tagname integer :: nvert(3), nvise(3), nbl(3), nsurf(3), nvisBC(3) - real(r8), allocatable :: tagValues(:) - integer :: arrsize, nloc call seq_comm_getinfo(cplid ,mpigrp=mpigrp_cplid) ! receiver group call seq_comm_getinfo(id_old,mpigrp=mpigrp_old) ! component group pes @@ -527,62 +522,22 @@ subroutine cplcomp_moab_init_ocn(infodata, comp, id_old, id_join, mpicom_old, mp call moab_exchange_domain_tags(comp, mpoid, mboxid, 'lat:lon:area:frac:mask', 'domo') endif -!!!!!!!!! OCEAN 2nd COPY - ! start copy - ! Do another ocean copy on the coupler so So_fswpen does not collide between - ! xao states and o2x states. Preserve the explicit second-copy context offset. - id_join = id_join + OCN_SECOND_COPY_ID_OFFSET - -!!!!!! OCEAN COMPONENT - if (MPI_COMM_NULL /= mpicom_old ) then ! it means we are on the component pes (ocean) - !!!! FULL OCEAN - if ( trim(ocn_domain) == 'none' ) then - ! send mesh to coupler, the second time! a copy would be cheaper - call moab_send_mesh(mpoid, mpicom_join, mpigrp_cplid, id_join, partMethod, subname) - endif - endif - -!!!!!! ON 2nd OCN ON CPL +!!!!!!!!! OCEAN 2nd COPY (mbofxid) -- ALIAS of mboxid (robust fix) + ! mbofxid shares mboxid's MOAB app / mesh / local ordering instead of being an + ! independent second load. Two independent LoadMesh (data ocn) or send/receive + ! (full ocn) calls do NOT guarantee identical local (handle) ordering: each rank + ! owns the same ocean cells but in a different order, which scattered every by-index + ! mboxid<->mbofxid transfer (atm-ocn flux xao, ofrac, domain copy, albedo, budget + ! diagnostics) whenever the PE layout changed (PEM/ERP). Sharing one mesh makes all + ! by-index access correct by construction, for both data and active ocean. + ! + ! The single field that lived on BOTH o2x and xao states, So_fswpen, is renamed to + ! So_fswpen_ao on the xao side (seq_flds_mod) so the two do not collide on the shared + ! mesh. The xao->atm second-hop map reuses mboxid's o2x->atm comm graph (prep_atm_mod); + ! no separate app id / context offset is needed for mbofxid any more. if (MPI_COMM_NULL /= mpicom_new ) then ! we are on the coupler pes - appname = "COUPLE_MPASOF" - ! migrated mesh gets another app id, moab ocean to coupler (mbox) - call moab_register_app(appname, mpicom_new, id_join, mbofxid, subname) - !!!!! FULL OCN - if ( trim(ocn_domain) == 'none' ) then - call moab_receive_mesh(mbofxid, mpicom_join, mpigrp_old, id_old, subname) - !!!!! DATA OCN - else - ! we need to read the ocean mesh on coupler, from domain file - infile = trim(ocn_domain) - ropts = 'PARALLEL=READ_PART;PARTITION_METHOD=SQIJ;VARIABLE=;NO_CULLING;REPARTITION' - if (seq_comm_iamroot(CPLID)) then - write(logunit,'(A)') subname//' load ocn domain mesh from file for second ocn instance '//trim(ocn_domain) & - , ' with options '//trim(ropts) - endif - call moab_load_mesh(mbofxid, infile, ropts, 0, subname) - call moab_define_global_id_tag(mbofxid, subname) - endif - - !! ON 2nd OCEAN ON COUPLER - call moab_define_double_tag(mbofxid, trim(seq_flds_dom_fields)//":norm8wt", subname) - - ! copy domain data to mbofxid - nloc = mbGetnCells(mbofxid) - arrsize=nloc*5 - allocate(tagValues(arrsize)) - call mbGetCellTagVals(mboxid,'lat:lon:area:frac:mask'//C_NULL_CHAR,tagValues,arrsize) - call mbSetCellTagVals(mbofxid,'lat:lon:area:frac:mask'//C_NULL_CHAR,tagValues,arrsize) - deallocate(tagValues) - - endif - -!!!!!! ON OCN COMPONENT - if (mpoid .ge. 0) then ! we are on component ocn pes again, release buffers - if ( trim(ocn_domain) == 'none' ) then - call moab_free_sender_buffers(mpoid, id_join, subname) - endif + mbofxid = mboxid endif - ! end copy #ifdef MOABDEBUG if (mbofxid >= 0) then outfile = 'recMeshOcnF.h5m'//C_NULL_CHAR diff --git a/driver-moab/main/prep_atm_mod.F90 b/driver-moab/main/prep_atm_mod.F90 index afb1bda84937..a33e352d3932 100644 --- a/driver-moab/main/prep_atm_mod.F90 +++ b/driver-moab/main/prep_atm_mod.F90 @@ -492,14 +492,16 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at ! OCN for the intersection of OCN-ATM context (coverage) call seq_comm_getinfo(CPLID ,mpigrp=mpigrp_CPLID) - ! we identified the app mbofxid with !id_join = id_join + 1000! kind of random - ! line 1267 in cplcomp_exchange_mod.F90 - context_id = ocn(1)%cplcompid + 1000 - mapper_Sof2a%src_mbid = mbofxid + ! mbofxid is now an ALIAS of mboxid (same MOAB app / mesh; see cplcomp_moab_init_ocn). + ! The xao->atm second hop therefore REUSES mboxid's o2x->atm comm graph exactly like + ! mapper_Fo2a does (same src_mbid, src_context and intx_context as mapper_So2a). No + ! separate app id, context offset (formerly ocn%cplcompid+1000), or ComputeCommGraph + ! is needed -- the (mboxid -> mbintxoa) graph under (ocn%cplcompid, idintx) already exists. + mapper_Sof2a%src_mbid = mboxid mapper_Sof2a%tgt_mbid = mbaxid mapper_Sof2a%intx_mbid = mbintxoa - mapper_Sof2a%src_context = context_id - mapper_Sof2a%intx_context = mapper_So2a%intx_context ! basically will use the same intx as ocean on coupler + mapper_Sof2a%src_context = mapper_So2a%src_context ! ocn(1)%cplcompid + mapper_Sof2a%intx_context = mapper_So2a%intx_context ! same intx as ocean on coupler mapper_Sof2a%weight_identifier = wgtIdSo2a mapper_Sof2a%mbname = 'mapper_Sof2a' @@ -508,45 +510,14 @@ subroutine prep_atm_init(infodata, ocn_c2_atm, ice_c2_atm, lnd_c2_atm, iac_c2_at write(logunit,F00) 'Initializing MOAB mapper_Fof2a with copy of mapper_Sof2a' endif ! now take care of the mapper - mapper_Fof2a%src_mbid = mbofxid + mapper_Fof2a%src_mbid = mboxid mapper_Fof2a%tgt_mbid = mbaxid mapper_Fof2a%intx_mbid = mbintxoa - mapper_Fof2a%src_context = mapper_Sof2a%src_context ! we use the same source 1000 + ? - mapper_Fof2a%intx_context = mapper_Sof2a%intx_context ! depends on samegrid_ao + mapper_Fof2a%src_context = mapper_So2a%src_context ! ocn(1)%cplcompid + mapper_Fof2a%intx_context = mapper_So2a%intx_context mapper_Fof2a%weight_identifier = wgtIdFo2a mapper_Fof2a%mbname = 'mapper_Fof2a' - - type1 = 3; ! fv for ocean and atm; - if (.not. samegrid_ao) then - ! Coverage/intersection mesh always has FV cells with GLOBAL_IDs, - ! so type2 must be 3 (element-based matching), regardless of - ! atm_pg_active. Using type2=2 (vertex matching) here would cause - ! MOAB to match against vertex GLOBAL_IDs of mbintxoa instead of - ! element GLOBAL_IDs, leading to an incorrect comm graph and - ! heap corruption in iMOAB_ReceiveElementTag. - ierr = iMOAB_ComputeCommGraph( mbofxid, mbintxoa, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, 3, & - context_id, idintx) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing comm graph for Sof2a, mbofxid-mbintxoa' - call shr_sys_abort(subname//' ERROR in computing comm graph for Sof2a, mbofxid-mbintxoa') - endif - else - ! samegrid: comm graph to ATM mesh directly - ! type2 depends on ATM discretization (PC for spectral, FV for PG2) - if (atm_pg_active .or. dead_comps) then - type2 = 3 - else - type2 = 2 ! PC cloud - endif - ! this is a case appearing in the data ocean case --res ne4pg2_ne4pg2 --compset FAQP - ! also in spectral case, monogrid --res ne4_ne4 --compset F2010-SCREAMv1 ( type2 is 2, point cloud ) - ierr = iMOAB_ComputeCommGraph( mbofxid, mbaxid, mpicom_CPLID, mpigrp_CPLID, mpigrp_CPLID, type1, type2, & - context_id, atm(1)%cplcompid ) - if (ierr .ne. 0) then - write(logunit,*) subname,' error in computing communication graph for second hop, ATM-OCN' - call shr_sys_abort(subname//' ERROR in computing communication graph for second hop, ATM-OCN') - endif - endif + ! comm graph reused from mapper_So2a/Fo2a (aliased mbofxid == mboxid); nothing to compute endif endif ! endif (ocn_present) then diff --git a/driver-moab/main/seq_flux_mct.F90 b/driver-moab/main/seq_flux_mct.F90 index 1965b04db809..39ce50c15013 100644 --- a/driver-moab/main/seq_flux_mct.F90 +++ b/driver-moab/main/seq_flux_mct.F90 @@ -1607,7 +1607,7 @@ subroutine seq_flux_atmocn_moab(infodata, tod, dt, a2x, o2x, xao, mbid, mbfid) index_xao_Faox_lwup = mct_aVect_indexRA(xao,'Faox_lwup') index_xao_Faox_swdn = mct_aVect_indexRA(xao,'Faox_swdn') index_xao_Faox_swup = mct_aVect_indexRA(xao,'Faox_swup') - index_xao_So_fswpen = mct_aVect_indexRA(xao,'So_fswpen') + index_xao_So_fswpen = mct_aVect_indexRA(xao,'So_fswpen_ao') index_xao_So_warm_diurn = mct_aVect_indexRA(xao,'So_warm_diurn') index_xao_So_salt_diurn = mct_aVect_indexRA(xao,'So_salt_diurn') index_xao_So_speed_diurn = mct_aVect_indexRA(xao,'So_speed_diurn') @@ -1935,7 +1935,9 @@ subroutine seq_flux_atmocn_moab(infodata, tod, dt, a2x, o2x, xao, mbid, mbfid) call mbSetCellTagVals(mbfid, 'So_u10', u10res, nloc) u10gust = sqrt(duu10n) call mbSetCellTagVals(mbfid, 'So_u10withgusts', u10gust, nloc) - call mbSetCellTagVals(mbfid, 'So_fswpen', fswpen, nloc) + ! xao copy of fswpen: renamed So_fswpen_ao so it does not collide with the o2x So_fswpen + ! tag on the shared (aliased) ocn coupler mesh. See seq_flds_mod / cplcomp_moab_init_ocn. + call mbSetCellTagVals(mbfid, 'So_fswpen_ao', fswpen, nloc) #ifdef MOABDEBUG ! debug out file diff --git a/driver-moab/main/seq_rest_mod.F90 b/driver-moab/main/seq_rest_mod.F90 index 14d963311ab7..1c8ae005f391 100644 --- a/driver-moab/main/seq_rest_mod.F90 +++ b/driver-moab/main/seq_rest_mod.F90 @@ -476,6 +476,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & integer (in), pointer :: l2racc_lm_cnt integer (in) :: nx_lnd ! will be used if land and atm are on same grid + integer (in) :: lnd_nx, lnd_ny ! global land grid dims (from infodata), for non-samegrid land restart I/O integer (in) :: ngv, nge ! global num vertices / elements from iMOAB_GetGlobalInfo integer (in) :: ierr @@ -523,6 +524,7 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & iac_prognostic=iac_prognostic, & ocn_c2_glcshelf=ocn_c2_glcshelf, & do_bgc_budgets=do_bgc_budgets, & + lnd_nx=lnd_nx, lnd_ny=lnd_ny, & case_name=case_name, & model_doi_url=model_doi_url) @@ -667,9 +669,14 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' whead=whead, wdata=wdata, nx=nx_lnd) else + ! land on its own (masked) grid, distinct from atm and rof: size the + ! PIO global dim from the land grid dims in infodata, exactly as the + ! cpl.hi history write does (seq_hist_mod). mblxid's num_global_elems + ! is the packed active-cell count and is smaller than the max land + ! GLOBAL_ID, which would scramble lfrac/lfrin on restart. call seq_io_write(rest_file, mblxid, 'fractions_lx', & 'afrac:lfrac:lfrin', & ! seq_frac_mod: character(*),parameter :: fraclist_l = 'afrac:lfrac:lfrin' - whead=whead, wdata=wdata) + whead=whead, wdata=wdata, nx=lnd_nx, ny=lnd_ny) endif endif if (lnd_present .and. rof_prognostic) then @@ -698,9 +705,10 @@ subroutine seq_rest_mb_write(EClock_d, seq_SyncClock, infodata, & trim(tagname), & whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=nx_lnd) else + ! land on its own (masked) grid: size PIO global dim from infodata land dims call seq_io_write(rest_file, mblxid, 'l2racc_lx', & trim(tagname), & - whead=whead, wdata=wdata, matrix = p_l2racc_lm ) + whead=whead, wdata=wdata, matrix = p_l2racc_lm, nx=lnd_nx, ny=lnd_ny ) endif call seq_io_write(rest_file, l2racc_lm_cnt, 'l2racc_lx_cnt', & whead=whead, wdata=wdata) diff --git a/driver-moab/shr/seq_flds_mod.F90 b/driver-moab/shr/seq_flds_mod.F90 index 99d6580183e9..f2f8bc74a502 100644 --- a/driver-moab/shr/seq_flds_mod.F90 +++ b/driver-moab/shr/seq_flds_mod.F90 @@ -1838,13 +1838,21 @@ subroutine seq_flds_set(nmlfile, ID, infodata) attname = 'So_bldepth' call metadata_set(attname, longname, stdname, units) - call seq_flds_add(xao_states,"So_fswpen") + ! NOTE: So_fswpen is the one field carried by BOTH o2x and xao states. In the MOAB + ! coupler the atm-ocn "flux" mesh (mbofxid) is an alias of the ocn coupler mesh (mboxid), + ! so o2x and xao tags live on the SAME MOAB mesh. To avoid a tag collision, the xao copy + ! is named So_fswpen_ao while o2x keeps So_fswpen. (So_fswpen is consumed only by the + ! atm-ocn flux as an o2x input; the xao copy is not merged to x2a/x2o, so this rename is + ! transparent.) See cplcomp_moab_init_ocn (alias) and seq_flux_atmocn_moab (write). + call seq_flds_add(xao_states,"So_fswpen_ao") call seq_flds_add(o2x_states,"So_fswpen") longname = 'Fraction of sw penetrating surface layer for diurnal cycle' stdname = 'Fraction of sw penetrating surface layer for diurnal cycle' units = '1' attname = 'So_fswpen' call metadata_set(attname, longname, stdname, units) + attname = 'So_fswpen_ao' + call metadata_set(attname, longname, stdname, units) !------------------------------ ! ice<->ocn only exchange - BGC