From 507d6b60f47ecc4271a3706f8ae0b0943d86a616 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Mon, 5 Feb 2024 14:34:35 -0500 Subject: [PATCH 01/43] First working attempt at implementing per-EB solints --- gaincal_wrapper.py | 8 +- image_analysis_helpers.py | 7 +- prepare_selfcal.py | 55 +++++++--- run_selfcal.py | 58 +++++++--- selfcal_helpers.py | 218 ++++++++++++++++++++------------------ 5 files changed, 210 insertions(+), 136 deletions(-) diff --git a/gaincal_wrapper.py b/gaincal_wrapper.py index d375c49a..6380ab14 100644 --- a/gaincal_wrapper.py +++ b/gaincal_wrapper.py @@ -260,7 +260,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap if mode != 'per_bb': gcdict=gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if applymode=="calflag" else False, - solint=solint.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + solint=selfcal_plan[vis]['solint_settings'][solint]['interval'].replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if applymode == 'calflag' else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ @@ -271,7 +271,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] gcdict=gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if applymode=="calflag" else False, - solint=solint.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + solint=selfcal_plan[vis]['solint_settings'][solint]['interval'].replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if applymode == 'calflag' else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ @@ -396,7 +396,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap if mode != 'per_bb': gcdict=gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if applymode=="calflag" else False, - solint=solint.replace('_EB','').replace('_ap','').replace('scan_',''),\ + solint=selfcal_plan[vis]['solint_settings'][solint]['interval'].replace('_EB','').replace('_ap','').replace('scan_',''),\ minsnr=gaincal_minsnr if applymode == 'calflag' else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=str(selfcal_library['sub-fields-fid_map'][vis][fid]),gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ @@ -407,7 +407,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] gcdict=gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if applymode=="calflag" else False, - solint=solint.replace('_EB','').replace('_ap','').replace('scan_',''),\ + solint=selfcal_plan[vis]['solint_settings'][solint]['interval'].replace('_EB','').replace('_ap','').replace('scan_',''),\ minsnr=gaincal_minsnr if applymode == 'calflag' else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=str(selfcal_library['sub-fields-fid_map'][vis][fid]),gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ diff --git a/image_analysis_helpers.py b/image_analysis_helpers.py index 652bdacf..96bf5593 100644 --- a/image_analysis_helpers.py +++ b/image_analysis_helpers.py @@ -15,7 +15,12 @@ def get_image_stats(image, mask, backup_mask, selfcal_library, use_nfmask, solin else: SNR_NF, RMS_NF = SNR, RMS - for vis in selfcal_library['vislist']: + if suffix in ['dirty','orig','initial','final']: + vislist = selfcal_library['vislist'] + else: + vislist = selfcal_library['vislist-to-gaincal'] + + for vis in vislist: if suffix in ['dirty','orig','initial','final']: if spw == 'all': update_dict = selfcal_library diff --git a/prepare_selfcal.py b/prepare_selfcal.py index eb8f757d..e0411684 100644 --- a/prepare_selfcal.py +++ b/prepare_selfcal.py @@ -446,14 +446,46 @@ def default(self, obj): selfcal_plan[target] = {} for band in selfcal_library[target]: - selfcal_plan[target][band] = {} if band in selfcal_library[target]: - selfcal_plan[target][band]['solints'],selfcal_plan[target][band]['integration_time'],selfcal_plan[target][band]['gaincal_combine'], \ - selfcal_plan[target][band]['solmode']=get_solints_simple(selfcal_library[target][band]['vislist'],scantimesdict[band], - scannfieldsdict[band],scanstartsdict[band],scanendsdict[band],integrationtimesdict[band],\ - inf_EB_gaincal_combine,do_amp_selfcal=do_amp_selfcal,mosaic=selfcal_library[target][band]['obstype'] == 'mosaic') + selfcal_plan[target][band] = {} + selfcal_plan[target][band]['solints'] = [] + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis] = {} + solints,selfcal_plan[target][band][vis]['integration_time'],selfcal_plan[target][band][vis]['gaincal_combine'], \ + selfcal_plan[target][band]['solmode']=get_solints_simple([vis],scantimesdict[band], + scannfieldsdict[band],scanstartsdict[band],scanendsdict[band],integrationtimesdict[band],\ + inf_EB_gaincal_combine,do_amp_selfcal=do_amp_selfcal,mosaic=selfcal_library[target][band]['obstype'] == 'mosaic') + + selfcal_plan[target][band][vis]['solint_settings']={} + + subscan_count = 0 + for solint in solints: + """ + if 'inf' in solint or 'int' in solint or 'ap' in solint: + solint_name = solint + else: + solint_name = 'subscan'+str(subscan_count) + subscan_count += 1 + """ + if solint == "inf_EB": + solint_name = "inf_EB" + else: + solint_name = "solint"+str(subscan_count) + if 'ap' in solint: + solint_name += '_ap' + subscan_count += 1 + + if solint_name not in selfcal_plan[target][band]['solints']: + selfcal_plan[target][band]['solints'].append(solint_name) + + selfcal_plan[target][band][vis]['solint_settings'][solint_name]={} + selfcal_plan[target][band][vis]['solint_settings'][solint_name]['interval'] = solint + + selfcal_plan[target][band]['applycal_mode']=[apply_cal_mode_default]*len(selfcal_plan[target][band]['solints']) + print(band,target,selfcal_plan[target][band]['solints']) - selfcal_plan[target][band]['applycal_mode']=[apply_cal_mode_default]*len(selfcal_plan[target][band]['solints']) + for vis in vislist: + print(vis,[selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] for solint in selfcal_plan[target][band]['solints']]) ## ## estimate per scan/EB S/N using time on source and median scan times @@ -462,7 +494,6 @@ def default(self, obj): for target in selfcal_plan: for band in selfcal_plan[target]: for vis in selfcal_library[target][band]['vislist']: - selfcal_plan[target][band][vis] = {} selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']=inf_EB_gaincal_combine #'scan' if selfcal_library[target][band]['obstype']=='mosaic': selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']+=',field' @@ -493,11 +524,11 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T if selfcal_library[target][band][vis]['baseband'][baseband]['nspws']> maxspws_per_bb: maxspws_per_bb=selfcal_library[target][band][vis]['baseband'][baseband]['nspws']+0.0 - selfcal_plan[target][band][vis]['solint_settings']={} + #selfcal_plan[target][band][vis]['solint_settings']={} for solint in selfcal_plan[target][band]['solints']: gaincal_combine='' filename_append='' - selfcal_plan[target][band][vis]['solint_settings'][solint]={} + #selfcal_plan[target][band][vis]['solint_settings'][solint]={} selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=False selfcal_plan[target][band][vis]['solint_settings'][solint]['gaincal_preapply_gaintable']=[] selfcal_plan[target][band][vis]['solint_settings'][solint]['gaincal_spwmap']=[] @@ -513,8 +544,8 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T selfcal_plan[target][band][vis]['solint_settings'][solint]['accepted_gaintable']='' selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=[] selfcal_plan[target][band][vis]['solint_settings'][solint]['gaincal_gaintype']='T' - min_SNR_spw=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_spw'][solint]) - min_SNR_bb=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_bb'][solint]) + min_SNR_spw=get_min_SNR_spw(selfcal_plan[target][band][vis]['solint_snr_per_spw'][solint]) + min_SNR_bb=get_min_SNR_spw(selfcal_plan[target][band][vis]['solint_snr_per_bb'][solint]) selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('combinespw') if 'spw' not in selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']: if min_SNR_spw > 2.0: @@ -581,7 +612,7 @@ def set_clean_thresholds(selfcal_library, selfcal_plan, dividing_factor=-99.0, r nsigma_init=np.max([selfcal_library[target][band]['SNR_NF_orig']/dividing_factor_band,5.0]) # count number of amplitude selfcal solints, repeat final clean depth of phase-only for amplitude selfcal - n_ap_solints=sum(1 for solint in selfcal_plan[target][band]['solints'] if 'ap' in solint) + n_ap_solints=sum(1 for solint in selfcal_plan[target][band]['solints'] if 'ap' in selfcal_plan[target][band][selfcal_library[target][band]['vislist'][0]]['solint_settings'][solint]['interval']) if rel_thresh_scaling == 'loge': selfcal_library[target][band]['nsigma'] = np.append(np.exp(np.linspace(np.log(nsigma_init),np.log(3.0),\ diff --git a/run_selfcal.py b/run_selfcal.py index 0e5defa2..b9ee32f0 100644 --- a/run_selfcal.py +++ b/run_selfcal.py @@ -41,8 +41,6 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, else: iteration = 0 - vislist=selfcal_library['vislist'].copy() - if mode == "cocal": # Check whether there are suitable calibrators, otherwise skip this target/band. include_targets, include_scans = triage_calibrators(vislist[0], target, calibrators[band][0]) @@ -56,7 +54,12 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, print('Starting selfcal procedure on: '+target+' '+band) while iteration < len(selfcal_plan['solints']): + vislist=[vis for vis in selfcal_library['vislist'] if selfcal_plan['solints'][iteration] in selfcal_plan[vis]['solint_settings']] + print("Solving for solint="+selfcal_plan['solints'][iteration]) + for vis in vislist: + print(" "+vis+": "+selfcal_plan['solints'][iteration]) + # Set some cocal parameters. if selfcal_plan['solints'][iteration] in ["inf_EB_fb","inf_fb1"]: @@ -82,11 +85,25 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, break # make sure the last phase-only selfcal table gets pre-applied # solints that use combinespw don't get pre-apply label by default - selfcal_plan[vis]['solint_settings'][selfcal_library['final_phase_solint']]['preapply_this_gaintable']=True + for vis in vislist: + selfcal_plan[vis]['solint_settings'][selfcal_library['final_phase_solint']]['preapply_this_gaintable']=True - - if mode == "selfcal" and selfcal_plan['solint_snr'][selfcal_plan['solints'][iteration]] < minsnr_to_proceed and np.all([selfcal_plan[fid]['solint_snr_per_field'][selfcal_plan['solints'][iteration]] < minsnr_to_proceed for fid in selfcal_library['sub-fields']]): - print('*********** estimated SNR for solint='+selfcal_plan['solints'][iteration]+' too low, measured: '+str(selfcal_plan['solint_snr'][selfcal_plan['solints'][iteration]])+', Min SNR Required: '+str(minsnr_to_proceed)+' **************') + if mode == "selfcal": + remove_vis = [] + for vis in vislist: + if selfcal_plan[vis]['solint_snr'][selfcal_plan['solints'][iteration]] < minsnr_to_proceed and \ + np.all([selfcal_plan[fid][vis]['solint_snr_per_field'][selfcal_plan['solints'][iteration]] < minsnr_to_proceed for fid in \ + selfcal_library['sub-fields']]): + print('*********** estimated SNR for EB='+vis+' for solint='+selfcal_plan['solints'][iteration]+' too low, measured: '+\ + str(selfcal_plan[vis]['solint_snr'][selfcal_plan['solints'][iteration]])+', Min SNR Required: '+str(minsnr_to_proceed)+\ + ' **************') + remove_vis.append(vis) + + for rvis in remove_vis: + vislist.remove(rvis) + + if len(vislist) == 0: + print('*********** estimated SNR for solint='+selfcal_plan['solints'][iteration]+' too low for all EBs **************') if iteration > 1 and selfcal_plan['solmode'][iteration] !='ap' and do_amp_selfcal: # if a solution interval shorter than inf for phase-only SC has passed, attempt amplitude selfcal iteration=selfcal_plan['solmode'].index('ap') print('****************Attempting amplitude selfcal*************') @@ -95,6 +112,11 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, selfcal_library['Stop_Reason']='Estimated_SNR_too_low_for_solint '+selfcal_plan['solints'][iteration] break else: + selfcal_library['vislist-to-gaincal'] = vislist + for fid in selfcal_library['sub-fields']: + selfcal_library[fid]['vislist-to-gaincal'] = [vis for vis in selfcal_library['vislist-to-gaincal'] if vis in + selfcal_library[fid]['vislist']] + solint=selfcal_plan['solints'][iteration] if iteration == 0: print('Starting with solint: '+solint) @@ -479,7 +501,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, selfcal_library[vis][solint]['Pass']=False for fid in selfcal_library['sub-fields-to-selfcal']: - for vis in selfcal_library[fid]['vislist']: + for vis in selfcal_library[fid]['vislist-to-gaincal']: selfcal_library[fid][vis][solint]['Pass']=False repeat_solint = False do_fallback_combinespw = False @@ -528,7 +550,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, if mosaic_reason[fid] == '': mosaic_reason[fid] = "Global selfcal failed" selfcal_library[fid]['Stop_Reason']=mosaic_reason[fid] - for vis in selfcal_library[fid]['vislist']: + for vis in selfcal_library[fid]['vislist-to-gaincal']: selfcal_library[fid][vis][solint]['Pass']=False selfcal_library[fid][vis][solint]['Fail_Reason']=mosaic_reason[fid] else: @@ -536,7 +558,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, # If any of the fields failed self-calibration, we need to re-apply calibrations for all fields because we need to revert flagging back # to the starting point. - if np.any([selfcal_library[fid][selfcal_library[fid]['vislist'][0]][solint]['Pass'] == False for fid in \ + if np.any([selfcal_library[fid][selfcal_library[fid]['vislist-to-gaincal'][0]][solint]['Pass'] == False for fid in \ selfcal_library['sub-fields-to-selfcal']]) or len(selfcal_library['sub-fields-to-selfcal']) < \ len(selfcal_library['sub-fields']): print('****************Selfcal failed for some sub-fields:*************') @@ -569,16 +591,18 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, if mode == "selfcal" and (iteration < len(selfcal_plan['solints'])-1) and (selfcal_library[vis][solint]['SNR_post'] > \ selfcal_library['SNR_orig']): #(iteration == 0) and print('Updating solint = '+selfcal_plan['solints'][iteration+1]+' SNR') - print('Was: ',selfcal_plan['solint_snr'][selfcal_plan['solints'][iteration+1]]) - get_SNR_self_update(selfcal_library,selfcal_plan,n_ants,solint,selfcal_plan['solints'][iteration+1],selfcal_plan['integration_time'], - selfcal_plan['solint_snr']) - print('Now: ',selfcal_plan['solint_snr'][selfcal_plan['solints'][iteration+1]]) + for vis in vislist: + print(vis+' was: ',selfcal_plan[vis]['solint_snr'][selfcal_plan['solints'][iteration+1]]) + get_SNR_self_update(vis, selfcal_library,selfcal_plan,n_ants,solint,selfcal_plan['solints'][iteration+1],selfcal_plan[vis]['integration_time'], + selfcal_plan[vis]['solint_snr']) + print(vis+' now: ',selfcal_plan[vis]['solint_snr'][selfcal_plan['solints'][iteration+1]]) for fid in selfcal_library['sub-fields-to-selfcal']: - print('Field '+str(fid)+' Was: ',selfcal_plan[fid]['solint_snr_per_field'][selfcal_plan['solints'][iteration+1]]) - get_SNR_self_update(selfcal_library[fid],selfcal_plan,n_ants,solint,selfcal_plan['solints'][iteration+1], - selfcal_plan['integration_time'],selfcal_plan[fid]['solint_snr_per_field']) - print('FIeld '+str(fid)+' Now: ',selfcal_plan[fid]['solint_snr_per_field'][selfcal_plan['solints'][iteration+1]]) + for vis in vislist: + print('Field '+str(fid)+' '+vis+' was: ',selfcal_plan[fid][vis]['solint_snr_per_field'][selfcal_plan['solints'][iteration+1]]) + get_SNR_self_update(vis, selfcal_library[fid],selfcal_plan,n_ants,solint,selfcal_plan['solints'][iteration+1], + selfcal_plan[vis]['integration_time'],selfcal_plan[fid][vis]['solint_snr_per_field']) + print('Field '+str(fid)+' '+vis+' now: ',selfcal_plan[fid][vis]['solint_snr_per_field'][selfcal_plan['solints'][iteration+1]]) # If not all fields succeed for inf_EB or scan_inf/inf, depending on mosaic or single field, then don't go on to amplitude selfcal, # even if *some* fields succeeded. diff --git a/selfcal_helpers.py b/selfcal_helpers.py index 4379d0e4..dab2c23b 100644 --- a/selfcal_helpers.py +++ b/selfcal_helpers.py @@ -988,42 +988,16 @@ def get_SNR_self(selfcal_library,selfcal_plan,n_ant,inf_EB_gaincal_combine,inf_E minsolint_spw=100 for target in selfcal_library: for band in selfcal_library[target].keys(): - selfcal_plan[target][band]['solint_snr'], selfcal_plan[target][band]['solint_snr_per_spw'], selfcal_plan[target][band]['solint_snr_per_bb'] = \ - get_SNR_self_individual(selfcal_library[target][band]['vislist'], selfcal_library[target][band], n_ant, selfcal_plan[target][band]['solints'], - selfcal_plan[target][band]['integration_time'], inf_EB_gaincal_combine, inf_EB_gaintype) - - print('Estimated SNR per solint:') - print(target,band) - for solint in selfcal_plan[target][band]['solints']: - if solint == 'inf_EB': - print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band]['solint_snr'][solint])) - ''' - for spw in solint_snr_per_spw[target][band][solint].keys(): - print('{}: spw: {}: {:0.2f}, BW: {} GHz'.format(solint,spw,solint_snr_per_spw[target][band][solint][spw],selfcal_library[target][band]['per_spw_stats'][str(spw)]['effective_bandwidth'])) - if solint_snr_per_spw[target][band][solint][spw] < minsolint_spw: - minsolint_spw=solint_snr_per_spw[target][band][solint][spw] - if minsolint_spw < 3.5 and minsolint_spw > 2.5 and inf_EB_override==False: # if below 3.5 but above 2.5 switch to gaintype T, but leave combine=scan - print('Switching Gaintype to T for: '+target) - inf_EB_gaintype_dict[target][band]='T' - elif minsolint_spw < 2.5 and inf_EB_override==False: - print('Switching Gaincal combine to spw,scan for: '+target) - inf_EB_gaincal_combine_dict[target][band]='scan,spw' # if below 2.5 switch to combine=spw to avoid losing spws - ''' - else: - print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band]['solint_snr'][solint])) - - for fid in selfcal_library[target][band]['sub-fields']: - selfcal_plan[target][band][fid] = {} - selfcal_plan[target][band][fid]['solint_snr_per_field'], selfcal_plan[target][band][fid]['solint_snr_per_field_per_spw'], selfcal_plan[target][band][fid]['solint_snr_per_field_per_bb'] = \ - get_SNR_self_individual(selfcal_library[target][band]['vislist'], selfcal_library[target][band][fid], n_ant, - selfcal_plan[target][band]['solints'], selfcal_plan[target][band]['integration_time'], inf_EB_gaincal_combine, - inf_EB_gaintype) - + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis]['solint_snr'], selfcal_plan[target][band][vis]['solint_snr_per_spw'], selfcal_plan[target][band][vis]['solint_snr_per_bb'] = \ + get_SNR_self_individual([vis], selfcal_library[target][band], n_ant, selfcal_plan[target][band]['solints'], + selfcal_plan[target][band][vis]['solint_settings'],selfcal_plan[target][band][vis]['integration_time'], inf_EB_gaincal_combine, inf_EB_gaintype) + print('Estimated SNR per solint:') - print(target,band,"field "+str(fid)) + print(target,band,vis) for solint in selfcal_plan[target][band]['solints']: - if solint == 'inf_EB': - print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][fid]['solint_snr_per_field'][solint])) + if selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] == 'inf_EB': + print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][vis]['solint_snr'][solint])) ''' for spw in solint_snr_per_spw[target][band][solint].keys(): print('{}: spw: {}: {:0.2f}, BW: {} GHz'.format(solint,spw,solint_snr_per_spw[target][band][solint][spw],selfcal_library[target][band]['per_spw_stats'][str(spw)]['effective_bandwidth'])) @@ -1037,11 +1011,41 @@ def get_SNR_self(selfcal_library,selfcal_plan,n_ant,inf_EB_gaincal_combine,inf_E inf_EB_gaincal_combine_dict[target][band]='scan,spw' # if below 2.5 switch to combine=spw to avoid losing spws ''' else: - print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][fid]['solint_snr_per_field'][solint])) + print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][vis]['solint_snr'][solint])) + + for fid in selfcal_library[target][band]['sub-fields']: + selfcal_plan[target][band][fid] = {} + for vis in selfcal_library[target][band][fid]['vislist']: + selfcal_plan[target][band][fid][vis] = {} + selfcal_plan[target][band][fid][vis]['solint_snr_per_field'], selfcal_plan[target][band][fid][vis]['solint_snr_per_field_per_spw'], \ + selfcal_plan[target][band][fid][vis]['solint_snr_per_field_per_bb'] = \ + get_SNR_self_individual([vis], selfcal_library[target][band][fid], n_ant, + selfcal_plan[target][band]['solints'], selfcal_plan[target][band][vis]['solint_settings'], \ + selfcal_plan[target][band][vis]['integration_time'], inf_EB_gaincal_combine, inf_EB_gaintype) + + print('Estimated SNR per solint:') + print(target,band,"field "+str(fid),vis) + for solint in selfcal_plan[target][band]['solints']: + if selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] == 'inf_EB': + print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][fid][vis]['solint_snr_per_field'][solint])) + ''' + for spw in solint_snr_per_spw[target][band][solint].keys(): + print('{}: spw: {}: {:0.2f}, BW: {} GHz'.format(solint,spw,solint_snr_per_spw[target][band][solint][spw],selfcal_library[target][band]['per_spw_stats'][str(spw)]['effective_bandwidth'])) + if solint_snr_per_spw[target][band][solint][spw] < minsolint_spw: + minsolint_spw=solint_snr_per_spw[target][band][solint][spw] + if minsolint_spw < 3.5 and minsolint_spw > 2.5 and inf_EB_override==False: # if below 3.5 but above 2.5 switch to gaintype T, but leave combine=scan + print('Switching Gaintype to T for: '+target) + inf_EB_gaintype_dict[target][band]='T' + elif minsolint_spw < 2.5 and inf_EB_override==False: + print('Switching Gaincal combine to spw,scan for: '+target) + inf_EB_gaincal_combine_dict[target][band]='scan,spw' # if below 2.5 switch to combine=spw to avoid losing spws + ''' + else: + print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][fid][vis]['solint_snr_per_field'][solint])) #return solint_snr, solint_snr_per_spw, solint_snr_per_field, solint_snr_per_field_per_spw -def get_SNR_self_individual(vislist,selfcal_library,n_ant,solints,integration_time,inf_EB_gaincal_combine,inf_EB_gaintype): +def get_SNR_self_individual(vislist,selfcal_library,n_ant,solints,solint_settings,integration_time,inf_EB_gaincal_combine,inf_EB_gaintype): if inf_EB_gaintype=='G': polscale=2.0 else: @@ -1057,96 +1061,100 @@ def get_SNR_self_individual(vislist,selfcal_library,n_ant,solints,integration_ti #selects spwlist from the visibilities with the greates number of spws maxspws=0 maxspwvis='' - for vis in selfcal_library['vislist']: + #for vis in selfcal_library['vislist']: + for vis in vislist: if selfcal_library[vis]['n_spws'] >= maxspws: maxspws=selfcal_library[vis]['n_spws'] maxspwvis=vis+'' solint_snr[solint]=0.0 solint_snr_per_spw[solint]={} solint_snr_per_bb[solint]={} - if solint == 'inf_EB': - SNR_self_EB=np.zeros(len(selfcal_library['vislist'])) + if solint_settings[solint]['interval'] == 'inf_EB': + SNR_self_EB=np.zeros(len(vislist)) #SNR_self_EB_spw=np.zeros([len(selfcal_library['vislist']),len(selfcal_library[maxspwvis]['spwsarray'])]) #SNR_self_EB_spw_mean=np.zeros([len(selfcal_library[maxspwvis]['spwsarray'])]) SNR_self_EB_spw={} SNR_self_EB_bb={} - for i in range(len(selfcal_library['vislist'])): - SNR_self_EB[i]=SNR/((n_ant)**0.5*(selfcal_library['Total_TOS']/selfcal_library[selfcal_library['vislist'][i]]['TOS'])**0.5) - SNR_self_EB_spw[selfcal_library['vislist'][i]]={} - SNR_self_EB_bb[selfcal_library['vislist'][i]]={} - for spw in selfcal_library[selfcal_library['vislist'][i]]['spwsarray']: - SNR_self_EB_spw[selfcal_library['vislist'][i]][str(spw)]=(polscale)**-0.5*SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[selfcal_library['vislist'][i]]['TOS'])**0.5)*(selfcal_library[selfcal_library['vislist'][i]]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[selfcal_library['vislist'][i]]['total_effective_bandwidth'])**0.5 - print(selfcal_library[vis]['baseband']) + for i in range(len(vislist)): + SNR_self_EB[i]=SNR/((n_ant)**0.5*(selfcal_library['Total_TOS']/selfcal_library[vislist[i]]['TOS'])**0.5) + SNR_self_EB_spw[vislist[i]]={} + SNR_self_EB_bb[vislist[i]]={} + for spw in selfcal_library[vislist[i]]['spwsarray']: + SNR_self_EB_spw[vislist[i]][str(spw)]=(polscale)**-0.5*SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[vislist[i]]['TOS'])**0.5)*(selfcal_library[vislist[i]]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[vislist[i]]['total_effective_bandwidth'])**0.5 + print(selfcal_library[vislist[i]]['baseband']) print('SNR_self_EB_spw: ',SNR_self_EB_spw) - for baseband in selfcal_library[vis]['baseband']: - SNR_self_EB_bb[selfcal_library['vislist'][i]][baseband]=(polscale)**-0.5*SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[selfcal_library['vislist'][i]]['TOS'])**0.5)*(selfcal_library[selfcal_library['vislist'][i]]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[selfcal_library['vislist'][i]]['total_effective_bandwidth'])**0.5 + for baseband in selfcal_library[vislist[i]]['baseband']: + SNR_self_EB_bb[vislist[i]][baseband]=(polscale)**-0.5*SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[vislist[i]]['TOS'])**0.5)*(selfcal_library[vislist[i]]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[vislist[i]]['total_effective_bandwidth'])**0.5 print('SNR_self_EB_bb: ',SNR_self_EB_bb) for spw in selfcal_library[maxspwvis]['spwsarray']: mean_SNR_spw=0.0 - for j in range(len(selfcal_library['vislist'])): - if str(spw) in SNR_self_EB_spw[selfcal_library['vislist'][j]].keys(): - mean_SNR_spw+=SNR_self_EB_spw[selfcal_library['vislist'][j]][str(spw)] - mean_SNR_spw=mean_SNR_spw/len(selfcal_library['vislist']) + for j in range(len(vislist)): + if str(spw) in SNR_self_EB_spw[vislist[j]].keys(): + mean_SNR_spw+=SNR_self_EB_spw[vislist[j]][str(spw)] + mean_SNR_spw=mean_SNR_spw/len(vislist) print('mean_SNR_spw',mean_SNR_spw,spw) solint_snr_per_spw[solint][str(spw)]=mean_SNR_spw - for baseband in selfcal_library[vis]['baseband']: + for baseband in selfcal_library[maxspwvis]['baseband']: mean_SNR_bb=0.0 - for j in range(len(selfcal_library['vislist'])): - if baseband in SNR_self_EB_bb[selfcal_library['vislist'][j]].keys(): - mean_SNR_bb+=SNR_self_EB_bb[selfcal_library['vislist'][j]][baseband] - mean_SNR_bb=mean_SNR_bb/len(selfcal_library['vislist']) + for j in range(len(vislist)): + if baseband in SNR_self_EB_bb[vislist[j]].keys(): + mean_SNR_bb+=SNR_self_EB_bb[vislist[j]][baseband] + mean_SNR_bb=mean_SNR_bb/len(vislist) print('mean_SNR_bb',mean_SNR_bb,baseband) solint_snr_per_bb[solint][baseband]=mean_SNR_bb solint_snr[solint]=np.mean(SNR_self_EB) selfcal_library['per_EB_SNR']=np.mean(SNR_self_EB) - elif solint =='scan_inf': - selfcal_library['per_scan_SNR']=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library['Median_scan_time'])**0.5) + elif solint_settings[solint]['interval'] =='scan_inf': + selfcal_library['per_scan_SNR']=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[maxspwvis]['Median_scan_time'])**0.5) solint_snr[solint]=selfcal_library['per_scan_SNR'] for spw in selfcal_library[maxspwvis]['spwsarray']: - solint_snr_per_spw[solint][str(spw)]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library['Median_scan_time'])**0.5)*(selfcal_library[maxspwvis]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 - for baseband in selfcal_library[vis]['baseband']: - solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library['Median_scan_time'])**0.5)*(selfcal_library[vis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 - elif solint =='inf' or solint == 'inf_ap': - selfcal_library['per_scan_SNR']=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/(selfcal_library['Median_scan_time']/selfcal_library['Median_fields_per_scan']))**0.5) + solint_snr_per_spw[solint][str(spw)]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[maxspwvis]['Median_scan_time'])**0.5)*(selfcal_library[maxspwvis]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 + for baseband in selfcal_library[maxspwvis]['baseband']: + solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[maxspwvis]['Median_scan_time'])**0.5)*(selfcal_library[maxspwvis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 + elif solint_settings[solint]['interval'] =='inf' or solint_settings[solint]['interval'] == 'inf_ap': + selfcal_library['per_scan_SNR']=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/(selfcal_library[maxspwvis]['Median_scan_time']/selfcal_library[maxspwvis]['Median_fields_per_scan']))**0.5) solint_snr[solint]=selfcal_library['per_scan_SNR'] for spw in selfcal_library[maxspwvis]['spwsarray']: - solint_snr_per_spw[solint][str(spw)]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/(selfcal_library['Median_scan_time']/selfcal_library['Median_fields_per_scan']))**0.5)*(selfcal_library[maxspwvis]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 - for baseband in selfcal_library[vis]['baseband']: - solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/(selfcal_library['Median_scan_time']/selfcal_library['Median_fields_per_scan']))**0.5)*(selfcal_library[vis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 - elif solint == 'int': + solint_snr_per_spw[solint][str(spw)]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/(selfcal_library[maxspwvis]['Median_scan_time']/selfcal_library[maxspwvis]['Median_fields_per_scan']))**0.5)*(selfcal_library[maxspwvis]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 + for baseband in selfcal_library[maxspwvis]['baseband']: + solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/(selfcal_library[maxspwvis]['Median_scan_time']/selfcal_library[maxspwvis]['Median_fields_per_scan']))**0.5)*(selfcal_library[maxspwvis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 + elif solint_settings[solint]['interval'] == 'int': solint_snr[solint]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/integration_time)**0.5) for spw in selfcal_library[maxspwvis]['spwsarray']: solint_snr_per_spw[solint][str(spw)]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/integration_time)**0.5)*(selfcal_library[maxspwvis]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 - for baseband in selfcal_library[vis]['baseband']: - solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/integration_time)**0.5)*(selfcal_library[vis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 + for baseband in selfcal_library[maxspwvis]['baseband']: + solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/integration_time)**0.5)*(selfcal_library[maxspwvis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 else: - solint_float=float(solint.replace('s','').replace('_ap','')) + solint_float=float(solint_settings[solint]['interval'].replace('s','').replace('_ap','')) solint_snr[solint]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/solint_float)**0.5) for spw in selfcal_library[maxspwvis]['spwsarray']: solint_snr_per_spw[solint][str(spw)]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/solint_float)**0.5)*(selfcal_library[maxspwvis]['per_spw_stats'][spw]['effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 - for baseband in selfcal_library[vis]['baseband']: - solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/solint_float)**0.5)*(selfcal_library[vis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 + for baseband in selfcal_library[maxspwvis]['baseband']: + solint_snr_per_bb[solint][baseband]=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/solint_float)**0.5)*(selfcal_library[maxspwvis]['baseband'][baseband]['total_effective_bandwidth']/selfcal_library[maxspwvis]['total_effective_bandwidth'])**0.5 return solint_snr,solint_snr_per_spw,solint_snr_per_bb -def get_SNR_self_update(selfcal_library,selfcal_plan,n_ant,solint_curr,solint_next,integration_time,solint_snr): +def get_SNR_self_update(vis,selfcal_library,selfcal_plan,n_ant,solint_curr,solint_next,integration_time,solint_snr): + """ maxspws=0 maxspwvis='' for vis in selfcal_library['vislist']: if selfcal_library[vis]['n_spws'] >= maxspws: maxspws=selfcal_library[vis]['n_spws'] maxspwvis=vis+'' - SNR = max(selfcal_library[selfcal_library['vislist'][0]][solint_curr]['SNR_post'],selfcal_library[selfcal_library['vislist'][0]][solint_curr]['intflux_post']/selfcal_library[selfcal_library['vislist'][0]][solint_curr]['e_intflux_post']) + """ + SNR = max(selfcal_library[vis][solint_curr]['SNR_post'],selfcal_library[vis][solint_curr]['intflux_post']/selfcal_library[vis][solint_curr]['e_intflux_post']) SNR_ratio=selfcal_library[vis][solint_curr]['SNR_post']/selfcal_library['SNR_orig'] #solint_snr[solint_next]=SNR_ratio*solint_snr[solint_next] + #for vis in selfcal_library['vislist']: solint_snr[solint_next]=SNR_ratio*solint_snr[solint_next] - for spw in selfcal_library[maxspwvis]['spwsarray']: - selfcal_plan['solint_snr_per_spw'][solint_next][str(spw)]=selfcal_plan['solint_snr_per_spw'][solint_next][str(spw)]*SNR_ratio + for spw in selfcal_library[vis]['spwsarray']: + selfcal_plan[vis]['solint_snr_per_spw'][solint_next][str(spw)]=selfcal_plan[vis]['solint_snr_per_spw'][solint_next][str(spw)]*SNR_ratio for baseband in selfcal_library[vis]['baseband']: - selfcal_plan['solint_snr_per_bb'][solint_next][baseband]=selfcal_plan['solint_snr_per_bb'][solint_next][baseband]*SNR_ratio + selfcal_plan[vis]['solint_snr_per_bb'][solint_next][baseband]=selfcal_plan[vis]['solint_snr_per_bb'][solint_next][baseband]*SNR_ratio @@ -2538,7 +2546,6 @@ def render_selfcal_solint_summary_table(htmlOut,sclib,target,band,selfcal_plan): line+=''+solint+'\n ' line+='\n' htmlOut.writelines(line) - vis_keys=list(sclib[target][band][vislist[len(vislist)-1]].keys()) quantities=['Pass','intflux_final','intflux_improvement','SNR_final','SNR_Improvement','SNR_NF_final','SNR_NF_Improvement','RMS_final','RMS_Improvement','RMS_NF_final','RMS_NF_Improvement','Beam_Ratio','clean_threshold','Plots'] for key in quantities: if key =='Pass': @@ -2570,51 +2577,58 @@ def render_selfcal_solint_summary_table(htmlOut,sclib,target,band,selfcal_plan): if key =='Plots': line='\n Plots: \n' for solint in solint_list: + if np.any([solint in sclib[target][band][vis] for vis in vislist]): + ivis = np.where([solint in sclib[target][band][vis] for vis in vislist])[0][0] + else: + ivis = len(vislist)-1 + + vis_keys=list(sclib[target][band][vislist[ivis]].keys()) + if solint in vis_keys: - vis_solint_keys=sclib[target][band][vislist[len(vislist)-1]][solint].keys() - if key != 'Pass' and sclib[target][band][vislist[len(vislist)-1]][solint]['Pass'] == 'None': + vis_solint_keys=sclib[target][band][vislist[ivis]][solint].keys() + if key != 'Pass' and sclib[target][band][vislist[ivis]][solint]['Pass'] == 'None': line+=' - \n' continue if key=='Pass': - if key in sclib[target][band][vislist[len(vislist)-1]][solint]: - if sclib[target][band][vislist[len(vislist)-1]][solint]['Pass'] == False: - line+=' {} {}\n'.format('Fail',sclib[target][band][vislist[len(vislist)-1]][solint]['Fail_Reason']) - elif sclib[target][band][vislist[len(vislist)-1]][solint]['Pass'] == 'None': - line+=' {} {}\n'.format('Not attempted',sclib[target][band][vislist[len(vislist)-1]][solint]['Fail_Reason']) + if key in sclib[target][band][vislist[ivis]][solint]: + if sclib[target][band][vislist[ivis]][solint]['Pass'] == False: + line+=' {} {}\n'.format('Fail',sclib[target][band][vislist[ivis]][solint]['Fail_Reason']) + elif sclib[target][band][vislist[ivis]][solint]['Pass'] == 'None': + line+=' {} {}\n'.format('Not attempted',sclib[target][band][vislist[ivis]][solint]['Fail_Reason']) else: line+=' {}\n'.format('Pass') else: line+=' {}\n'.format('None') if key=='intflux_final': - line+=' {:0.3f} +/- {:0.3f} mJy\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['intflux_post']*1000.0,sclib[target][band][vislist[len(vislist)-1]][solint]['e_intflux_post']*1000.0) + line+=' {:0.3f} +/- {:0.3f} mJy\n'.format(sclib[target][band][vislist[ivis]][solint]['intflux_post']*1000.0,sclib[target][band][vislist[ivis]][solint]['e_intflux_post']*1000.0) if key=='intflux_improvement': - if sclib[target][band][vislist[len(vislist)-1]][solint]['intflux_pre'] == 0: + if sclib[target][band][vislist[ivis]][solint]['intflux_pre'] == 0: line+=' {:0.3f}\n'.format(1.0) else: - line+=' {:0.3f}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['intflux_post']/sclib[target][band][vislist[len(vislist)-1]][solint]['intflux_pre']) + line+=' {:0.3f}\n'.format(sclib[target][band][vislist[ivis]][solint]['intflux_post']/sclib[target][band][vislist[ivis]][solint]['intflux_pre']) if key=='SNR_final': - line+=' {:0.3f}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['SNR_post']) + line+=' {:0.3f}\n'.format(sclib[target][band][vislist[ivis]][solint]['SNR_post']) if key=='SNR_Improvement': - line+=' {:0.3f}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['SNR_post']/sclib[target][band][vislist[len(vislist)-1]][solint]['SNR_pre']) + line+=' {:0.3f}\n'.format(sclib[target][band][vislist[ivis]][solint]['SNR_post']/sclib[target][band][vislist[ivis]][solint]['SNR_pre']) if key=='SNR_NF_final': - line+=' {:0.3f}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['SNR_NF_post']) + line+=' {:0.3f}\n'.format(sclib[target][band][vislist[ivis]][solint]['SNR_NF_post']) if key=='SNR_NF_Improvement': - line+=' {:0.3f}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['SNR_NF_post']/sclib[target][band][vislist[len(vislist)-1]][solint]['SNR_NF_pre']) + line+=' {:0.3f}\n'.format(sclib[target][band][vislist[ivis]][solint]['SNR_NF_post']/sclib[target][band][vislist[ivis]][solint]['SNR_NF_pre']) if key=='RMS_final': - line+=' {:0.3e} mJy/bm\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['RMS_post']*1000.0) + line+=' {:0.3e} mJy/bm\n'.format(sclib[target][band][vislist[ivis]][solint]['RMS_post']*1000.0) if key=='RMS_Improvement': - line+=' {:0.3e}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['RMS_pre']/sclib[target][band][vislist[len(vislist)-1]][solint]['RMS_post']) + line+=' {:0.3e}\n'.format(sclib[target][band][vislist[ivis]][solint]['RMS_pre']/sclib[target][band][vislist[ivis]][solint]['RMS_post']) if key=='RMS_NF_final': - line+=' {:0.3e} mJy/bm\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['RMS_NF_post']*1000.0) + line+=' {:0.3e} mJy/bm\n'.format(sclib[target][band][vislist[ivis]][solint]['RMS_NF_post']*1000.0) if key=='RMS_NF_Improvement': - line+=' {:0.3e}\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['RMS_NF_pre']/sclib[target][band][vislist[len(vislist)-1]][solint]['RMS_NF_post']) + line+=' {:0.3e}\n'.format(sclib[target][band][vislist[ivis]][solint]['RMS_NF_pre']/sclib[target][band][vislist[ivis]][solint]['RMS_NF_post']) if key=='Beam_Ratio': - line+=' {:0.3e}\n'.format((sclib[target][band][vislist[len(vislist)-1]][solint]['Beam_major_post']*sclib[target][band][vislist[len(vislist)-1]][solint]['Beam_minor_post'])/(sclib[target][band]['Beam_major_orig']*sclib[target][band]['Beam_minor_orig'])) + line+=' {:0.3e}\n'.format((sclib[target][band][vislist[ivis]][solint]['Beam_major_post']*sclib[target][band][vislist[ivis]][solint]['Beam_minor_post'])/(sclib[target][band]['Beam_major_orig']*sclib[target][band]['Beam_minor_orig'])) if key =='clean_threshold': if key in vis_solint_keys: - line+=' {:0.3e} mJy/bm\n'.format(sclib[target][band][vislist[len(vislist)-1]][solint]['clean_threshold']*1000.0) + line+=' {:0.3e} mJy/bm\n'.format(sclib[target][band][vislist[ivis]][solint]['clean_threshold']*1000.0) else: line+=' Not Available\n' if key =='Plots': @@ -2628,7 +2642,7 @@ def render_selfcal_solint_summary_table(htmlOut,sclib,target,band,selfcal_plan): for vis in vislist: line='\n '+vis+': \n' for solint in solint_list: - if solint in vis_keys and sclib[target][band][vis][solint]['Pass'] != 'None' and 'gaintable' in sclib[target][band][vis][solint]: + if solint in sclib[target][band][vis] and sclib[target][band][vis][solint]['Pass'] != 'None' and 'gaintable' in sclib[target][band][vis][solint]: # only evaluate last gaintable not the pre-apply table gaintable=sclib[target][band][vis][solint]['gaintable'][len(sclib[target][band][vis][solint]['gaintable'])-1] line+='antenna positions with flagging plot\n' @@ -2639,7 +2653,7 @@ def render_selfcal_solint_summary_table(htmlOut,sclib,target,band,selfcal_plan): for quantity in ['Nsols_with_preflagged_data','Flagged_Sols_with_preflagged_data','Frac_Flagged_with_preflagged_data','Nsols_without_preflagged_data','Flagged_Sols_without_preflagged_data','Frac_Flagged_without_preflagged_data','SPW_Combine_Mode']: line='\n '+quantity+'\n' for solint in solint_list: - if solint in vis_keys and sclib[target][band][vis][solint]['Pass'] != 'None' and 'gaintable' in sclib[target][band][vis][solint]: + if solint in sclib[target][band][vis] and sclib[target][band][vis][solint]['Pass'] != 'None' and 'gaintable' in sclib[target][band][vis][solint]: # only evaluate last gaintable not the pre-apply table #gaintable=sclib[target][band][vis][solint]['gaintable'][len(sclib[target][band][vis][solint]['gaintable'])-1] #nflagged_sols, nsols=get_sols_flagged_solns(gaintable) @@ -2843,7 +2857,7 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): htmlOutSolint.writelines('

Phase vs. Time Plots:

\n') for vis in vislist: htmlOutSolint.writelines('

MS: '+vis+'

\n') - if 'gaintable' not in sclib[target][band][vis][selfcal_plan[target][band]['solints'][i]]: + if selfcal_plan[target][band]['solints'][i] not in sclib[target][band][vis] or 'gaintable' not in sclib[target][band][vis][selfcal_plan[target][band]['solints'][i]]: htmlOutSolint.writelines('No gaintable available

') continue ant_list=get_ant_list(vis) From 9a32ff7ce211600a6307657206e7012d70925647 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 6 Feb 2024 13:30:59 -0500 Subject: [PATCH 02/43] Name the solints p0,p1,p2,...,ap0,ap1,... --- prepare_selfcal.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/prepare_selfcal.py b/prepare_selfcal.py index e0411684..99ba6662 100644 --- a/prepare_selfcal.py +++ b/prepare_selfcal.py @@ -470,10 +470,15 @@ def default(self, obj): if solint == "inf_EB": solint_name = "inf_EB" else: - solint_name = "solint"+str(subscan_count) if 'ap' in solint: - solint_name += '_ap' - subscan_count += 1 + solint_name = 'ap' + else: + solint_name = 'p' + solint_name += str(subscan_count) + if solint == 'int': + subscan_count = 0 + else: + subscan_count += 1 if solint_name not in selfcal_plan[target][band]['solints']: selfcal_plan[target][band]['solints'].append(solint_name) From 4316601dd04ddd99f8f1a91c9cb380cb9ea06b32 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Mon, 29 Jan 2024 14:17:50 -0500 Subject: [PATCH 03/43] Catch cases where spw_baseband == '' in check_spw_widest_in_bb --- selfcal_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/selfcal_helpers.py b/selfcal_helpers.py index dab2c23b..7f88e9f1 100644 --- a/selfcal_helpers.py +++ b/selfcal_helpers.py @@ -3318,7 +3318,7 @@ def select_best_gaincal_mode(selfcal_library,selfcal_plan,vis,gaintable_prefix,s if np.min(selfcal_plan[vis]['solint_settings'][solint]['delta_nflags']['per_spw'][i]) >= max_flagged_ants_spwmap: fallback='spwmap' spwmap[i]=1.0 - spwmap_widest_window_in_bb=check_spw_widest_in_bb(selfcal_library,vis,spwlist[i]) + spwmap_widest_window_in_bb[i]=check_spw_widest_in_bb(selfcal_library,vis,spwlist[i]) if np.sum(spwmap)/len(spwmap) > 0.5: # if greater than 1/2 of spws need mapping, just assume that we should do combinespw or per_bb fallback='' if np.sum(spwmap_widest_window_in_bb) >= 1.0: # don't do spw mapping within a baseband if the spws to be mapped are the widest in the baseband @@ -3408,6 +3408,8 @@ def find_nearest(array, value): return 1.0 else: return 0.0 + else: + return 1.0 def get_nearest_wide_bw_spw(selfcal_library,vis,spw): mapped_spw=-99 From 5de7f7c4ad48e352dc934f87595ed381e4aa3f49 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 22 Feb 2024 14:08:36 -0500 Subject: [PATCH 04/43] Fixes to get per-EB solints working when EBs have different #s of solints. --- auto_selfcal.py | 2 +- gaincal_wrapper.py | 8 ++++++-- prepare_selfcal.py | 13 ++++++++++--- run_selfcal.py | 28 +++++++++++++++++----------- selfcal_helpers.py | 39 ++++++++++++++++++++++++--------------- 5 files changed, 58 insertions(+), 32 deletions(-) diff --git a/auto_selfcal.py b/auto_selfcal.py index 697ea706..1166edb9 100644 --- a/auto_selfcal.py +++ b/auto_selfcal.py @@ -524,7 +524,7 @@ def default(self, obj): for band in selfcal_library[target].keys(): if selfcal_library[target][band]['SC_success']: for vis in selfcal_library[target][band]['vislist']: - solint=selfcal_library[target][band]['final_solint'] + solint=selfcal_library[target][band][vis]['final_solint'] iteration=selfcal_library[target][band][vis][solint]['iteration'] line='applycal(vis="'+vis.replace('.selfcal','')+'",gaintable='+str(selfcal_library[target][band][vis]['gaintable_final'])+',interp='+str(selfcal_library[target][band][vis]['applycal_interpolate_final'])+', calwt=False,spwmap='+str(selfcal_library[target][band][vis]['spwmap_final'])+', applymode="'+selfcal_library[target][band][vis]['applycal_mode_final']+'",field="'+target+'",spw="'+selfcal_library[target][band][vis]['spws_orig']+'")\n' applyCalOut.writelines(line) diff --git a/gaincal_wrapper.py b/gaincal_wrapper.py index 6380ab14..017dee4b 100644 --- a/gaincal_wrapper.py +++ b/gaincal_wrapper.py @@ -18,6 +18,8 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap ## Solve gain solutions per MS, target, solint, and band ## + print(selfcal_plan['solmode']) + print(iteration) os.system('rm -rf '+sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'*.g') ## ## Set gaincal parameters depending on which iteration and whether to use combine=spw for inf_EB or not @@ -42,7 +44,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap if selfcal_plan['solmode'][iteration]=='p': previous_solint = "inf_EB" else: - previous_solint = selfcal_library['final_phase_solint'] + previous_solint = selfcal_library[vis]['final_phase_solint'] gaincal_spwmap=[] gaincal_preapply_gaintable=[] gaincal_interpolate=[] @@ -77,7 +79,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap # Revert back to applying the inf_EB solution if calculate_inf_EB_fb_anyways, i.e. we just use the inf_EB_fb solution # for gaincal. if mode == "cocal": - if selfcal_library['final_solint'] == 'inf_EB' and calculate_inf_EB_fb_anyways: + if selfcal_library[vis]['final_solint'] == 'inf_EB' and calculate_inf_EB_fb_anyways: previous_solint = "inf_EB" fallback='' @@ -350,6 +352,8 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap applycal_interpolate=[] applycal_spwmap=[] for j in range(current_solint_index): + if not selfcal_plan['solints'][j] in selfcal_plan[vis]['solint_settings']: + continue if selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['preapply_this_gaintable'] and selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['solmode']=='p': gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['accepted_gaintable']) gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_spwmap']) diff --git a/prepare_selfcal.py b/prepare_selfcal.py index 99ba6662..f8920b82 100644 --- a/prepare_selfcal.py +++ b/prepare_selfcal.py @@ -449,10 +449,11 @@ def default(self, obj): if band in selfcal_library[target]: selfcal_plan[target][band] = {} selfcal_plan[target][band]['solints'] = [] + selfcal_plan[target][band]['solmode'] = [] for vis in selfcal_library[target][band]['vislist']: selfcal_plan[target][band][vis] = {} solints,selfcal_plan[target][band][vis]['integration_time'],selfcal_plan[target][band][vis]['gaincal_combine'], \ - selfcal_plan[target][band]['solmode']=get_solints_simple([vis],scantimesdict[band], + tmp_solmodes=get_solints_simple([vis],scantimesdict[band], scannfieldsdict[band],scanstartsdict[band],scanendsdict[band],integrationtimesdict[band],\ inf_EB_gaincal_combine,do_amp_selfcal=do_amp_selfcal,mosaic=selfcal_library[target][band]['obstype'] == 'mosaic') @@ -482,6 +483,10 @@ def default(self, obj): if solint_name not in selfcal_plan[target][band]['solints']: selfcal_plan[target][band]['solints'].append(solint_name) + if 'ap' in solint_name: + selfcal_plan[target][band]['solmode'].append('ap') + else: + selfcal_plan[target][band]['solmode'].append('p') selfcal_plan[target][band][vis]['solint_settings'][solint_name]={} selfcal_plan[target][band][vis]['solint_settings'][solint_name]['interval'] = solint @@ -490,7 +495,7 @@ def default(self, obj): print(band,target,selfcal_plan[target][band]['solints']) for vis in vislist: - print(vis,[selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] for solint in selfcal_plan[target][band]['solints']]) + print(vis,[selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] for solint in selfcal_plan[target][band]['solints'] if solint in selfcal_plan[target][band][vis]['solint_settings']]) ## ## estimate per scan/EB S/N using time on source and median scan times @@ -531,6 +536,8 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T #selfcal_plan[target][band][vis]['solint_settings']={} for solint in selfcal_plan[target][band]['solints']: + if solint not in selfcal_plan[target][band][vis]['solint_settings']: + continue gaincal_combine='' filename_append='' #selfcal_plan[target][band][vis]['solint_settings'][solint]={} @@ -617,7 +624,7 @@ def set_clean_thresholds(selfcal_library, selfcal_plan, dividing_factor=-99.0, r nsigma_init=np.max([selfcal_library[target][band]['SNR_NF_orig']/dividing_factor_band,5.0]) # count number of amplitude selfcal solints, repeat final clean depth of phase-only for amplitude selfcal - n_ap_solints=sum(1 for solint in selfcal_plan[target][band]['solints'] if 'ap' in selfcal_plan[target][band][selfcal_library[target][band]['vislist'][0]]['solint_settings'][solint]['interval']) + n_ap_solints=sum(1 for solint in selfcal_plan[target][band]['solints'] if 'ap' in solint) if rel_thresh_scaling == 'loge': selfcal_library[target][band]['nsigma'] = np.append(np.exp(np.linspace(np.log(nsigma_init),np.log(3.0),\ diff --git a/run_selfcal.py b/run_selfcal.py index b9ee32f0..e4425e83 100644 --- a/run_selfcal.py +++ b/run_selfcal.py @@ -58,7 +58,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, print("Solving for solint="+selfcal_plan['solints'][iteration]) for vis in vislist: - print(" "+vis+": "+selfcal_plan['solints'][iteration]) + print(" "+vis+": "+selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][iteration]]['interval']) # Set some cocal parameters. @@ -86,7 +86,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, # make sure the last phase-only selfcal table gets pre-applied # solints that use combinespw don't get pre-apply label by default for vis in vislist: - selfcal_plan[vis]['solint_settings'][selfcal_library['final_phase_solint']]['preapply_this_gaintable']=True + selfcal_plan[vis]['solint_settings'][selfcal_library[vis]['final_phase_solint']]['preapply_this_gaintable']=True if mode == "selfcal": remove_vis = [] @@ -427,8 +427,9 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, selfcal_library[vis]['gaincal_combine_final']=selfcal_library[vis][solint]['gaincal_combine'] selfcal_library[vis][solint]['Pass']=True selfcal_library[vis][solint]['Fail_Reason']='None' - if selfcal_plan['solmode'][iteration]=='p': - selfcal_library['final_phase_solint']=solint + if selfcal_plan['solmode'][iteration]=='p': + selfcal_library[vis]['final_phase_solint']=solint + selfcal_library[vis]['final_solint']=solint selfcal_library['final_solint']=solint selfcal_library['final_solint_mode']=selfcal_plan['solmode'][iteration] @@ -438,7 +439,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, if field_by_field_success[ind]: selfcal_library[fid]['SC_success']=True selfcal_library[fid]['Stop_Reason']='None' - for vis in selfcal_library[fid]['vislist']: + for vis in selfcal_library[fid]['vislist-to-gaincal']: selfcal_library[fid][vis]['gaintable_final']=selfcal_library[fid][vis][solint]['gaintable'] selfcal_library[fid][vis]['spwmap_final']=selfcal_library[fid][vis][solint]['spwmap'].copy() selfcal_library[fid][vis]['applycal_mode_final']=selfcal_library[fid][vis][solint]['applycal_mode'] @@ -446,8 +447,9 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, selfcal_library[fid][vis]['gaincal_combine_final']=selfcal_library[fid][vis][solint]['gaincal_combine'] selfcal_library[fid][vis][solint]['Pass']=True selfcal_library[fid][vis][solint]['Fail_Reason']='None' - if selfcal_plan['solmode'][iteration]=='p': - selfcal_library[fid]['final_phase_solint']=solint + if selfcal_plan['solmode'][iteration]=='p': + selfcal_library[fid][vis]['final_phase_solint']=solint + selfcal_library[fid][vis]['final_solint']=solint selfcal_library[fid]['final_solint']=solint selfcal_library[fid]['final_solint_mode']=selfcal_plan['solmode'][iteration] selfcal_library[fid]['iteration']=iteration @@ -511,7 +513,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, ## if S/N worsens, and/or beam area increases reject current solutions and reapply previous (or revert to origional data) ## - if not selfcal_library[vislist[0]][solint]['Pass']: + if not selfcal_library[selfcal_library['vislist-to-gaincal'][0]][solint]['Pass']: reason='' if (post_SNR <= SNR): reason=reason+' S/N decrease' @@ -535,7 +537,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, mosaic_reason = {} new_fields_to_selfcal = [] for fid in selfcal_library['sub-fields-to-selfcal']: - if not selfcal_library[fid][selfcal_library[fid]['vislist'][0]][solint]['Pass']: + if not selfcal_library[fid][selfcal_library[fid]['vislist-to-gaincal'][0]][solint]['Pass']: mosaic_reason[fid]='' if (post_mosaic_SNR[fid] <= mosaic_SNR[fid]): mosaic_reason[fid]=mosaic_reason[fid]+' SNR decrease' @@ -586,12 +588,14 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, # If any of the sub-fields passed, and the whole mosaic passed, then we can move on to the next solint, otherwise we have to back out. if selfcal_library[vislist[0]][solint]['Pass'] == True and \ - np.any([selfcal_library[fid][selfcal_library[fid]['vislist'][0]][solint]['Pass'] == True for fid in \ + np.any([selfcal_library[fid][selfcal_library[fid]['vislist-to-gaincal'][0]][solint]['Pass'] == True for fid in \ selfcal_library['sub-fields-to-selfcal']]): if mode == "selfcal" and (iteration < len(selfcal_plan['solints'])-1) and (selfcal_library[vis][solint]['SNR_post'] > \ selfcal_library['SNR_orig']): #(iteration == 0) and print('Updating solint = '+selfcal_plan['solints'][iteration+1]+' SNR') for vis in vislist: + if selfcal_plan['solints'][iteration+1] not in selfcal_plan[vis]['solint_snr']: + continue print(vis+' was: ',selfcal_plan[vis]['solint_snr'][selfcal_plan['solints'][iteration+1]]) get_SNR_self_update(vis, selfcal_library,selfcal_plan,n_ants,solint,selfcal_plan['solints'][iteration+1],selfcal_plan[vis]['integration_time'], selfcal_plan[vis]['solint_snr']) @@ -599,6 +603,8 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, for fid in selfcal_library['sub-fields-to-selfcal']: for vis in vislist: + if selfcal_plan['solints'][iteration+1] not in selfcal_plan[fid][vis]['solint_snr_per_field']: + continue print('Field '+str(fid)+' '+vis+' was: ',selfcal_plan[fid][vis]['solint_snr_per_field'][selfcal_plan['solints'][iteration+1]]) get_SNR_self_update(vis, selfcal_library[fid],selfcal_plan,n_ants,solint,selfcal_plan['solints'][iteration+1], selfcal_plan[vis]['integration_time'],selfcal_plan[fid][vis]['solint_snr_per_field']) @@ -606,7 +612,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, telescope, n_ants, # If not all fields succeed for inf_EB or scan_inf/inf, depending on mosaic or single field, then don't go on to amplitude selfcal, # even if *some* fields succeeded. - if iteration <= 1 and ((not np.all([selfcal_library[fid][selfcal_library[fid]['vislist'][0]][solint]['Pass'] == True for fid in \ + if iteration <= 1 and ((not np.all([selfcal_library[fid][selfcal_library[fid]['vislist-to-gaincal'][0]][solint]['Pass'] == True for fid in \ selfcal_library['sub-fields-to-selfcal']])) or len(selfcal_library['sub-fields-to-selfcal']) < \ len(selfcal_library['sub-fields'])) and do_amp_selfcal: print("***** NOTE: Amplitude self-calibration turned off because not all fields succeeded at non-inf_EB phase self-calibration") diff --git a/selfcal_helpers.py b/selfcal_helpers.py index 7f88e9f1..14078bd0 100644 --- a/selfcal_helpers.py +++ b/selfcal_helpers.py @@ -989,13 +989,15 @@ def get_SNR_self(selfcal_library,selfcal_plan,n_ant,inf_EB_gaincal_combine,inf_E for target in selfcal_library: for band in selfcal_library[target].keys(): for vis in selfcal_library[target][band]['vislist']: + solints_per_vis = [solint for solint in selfcal_plan[target][band]['solints'] if solint in selfcal_plan[target][band][vis]['solint_settings']] + selfcal_plan[target][band][vis]['solint_snr'], selfcal_plan[target][band][vis]['solint_snr_per_spw'], selfcal_plan[target][band][vis]['solint_snr_per_bb'] = \ - get_SNR_self_individual([vis], selfcal_library[target][band], n_ant, selfcal_plan[target][band]['solints'], + get_SNR_self_individual([vis], selfcal_library[target][band], n_ant, solints_per_vis, selfcal_plan[target][band][vis]['solint_settings'],selfcal_plan[target][band][vis]['integration_time'], inf_EB_gaincal_combine, inf_EB_gaintype) print('Estimated SNR per solint:') print(target,band,vis) - for solint in selfcal_plan[target][band]['solints']: + for solint in solints_per_vis: if selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] == 'inf_EB': print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][vis]['solint_snr'][solint])) ''' @@ -1017,15 +1019,16 @@ def get_SNR_self(selfcal_library,selfcal_plan,n_ant,inf_EB_gaincal_combine,inf_E selfcal_plan[target][band][fid] = {} for vis in selfcal_library[target][band][fid]['vislist']: selfcal_plan[target][band][fid][vis] = {} + solints_per_vis = [solint for solint in selfcal_plan[target][band]['solints'] if solint in selfcal_plan[target][band][vis]['solint_settings']] selfcal_plan[target][band][fid][vis]['solint_snr_per_field'], selfcal_plan[target][band][fid][vis]['solint_snr_per_field_per_spw'], \ selfcal_plan[target][band][fid][vis]['solint_snr_per_field_per_bb'] = \ get_SNR_self_individual([vis], selfcal_library[target][band][fid], n_ant, - selfcal_plan[target][band]['solints'], selfcal_plan[target][band][vis]['solint_settings'], \ + solints_per_vis, selfcal_plan[target][band][vis]['solint_settings'], \ selfcal_plan[target][band][vis]['integration_time'], inf_EB_gaincal_combine, inf_EB_gaintype) print('Estimated SNR per solint:') print(target,band,"field "+str(fid),vis) - for solint in selfcal_plan[target][band]['solints']: + for solint in solints_per_vis: if selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] == 'inf_EB': print('{}: {:0.2f}'.format(solint,selfcal_plan[target][band][fid][vis]['solint_snr_per_field'][solint])) ''' @@ -2778,7 +2781,8 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): final_solint_to_plot=selfcal_plan[target][band]['solints'][final_solint_index+index_addition-1] - keylist=sclib[target][band][vislist[0]].keys() + #keylist=sclib[target][band][vislist[0]].keys() + keylist = [solint for solint in selfcal_plan[target][band]['solints'] if np.any([solint in sclib[target][band][vis] for vis in vislist])] if index_addition == 2 and final_solint_to_plot not in keylist: index_addition=index_addition-1 @@ -2787,7 +2791,10 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): #for i in range(final_solint_index+index_addition): for i in range(len(selfcal_plan[target][band]['solints'])): - if selfcal_plan[target][band]['solints'][i] not in keylist or sclib[target][band][vislist[len(vislist)-1]][selfcal_plan[target][band]['solints'][i]]['Pass'] == 'None': + representative_vislist = [vis for vis in vislist if selfcal_plan[target][band]['solints'][i] in sclib[target][band][vis]] + if len(representative_vislist) == 0: + continue + if selfcal_plan[target][band]['solints'][i] not in keylist or sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Pass'] == 'None': continue htmlOutSolint=open(directory+'/'+target+'_'+band+'_'+selfcal_plan[target][band]['solints'][i]+'.html','w') htmlOutSolint.writelines('\n') @@ -2799,7 +2806,7 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): htmlOutSolint.writelines('

'+target+' Plots

\n') htmlOutSolint.writelines('

'+band+'

\n') htmlOutSolint.writelines('

Targets:

\n') - keylist=sclib[target][band][vislist[0]].keys() + #keylist=sclib[target][band][vislist[0]].keys() solints_string='' for j in range(final_solint_index+index_addition): if selfcal_plan[target][band]['solints'][j] not in keylist: @@ -2813,8 +2820,8 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): #must select last key for pre Jan 14th runs since they only wrote pass to the last MS dictionary entry - if "Pass" in sclib[target][band][vislist[len(vislist)-1]][selfcal_plan[target][band]['solints'][i]]: - passed=sclib[target][band][vislist[len(vislist)-1]][selfcal_plan[target][band]['solints'][i]]['Pass'] + if "Pass" in sclib[target][band][representative_vislist[-1]][selfcal_plan[target][band]['solints'][i]]: + passed=sclib[target][band][representative_vislist[-1]][selfcal_plan[target][band]['solints'][i]]['Pass'] else: passed = 'None' @@ -2845,17 +2852,17 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): htmlOutSolint.writelines('pre-SC-solint image\n') htmlOutSolint.writelines('pre-SC-solint image
\n') - htmlOutSolint.writelines('Post SC SNR: {:0.3f}'.format(sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['SNR_post'])+'
Pre SC SNR: {:0.3f}'.format(sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['SNR_pre'])+'

\n') - htmlOutSolint.writelines('Post SC RMS: {:0.7f}'.format(sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['RMS_post'])+' Jy/beam
Pre SC RMS: {:0.7f}'.format(sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['RMS_pre'])+' Jy/beam
\n') - htmlOutSolint.writelines('Post Beam: {:0.3f}"x{:0.3f}" {:0.3f} deg'.format(sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_major_post'],sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_minor_post'],sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_PA_post'])+'
\n') - htmlOutSolint.writelines('Pre Beam: {:0.3f}"x{:0.3f}" {:0.3f} deg'.format(sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_major_pre'],sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_minor_pre'],sclib[target][band][vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_PA_pre'])+'

\n') + htmlOutSolint.writelines('Post SC SNR: {:0.3f}'.format(sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['SNR_post'])+'
Pre SC SNR: {:0.3f}'.format(sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['SNR_pre'])+'

\n') + htmlOutSolint.writelines('Post SC RMS: {:0.7f}'.format(sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['RMS_post'])+' Jy/beam
Pre SC RMS: {:0.7f}'.format(sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['RMS_pre'])+' Jy/beam
\n') + htmlOutSolint.writelines('Post Beam: {:0.3f}"x{:0.3f}" {:0.3f} deg'.format(sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_major_post'],sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_minor_post'],sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_PA_post'])+'
\n') + htmlOutSolint.writelines('Pre Beam: {:0.3f}"x{:0.3f}" {:0.3f} deg'.format(sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_major_pre'],sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_minor_pre'],sclib[target][band][representative_vislist[0]][selfcal_plan[target][band]['solints'][i]]['Beam_PA_pre'])+'

\n') if 'inf_EB' in selfcal_plan[target][band]['solints'][i]: htmlOutSolint.writelines('

Phase vs. Frequency Plots:

\n') else: htmlOutSolint.writelines('

Phase vs. Time Plots:

\n') - for vis in vislist: + for vis in representative_vislist: htmlOutSolint.writelines('

MS: '+vis+'

\n') if selfcal_plan[target][band]['solints'][i] not in sclib[target][band][vis] or 'gaintable' not in sclib[target][band][vis][selfcal_plan[target][band]['solints'][i]]: htmlOutSolint.writelines('No gaintable available

') @@ -4106,10 +4113,12 @@ def get_min_SNR_spw(snr_per_spw): return minsnr def remove_modes(selfcal_plan,vis,start_index): + preferred_mode=selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][start_index]]['final_mode'] for j in range(start_index+1,len(selfcal_plan['solints'])): + if selfcal_plan['solints'][j] not in selfcal_plan[vis]['solint_settings']: + continue if 'ap' in selfcal_plan['solints'][j] and 'ap' not in selfcal_plan['solints'][start_index]: # exempt over ap solints since they go back to a longer solint continue - preferred_mode=selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['final_mode'] if preferred_mode == 'per_bb' or preferred_mode == 'combinespw': if 'per_spw' in selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['modes_to_attempt']: selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['modes_to_attempt'].remove('per_spw') From 7f477232ddcb4965de3f454e3f2d9d9e9e169f54 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 1 Apr 2025 06:58:11 -0500 Subject: [PATCH 05/43] Show solution interval per-EB in the weblog. --- auto_selfcal/weblog_creation.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/weblog_creation.py b/auto_selfcal/weblog_creation.py index 0fa4439c..9d112468 100644 --- a/auto_selfcal/weblog_creation.py +++ b/auto_selfcal/weblog_creation.py @@ -221,6 +221,17 @@ def render_selfcal_solint_summary_table(htmlOut,sclib,target,band,selfcal_plan): line+=''+solint+'\n ' line+='\n' htmlOut.writelines(line) + htmlOut.writelines('\n Solution interval by EB: \n') + for vis in vislist: + line=f'\n {vis}: \n' + for solint in solint_list: + if solint in selfcal_plan[target][band][vis]['solint_settings']: + line += f' {selfcal_plan[target][band][vis]["solint_settings"][solint]["interval"]} \n' + else: + line += ' - \n' + line += '\n' + htmlOut.writelines(line) + htmlOut.writelines('\n Selfcal stats: \n') quantities=['Pass','intflux_final','intflux_improvement','SNR_final','SNR_Improvement','SNR_NF_final','SNR_NF_Improvement','RMS_final','RMS_Improvement','RMS_NF_final','RMS_NF_Improvement','Beam_Ratio','clean_threshold','Plots'] for key in quantities: if key =='Pass': @@ -492,13 +503,20 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): htmlOutSolint.writelines('

Targets:

\n') #keylist=sclib[target][band][vislist[0]].keys() solints_string='' + print(keylist) for j in range(final_solint_index+index_addition): if selfcal_plan[target][band]['solints'][j] not in keylist: continue solints_string+=''+selfcal_plan[target][band]['solints'][j]+'
\n' htmlOutSolint.writelines('
Solints: '+solints_string) - htmlOutSolint.writelines('

Solint: '+selfcal_plan[target][band]['solints'][i]+'

\n') + htmlOutSolint.writelines('

Solint: '+selfcal_plan[target][band]['solints'][i]+'

\n') + for ivis, vis in enumerate(representative_vislist): + if ivis == len(representative_vislist)-1: + margin_string = 'style="margin-top: 0; padding-top:0;"' + else: + margin_string = 'style="margin : 0; padding-top:0;"' + htmlOutSolint.writelines(f'

{vis}: {selfcal_plan[target][band][vis]["solint_settings"][selfcal_plan[target][band]["solints"][i]]["interval"]}

\n') keylist_top=sclib[target][band].keys() htmlOutSolint.writelines('Back to Main Target/Band
\n') From b22339ab7789459a8ed00b98d0132a86712da292 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 8 Jul 2025 18:34:55 +0000 Subject: [PATCH 06/43] Add option to use uniform solution intervals across all EBs --- auto_selfcal/__main__.py | 1 + auto_selfcal/auto_selfcal.py | 3 ++- auto_selfcal/prepare_selfcal.py | 21 +++++++++++++++++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/auto_selfcal/__main__.py b/auto_selfcal/__main__.py index 7f6b8237..87c3f7d7 100644 --- a/auto_selfcal/__main__.py +++ b/auto_selfcal/__main__.py @@ -26,6 +26,7 @@ parser.add_argument('--do_amp_selfcal', default=True) parser.add_argument('--usermask', default={}, type=ast.literal_eval) # require that it is a CRTF region (CASA region format) parser.add_argument('--usermodel', default={}, type=ast.literal_eval) +parser.add_argument('--uniform_solints', default=False) parser.add_argument('--inf_EB_gaincal_combine', default='scan', type=str) # should we get rid of this option? parser.add_argument('--inf_EB_gaintype', default='G', type=str) parser.add_argument('--inf_EB_override', action='store_true') diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index 52415825..2f62b2b6 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -46,6 +46,7 @@ def auto_selfcal( inf_EB_gaintype='G', inf_EB_override=False, optimize_spw_combine=True, # if False, will not attempt per spw or per baseband solutions for any solint except inf_EB + uniform_solints=False, gaincal_minsnr=2.0, gaincal_unflag_minsnr=5.0, minsnr_to_proceed=2.95, @@ -134,7 +135,7 @@ def auto_selfcal( selfcal_library, selfcal_plan, gaincalibrator_dict = prepare_selfcal(vislist, spectral_average=spectral_average, sort_targets_and_EBs=sort_targets_and_EBs, scale_fov=scale_fov, inf_EB_gaincal_combine=inf_EB_gaincal_combine, inf_EB_gaintype=inf_EB_gaintype, apply_cal_mode_default=apply_cal_mode_default, do_amp_selfcal=do_amp_selfcal, - usermask=usermask, usermodel=usermodel,debug=debug) + uniform_solints=uniform_solints, usermask=usermask, usermodel=usermodel,debug=debug) with open('selfcal_library.pickle', 'wb') as handle: diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index 337e73c9..bd0211af 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -9,6 +9,7 @@ def prepare_selfcal(vislist, inf_EB_gaintype='G', apply_cal_mode_default='calflag', do_amp_selfcal=True, + uniform_solints=False, usermask={}, usermodel={}, debug=False): @@ -492,13 +493,25 @@ def default(self, obj): selfcal_plan[target][band] = {} selfcal_plan[target][band]['solints'] = [] selfcal_plan[target][band]['solmode'] = [] - for vis in selfcal_library[target][band]['vislist']: - selfcal_plan[target][band][vis] = {} - solints,selfcal_plan[target][band][vis]['integration_time'],selfcal_plan[target][band][vis]['gaincal_combine'], \ - tmp_solmodes=get_solints_simple([vis],scantimesdict[band], + + if uniform_solints: + solints,tmp_integration_time,tmp_gaincal_combine, \ + tmp_solmodes=get_solints_simple(vislist,scantimesdict[band], scannfieldsdict[band],scanstartsdict[band],scanendsdict[band],integrationtimesdict[band],\ inf_EB_gaincal_combine,do_amp_selfcal=do_amp_selfcal,mosaic=selfcal_library[target][band]['obstype'] == 'mosaic') + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis] = {} + + if not uniform_solints: + solints,selfcal_plan[target][band][vis]['integration_time'],selfcal_plan[target][band][vis]['gaincal_combine'], \ + tmp_solmodes=get_solints_simple([vis],scantimesdict[band], + scannfieldsdict[band],scanstartsdict[band],scanendsdict[band],integrationtimesdict[band],\ + inf_EB_gaincal_combine,do_amp_selfcal=do_amp_selfcal,mosaic=selfcal_library[target][band]['obstype'] == 'mosaic') + else: + selfcal_plan[target][band][vis]['integration_time'] = tmp_integration_time + selfcal_plan[target][band][vis]['gaincal_combine'] = tmp_gaincal_combine + selfcal_plan[target][band][vis]['solint_settings']={} subscan_count = 0 From 4d9b729388019e39b2c1627324573677ba06f0e9 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 8 Jul 2025 18:56:44 +0000 Subject: [PATCH 07/43] Use uniform_solints=True for tests for now to match what was previously happening so that tests pass (if changes haven't messed that up). --- auto_selfcal/tests/test_auto_selfcal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 04ab1f8e..3b292ec7 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -50,7 +50,7 @@ def test_benchmark(tmp_path, dataset): ex.update_parameters(partition="batch2", nodes=1, ntasks_per_node=8, cpus_per_task=1, use_srun=False, time=10080, \ mem="128gb", job_name=dataset) - job = ex.submit(auto_selfcal, sort_targets_and_EBs=True, weblog=True, parallel=True) + job = ex.submit(auto_selfcal, sort_targets_and_EBs=True, uniform_solints=True, weblog=True, parallel=True) job.wait() assert job.state in ['DONE','COMPLETED'] @@ -83,7 +83,7 @@ def test_on_github(tmp_path, request, zip_file, link): os.system(f'tar xf {zip_file}') os.system(f'rm -rf {zip_file}') - auto_selfcal(sort_targets_and_EBs=True, weblog=True) + auto_selfcal(sort_targets_and_EBs=True, uniform_solints=True, weblog=True) os.system('rm -rf *.ms*') # Delete MS files as space is limited on GitHub. From 1562607988b995a3d72fa1052dc1f87ee2eeb9da Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 25 Sep 2025 16:42:53 +0000 Subject: [PATCH 08/43] Add code to enable testing across the major changes brought by per_EB_solints. --- .github/workflows/run_E2E_test.yml | 2 +- auto_selfcal/tests/test_auto_selfcal.py | 44 ++++++++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run_E2E_test.yml b/.github/workflows/run_E2E_test.yml index c02f4852..fd1f0d40 100644 --- a/.github/workflows/run_E2E_test.yml +++ b/.github/workflows/run_E2E_test.yml @@ -5,7 +5,7 @@ name: E2E Tests on: push: - branches: [ "make_pip_installable" ] + branches: [ "make_pip_installable", "make_pip_installable+per_EB_solints+testing" ] pull_request: branches: [ "make_pip_installable" ] diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 3b292ec7..ce14a783 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -92,7 +92,20 @@ def test_on_github(tmp_path, request, zip_file, link): with open('selfcal_library.pickle', 'rb') as handle: selfcal_library2 = pickle.load(handle) - difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001) + with open('selfcal_plan.pickle', 'rb') as handle: + selfcal_plan = pickle.load(handle) + + solint_map = {} + for target in selfcal_library2: + for band in selfcal_library2[target]: + for vis in selfcal_library2[target][band]['vislist']: + for solint in selfcal_plan[target][band][vis]['solint_settings']: + if solint not in solint_map: + solint_map[solint] = [] + + solint_map[solint].append(selfcal_plan[target][band][vis]['solint_settings'][solint]['interval']) + + difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001, key_map=solint_map) assert difference_count == 0 @@ -114,7 +127,7 @@ def compare_values(list1, list2, tol=1e-3): else: return abs(list1 - list2) < abs(list1*tol) -def compare_two_dictionaries(dictionary1, dictionary2, path=[], exclude=[], tolerance=1e-3): +def compare_two_dictionaries(dictionary1, dictionary2, path=[], exclude=[], tolerance=1e-3, key_map={}): difference_count = 0 all_keys = np.unique(list(dictionary1.keys()) + list(dictionary2.keys())) @@ -123,7 +136,7 @@ def compare_two_dictionaries(dictionary1, dictionary2, path=[], exclude=[], tole if key in exclude: continue - if key not in intersect_keys: + if key not in intersect_keys and key not in key_map and not np.any([key in key_map[k] for k in key_map]): if key not in dictionary1: print('/'.join([str(p) for p in path])+"/"+key+" not in dictionary1") else: @@ -132,20 +145,33 @@ def compare_two_dictionaries(dictionary1, dictionary2, path=[], exclude=[], tole difference_count += 1 continue + elif key not in intersect_keys and key not in key_map and np.any([key in key_map[k] for k in key_map]): + continue - if key not in dictionary1 and int(key) in dictionary1: - key = int(key) + try: + if key not in dictionary1 and int(key) in dictionary1: + key = int(key) + except: + continue + + if key in dictionary2 and not key in dictionary1 and key in key_map: + for alt_key in key_map: + if alt_key in dictionary1: + break + else: + alt_key = key + - if type(dictionary1[key]) == dict: - difference_count += compare_two_dictionaries(dictionary1[key], dictionary2[key], path.copy()+[key], exclude=exclude, tolerance=tolerance) + if type(dictionary1[alt_key]) == dict: + difference_count += compare_two_dictionaries(dictionary1[alt_key], dictionary2[key], path.copy()+[key], exclude=exclude, tolerance=tolerance, key_map=key_map) else: - value1 = np.array(dictionary1[key])[np.argsort(dictionary1['vislist'])] if key in ['spws_per_vis','vislist'] else dictionary1[key] + value1 = np.array(dictionary1[alt_key])[np.argsort(dictionary1['vislist'])] if alt_key in ['spws_per_vis','vislist'] else dictionary1[alt_key] value2 = np.array(dictionary2[key])[np.argsort(dictionary2['vislist'])] if key in ['spws_per_vis','vislist'] else dictionary2[key] #value1 = np.array(dictionary1[key])[np.argsort(dictionary1['vislist'])] if key in ['spws_per_vis'] else dictionary1[key] #value2 = np.array(dictionary2[key])[np.argsort(dictionary2['vislist'])] if key in ['spws_per_vis'] else dictionary2[key] if key == 'gaincal_combine': - value1 = dictionary1[key].split(',') + value1 = dictionary1[alt_key].split(',') value1.sort() value2 = dictionary2[key].split(',') value2.sort() From 15edeea62d2e86cd45ca2d64bd08d7f3e430492e Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 30 Sep 2025 19:20:31 +0000 Subject: [PATCH 09/43] Make sure the SNR_self_EB array is only as long as the vislist passed to the function. --- auto_selfcal/selfcal_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index cf5e2ad7..f049b925 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -1283,7 +1283,7 @@ def get_SNR_self_individual(vislist,selfcal_library,n_ant,solints,solint_setting solint_snr_per_spw[solint]={} solint_snr_per_bb[solint]={} if solint_settings[solint]['interval'] == 'inf_EB': - SNR_self_EB=np.zeros(len(selfcal_library['vislist'])) + SNR_self_EB=np.zeros(len(vislist)) SNR_self_EB_spw={} SNR_self_EB_bb={} for i in range(len(vislist)): From 3b50cb0807ce9b02e8f4c99124adaf80379f39da Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 30 Sep 2025 19:23:20 +0000 Subject: [PATCH 10/43] Add some keywords to exclude from comparison due to expected changes in this branch. --- auto_selfcal/tests/test_auto_selfcal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index ce14a783..b8d91a21 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -105,7 +105,8 @@ def test_on_github(tmp_path, request, zip_file, link): solint_map[solint].append(selfcal_plan[target][band][vis]['solint_settings'][solint]['interval']) - difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001, key_map=solint_map) + difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001, key_map=solint_map, + exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal"]) assert difference_count == 0 From d23ce03c1917a7af01c1db939af9d5292fd7a358 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Wed, 8 Apr 2026 18:35:00 +0000 Subject: [PATCH 11/43] Ensure inf_EB and *_ap solints are mapped correctly now that we record the interval instead of the actual stage name in the solint settings. Also enable proper comparison of None types. --- auto_selfcal/tests/test_auto_selfcal.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index f880cf0c..179a4d10 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -116,7 +116,14 @@ def test_on_github(tmp_path, request, zip_file, link): if solint not in solint_map: solint_map[solint] = [] - solint_map[solint].append(selfcal_plan[target][band][vis]['solint_settings'][solint]['interval']) + mapped_solint = selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] + if solint == 'inf_EB': + mapped_solint += '_EB' + elif 'ap' in solint: + mapped_solint += '_ap' + + solint_map[solint].append(mapped_solint) + print(solint_map) difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001, key_map=solint_map, exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal"]) @@ -135,6 +142,8 @@ def compare_values(list1, list2, tol=1e-3): return np.all([compare_values(list1[i], list2[i], tol=tol) for i in range(len(list1))]) elif type(list1) == str or type(list1) == np.str_ or type(list1) == bool: return list1 == list2 + elif type(list1) == type(None): + return list1 == list2 else: if list1 == 0: return abs(list2) < tol From c28ed0902d94da9bed51c3fc55dce98c27817c71 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Fri, 24 Apr 2026 20:13:31 +0000 Subject: [PATCH 12/43] Properly handle the key_map so that keys whose names changed are looked up properly. Also add some informative messaging, and add telescope to the list of excluded keys because it is new. --- auto_selfcal/tests/test_auto_selfcal.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index abba2447..746ced34 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -130,7 +130,7 @@ def test_on_github(tmp_path, request, zip_file, link): print(solint_map) difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001, key_map=solint_map, - exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal"]) + exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal", "telescope"]) assert difference_count == 0 @@ -180,18 +180,30 @@ def compare_two_dictionaries(dictionary1, dictionary2, path=[], exclude=[], tole continue elif key not in intersect_keys and key not in key_map and np.any([key in key_map[k] for k in key_map]): + print(f'key {key} has changed in dictionary2 and will be matched elsewhere') continue try: - if key not in dictionary1 and int(key) in dictionary1: + if key not in dictionary1 and key not in key_map and int(key) in dictionary1: key = int(key) except: continue if key in dictionary2 and not key in dictionary1 and key in key_map: - for alt_key in key_map: + print(f'Checking whether key {key} has its name changed') + found = False + for alt_key in key_map[key]: + print(f'Checking for {alt_key} in dictionary1') if alt_key in dictionary1: + found = True break + + if found: + print(f"Using alternative key {alt_key} to match with key {key}") + else: + print(f"No match found in dictionary1, this is a difference") + difference_count += 1 + continue else: alt_key = key From 8c1438df813b8d9555827c7992a3cd30e0d8a7c2 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 09:23:46 -0400 Subject: [PATCH 13/43] fixes to get auto_selfcal to run cleanly with allow_cocal=True --- auto_selfcal/auto_selfcal.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index a475e76c..e5993cbe 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -751,7 +751,7 @@ def default(self, obj): inf_fields = {} fallback_fields = {} calibrators = {} - for band in bands: + for band in vis_for_targets[target]['Bands']: # Initialize the lists for this band. inf_EB_fields[band] = [] inf_fields[band] = [] @@ -773,7 +773,8 @@ def default(self, obj): selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] - applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] + selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] + #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] calibrators[band] = [inf_EB_fields[band], inf_fields[band], inf_fields[band], inf_fields[band]] selfcal_library[target][band]["nsigma"] = np.concatenate((selfcal_library[target][band]["nsigma"],[selfcal_library[target][band]["nsigma"][0], \ selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1]])) @@ -787,7 +788,7 @@ def default(self, obj): ## for target in selfcal_library: - for band in solint_snr[target].keys(): + for band in selfcal_library[target].keys(): # If the target had a successful inf_EB solution, no need to reset. if target in inf_EB_fields[band]: continue @@ -812,7 +813,7 @@ def default(self, obj): for band in selfcal_library[target].keys(): if target not in fallback_fields[band]: continue - if 'gaintable_final' in selfcal_library[target][band][vislist[0]]: + if 'gaintable_final' in selfcal_library[target][band]['vislist'][0]: print('****************Reapplying previous solint solutions*************') for vis in selfcal_library[target][band]['vislist']: print('****************Applying '+str(selfcal_library[target][band][vis]['gaintable_final'])+' to '+target+' '+band+'*************') From 9e9712f117177f4137d20e401ef0d5c999210608 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 10:02:12 -0400 Subject: [PATCH 14/43] changes to get cocal to look at original targets ms for possible calibrators --- auto_selfcal/run_selfcal.py | 2 +- auto_selfcal/selfcal_helpers.py | 20 ++++++++++++++++---- bin/auto_selfcal.py | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index f0b9165e..bb86f9ec 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -48,7 +48,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if mode == "cocal": # Check whether there are suitable calibrators, otherwise skip this target/band. - include_targets, include_scans = triage_calibrators(vislist[0], target, calibrators[band][0]) + include_targets, include_scans = triage_calibrators(vislist[0], target, band, calibrators[band][0]) if include_targets == "": print("No suitable calibrators found, skipping "+target) selfcal_library['Stop_Reason'] += '; No suitable co-calibrators' diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 2a7f6da5..2e302416 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -516,7 +516,7 @@ def tclean_wrapper(selfcal_library, imagename, band, scales=[0], smallscalebias uvrange=selfcal_library['uvrange'], reffreq = reffreq, threshold=threshold, - parallel=parallel, + parallel=False, phasecenter=phasecenter,spw=spws_per_vis,wprojplanes=wprojplanes) elif savemodel=='modelcolumn' and selfcal_library['usermodel'] !='': @@ -4643,11 +4643,23 @@ def unflag_failed_antennas(vis, caltable, gaincal_return, telescope, flagged_fra -def triage_calibrators(vis, target, potential_calibrators, max_distance=10.0, max_time=600.): +def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10.0, max_time=600.): gaincalibrator_dict = {} + # account for possible different naming conventions in original visibilities + sani_target=sanitize_string(target) + orig_vis='' + if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): + orig_vis=os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) + elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): + orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) - if os.path.exists(vis.replace("_target.selfcal.ms",".ms")): - msmd.open(vis.replace("_target.selfcal.ms",".ms")) + # original visibilities, with all sources have a different filename now + + orig_targets_vis=vis.replace(".selfcal.ms",".ms").replace(sani_target+'_'+band+'_') + vis=orig_targets_vis + + if orig_vis !='': + msmd.open(orig_vis) for field in msmd.fieldsforintent("*CALIBRATE_PHASE*"): scans_for_field = msmd.scansforfield(field) diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index b6fbde6c..79976d9a 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, parallel=parallel) +auto_selfcal(vislist, allow_cocal=True,parallel=parallel) From e5cf7d0d37c1e3eb4e8d91d568ae6e22eca07469 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 10:06:51 -0400 Subject: [PATCH 15/43] fix missing argument for a replace() call --- auto_selfcal/selfcal_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 2e302416..38827cba 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -4655,7 +4655,7 @@ def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10 # original visibilities, with all sources have a different filename now - orig_targets_vis=vis.replace(".selfcal.ms",".ms").replace(sani_target+'_'+band+'_') + orig_targets_vis=vis.replace(".selfcal.ms",".ms").replace(sani_target+'_'+band+'_','') vis=orig_targets_vis if orig_vis !='': From 05e0202b4f34ce586abf5372ff2dcc5b226d731b Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 12:10:40 -0400 Subject: [PATCH 16/43] fix another missing second argument on a replace() --- auto_selfcal/selfcal_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 38827cba..16c8bcbf 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -4648,7 +4648,7 @@ def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10 # account for possible different naming conventions in original visibilities sani_target=sanitize_string(target) orig_vis='' - if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): + if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')): orig_vis=os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) From f4db2a1b54cab5dc613e723528ef5b03cca78fbe Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 14:16:14 -0400 Subject: [PATCH 17/43] missing additions to selfcal_plan solint_interval --- auto_selfcal/auto_selfcal.py | 1 + auto_selfcal/gaincal_wrapper.py | 66 +++++++++++++++++++++++---------- auto_selfcal/run_selfcal.py | 3 +- auto_selfcal/selfcal_helpers.py | 4 +- bin/auto_selfcal.py | 2 +- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index e5993cbe..be8f4464 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -772,6 +772,7 @@ def default(self, obj): if len(fallback_fields[band]) > 0: selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] + selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index dab9e4cf..c6fff6d6 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -101,7 +101,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so # Check which targets are acceptable to use as calibrators. targets = calibrators[band][iteration - len(selfcal_plan['solints'])] - include_targets, include_scans = triage_calibrators(vis, target, targets) + include_targets, include_scans = triage_calibrators(vis, target, band, targets) else: #include_targets = str(selfcal_library['sub-fields-fid_map'][vis][0]) include_targets = selfcal_library['bands_for_targets'][vis]['field_str'] @@ -309,6 +309,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) for incl_scans, incl_targets in zip(include_scans, include_targets): for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + print(vis,solint,mode) print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][mode] @@ -331,27 +332,52 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so spwselect=selfcal_library[vis]['spws'] gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' print('prior to gaincal',gaintable_name, mode) - if mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': + vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + for incl_target in incl_targets: + vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + if mode != 'per_bb': + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + else: + for baseband in selfcal_library[vis]['baseband'].keys(): + spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) else: - for baseband in selfcal_library[vis]['baseband'].keys(): - spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + if mode != 'per_bb': + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) - + else: + for baseband in selfcal_library[vis]['baseband'].keys(): + spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][mode] = gaintable_name # restricted gaincal table comparisons to only inf_EB prior to changes diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index bb86f9ec..41cd4872 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -69,7 +69,8 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ do_fallback_calonly=False print('Starting selfcal procedure on: '+target+' '+band) while iteration < len(selfcal_plan['solints']): - + print(selfcal_plan['solints']) + print(selfcal_plan['solint_interval'][iteration]) print("Solving for solint="+selfcal_plan['solints'][iteration]+' with interval '+selfcal_plan['solint_interval'][iteration]) # Set some cocal parameters. diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 16c8bcbf..ddf3d06f 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -4650,8 +4650,8 @@ def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10 orig_vis='' if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')): orig_vis=os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) - elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): - orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) + elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')): + orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')) # original visibilities, with all sources have a different filename now diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index 79976d9a..a73cb692 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True,parallel=parallel) +auto_selfcal(vislist, allow_cocal=True,do_amp_selfcal=False,parallel=parallel) From 4e005e0d0123073c89822951ff384d450030d12a Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 16:44:06 -0400 Subject: [PATCH 18/43] more progress toward getting cocal to run --- auto_selfcal/auto_selfcal.py | 5 +++++ auto_selfcal/run_selfcal.py | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index be8f4464..ef23b1d0 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -773,6 +773,11 @@ def default(self, obj): selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]=selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 41cd4872..034784ef 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -442,9 +442,12 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ marginal_inf_EB_will_attempt_next_solint = False else: marginal_inf_EB_will_attempt_next_solint = True - - RMS_change_acceptable = (post_RMS/RMS < 1.05 and post_RMS_NF/RMS_NF < 1.05) or \ - ((post_RMS/RMS > 1.05 or post_RMS_NF/RMS_NF > 1.05) and selfcal_plan['solint_snr'][solint] > 5) + RMS_change_acceptable= False + if mode != 'cocal': + RMS_change_acceptable = (post_RMS/RMS < 1.05 and post_RMS_NF/RMS_NF < 1.05) or \ + ((post_RMS/RMS > 1.05 or post_RMS_NF/RMS_NF > 1.05) and selfcal_plan['solint_snr'][solint] > 5) + else: + RMS_change_acceptable = (post_RMS/RMS < 1.05 and post_RMS_NF/RMS_NF < 1.05) if (((post_SNR >= SNR) and (post_SNR_NF >= SNR_NF) and (delta_beamarea < delta_beam_thresh)) or ((('inf_EB' in solint) or 'delay' in solint) and marginal_inf_EB_will_attempt_next_solint and ((post_SNR-SNR)/SNR > -0.02) and ((post_SNR_NF - SNR_NF)/SNR_NF > -0.02) and (delta_beamarea < delta_beam_thresh))) and np.any(field_by_field_success) and RMS_change_acceptable: From c64e5799e6e2706b4e7fcc972c95e2de4d7814c0 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Mon, 4 May 2026 14:55:22 -0400 Subject: [PATCH 19/43] fix variable name used twice and another fail statement invalid for co-cal --- auto_selfcal/gaincal_wrapper.py | 27 ++++++++++++++------------- auto_selfcal/run_selfcal.py | 28 ++++++++++++++++++++-------- bin/auto_selfcal.py | 2 +- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index c6fff6d6..13bc59bd 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -106,6 +106,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so #include_targets = str(selfcal_library['sub-fields-fid_map'][vis][0]) include_targets = selfcal_library['bands_for_targets'][vis]['field_str'] include_scans = "" + print('Include targets: ', include_targets) if solint == "scan_inf": if len(gaincalibrator_dict[vis]) > 0: @@ -308,12 +309,12 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print(solint,'Include targets: ', include_targets) print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) for incl_scans, incl_targets in zip(include_scans, include_targets): - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - - print(vis,solint,mode) + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + print(incl_targets) + print(vis,solint,gc_mode) print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) - gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][mode] - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][mode] + gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] if 'spw' in gaincal_combine: if selfcal_library['spws_set'][vis].ndim == 1: @@ -331,12 +332,12 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: spwselect=selfcal_library[vis]['spws'] gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - print('prior to gaincal',gaintable_name, mode) + print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') for incl_target in incl_targets: vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') - if mode != 'per_bb': + if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ @@ -344,7 +345,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] @@ -355,9 +356,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: - if mode != 'per_bb': + if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ @@ -365,7 +366,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] @@ -376,9 +377,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][mode] = gaintable_name + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name # restricted gaincal table comparisons to only inf_EB prior to changes # commenting because we want to do comparisons for other solints as well diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 034784ef..5737d8f9 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -608,14 +608,26 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if reason !='': reason=reason+'; ' reason=reason+'Beam change beyond '+str(delta_beam_thresh) - if (post_RMS/RMS > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): - if reason != '': - reason=reason+'; ' - reason=reason+'RMS increase beyond 5%' - if (post_RMS_NF/RMS_NF > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): - if reason != '': - reason=reason+'; ' - reason=reason+'NF RMS increase beyond 5%' + + if mode != 'cocal': + if (post_RMS/RMS > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): + if reason != '': + reason=reason+'; ' + reason=reason+'RMS increase beyond 5%' + if (post_RMS_NF/RMS_NF > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): + if reason != '': + reason=reason+'; ' + reason=reason+'NF RMS increase beyond 5%' + else: + if (post_RMS/RMS > 1.05): + if reason != '': + reason=reason+'; ' + reason=reason+'RMS increase beyond 5%' + if (post_RMS_NF/RMS_NF > 1.05): + if reason != '': + reason=reason+'; ' + reason=reason+'NF RMS increase beyond 5%' + if not np.any(field_by_field_success): if reason != '': reason=reason+'; ' diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index a73cb692..4285c610 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True,do_amp_selfcal=False,parallel=parallel) +auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,parallel=parallel) From 96dab1389bb50cd8ad32e5785ad36ffc6216c960 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Mon, 4 May 2026 15:31:47 -0400 Subject: [PATCH 20/43] fix more double use of mode variable --- auto_selfcal/gaincal_wrapper.py | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 13bc59bd..2c37f5d0 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -29,8 +29,8 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so os.system('rm -rf '+sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'*.g') ## Reset the gaincal return dictionaries, in case this is a repeat of the current solution interval. - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode] = [] + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode] = [] ## ## Set gaincal parameters depending on which iteration and whether to use combine=spw for inf_EB or not ## Defaults should assume combine='scan' and gaintpe='G' will fallback to combine='scan,spw' if too much flagging @@ -334,11 +334,12 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': - vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + print('in cocal if statement') for incl_target in incl_targets: vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + print('Vis to gaincal: ',vis_to_gaincal) if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -349,7 +350,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -359,7 +360,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -370,7 +371,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -478,9 +479,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: solnorm=False - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][mode] - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][mode] + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] if 'spw' in gaincal_combine: if selfcal_library['spws_set'][vis].ndim == 1: @@ -498,28 +499,28 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: spwselect=selfcal_library[vis]['spws'] gaintable_name='temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - if mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + if gc_mode != 'per_bb': + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=str(selfcal_library['sub-fields-fid_map'][vis][fid]),gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(gaintable_name)) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=str(selfcal_library['sub-fields-fid_map'][vis][fid]),gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(gaintable_name)) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][mode] = gaintable_name + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name gaintable_prefix='temp_' if len(selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) > 1: @@ -575,18 +576,18 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][preferred_mode] - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: ##### send chosen subtable to this routine for final copying to the gain table we want. - tb.open('temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+mode+'.g') + tb.open('temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+gc_mode+'.g') subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") tb.close() - subt.copy(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+mode+'.g', deep=True) + subt.copy(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+gc_mode+'.g', deep=True) subt.close() # commented so that we keep all the gain tables around # once well-tested, we might remove the tables for the non-chosen modes to avoid generating too many useless files. - os.system('rm -rf '+'temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+mode+'.g') + os.system('rm -rf '+'temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+gc_mode+'.g') if rerank_refants: selfcal_library[vis]["refant"] = rank_refants(vis, selfcal_library['telescope'], caltable=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+selfcal_library[vis][solint]['final_mode']+'.g') From 8aea04864f1a8774b63c8cc0615851460d18df51 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Mon, 4 May 2026 16:43:17 -0400 Subject: [PATCH 21/43] pass preapply_targets_own_inf_EB to gaincal_wrapper --- auto_selfcal/gaincal_wrapper.py | 2 +- auto_selfcal/run_selfcal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 2c37f5d0..dbf1b5ce 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -5,7 +5,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so gaincal_minsnr, gaincal_unflag_minsnr=5.0, minsnr_to_proceed=3.0, rerank_refants=False, unflag_only_lbants=False, unflag_only_lbants_onlyap=False, calonly_max_flagged=0.0, second_iter_solmode="", unflag_fb_to_prev_solint=False, \ refantmode="flex", mode="selfcal", calibrators="", gaincalibrator_dict={}, allow_gain_interpolation=False,spectral_solution_fraction=0.3, - guess_scan_combine=False, do_fallback_calonly=False): + guess_scan_combine=False, do_fallback_calonly=False, preapply_targets_own_inf_EB=False): """ This function runs gaincal for a given target, band, and solint, and updates the selfcal_library and selfcal_plan dictionaries with the results. It also handles the pre-application of inf_EB solutions if necessary. diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 5737d8f9..9e892367 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -251,7 +251,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ second_iter_solmode=second_iter_solmode, unflag_fb_to_prev_solint=unflag_fb_to_prev_solint, \ refantmode=refantmode, mode=mode, calibrators=calibrators, gaincalibrator_dict=gaincalibrator_dict, allow_gain_interpolation=allow_gain_interpolation,spectral_solution_fraction=spectral_solution_fraction, - do_fallback_calonly=do_fallback_calonly, guess_scan_combine=guess_scan_combine) + do_fallback_calonly=do_fallback_calonly, guess_scan_combine=guess_scan_combine, preapply_targets_own_inf_EB=preapply_targets_own_inf_EB) # With gaincal done and bad fields removed from gain tables if necessary, check whether any fields should no longer be # selfcal'd because they have too much interpolation. From cd3feb9d0a2bf057183ea4dff35aa13520a88d65 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Tue, 5 May 2026 10:49:46 -0400 Subject: [PATCH 22/43] fix the target and vis file specification for cocal gaincal calls --- auto_selfcal/gaincal_wrapper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index dbf1b5ce..c3a97f55 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -311,6 +311,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so for incl_scans, incl_targets in zip(include_scans, include_targets): for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: print(incl_targets) + print(incl_scans) print(vis,solint,gc_mode) print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] @@ -335,15 +336,15 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': print('in cocal if statement') - for incl_target in incl_targets: - vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + for incl_target in include_targets: + vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') print('Vis to gaincal: ',vis_to_gaincal) if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) @@ -354,7 +355,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) From d55e0d63a25f44e38d9721c12d3501b568e8a3c5 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Tue, 5 May 2026 16:21:08 -0400 Subject: [PATCH 23/43] add a concat step to add co-calibrators to the MS of the source needing cocal --- auto_selfcal/gaincal_wrapper.py | 5 +++-- auto_selfcal/run_selfcal.py | 19 ++++++++++++++++--- bin/auto_selfcal.py | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index c3a97f55..70b224c1 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -336,8 +336,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': print('in cocal if statement') - for incl_target in include_targets: - vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') + for incl_target in include_targets[0].split(','): + #vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') + vis_to_gaincal = vis print('Vis to gaincal: ',vis_to_gaincal) if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 9e892367..fb347e8e 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -49,11 +49,22 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if mode == "cocal": # Check whether there are suitable calibrators, otherwise skip this target/band. include_targets, include_scans = triage_calibrators(vislist[0], target, band, calibrators[band][0]) + print('Co-calibrators: ',include_targets) + print('Co-calibrator scans: ',include_scans) if include_targets == "": print("No suitable calibrators found, skipping "+target) selfcal_library['Stop_Reason'] += '; No suitable co-calibrators' return - + else: + for vis in vislist: + os.system('mv '+vis+' '+vis.replace('.ms','_orig.ms')) + os.system('mv '+vis+'.flagversions '+vis.replace('.ms','_orig.ms.flagversions')) + concatvislist=[] + for cal_target in include_targets.split(','): + concatvislist.append(vis.replace(target,cal_target)) + concatvislist.append(vis.replace('.ms','_orig.ms')) + print('Concat vislist: ',concatvislist) + concat(vis=concatvislist,concatvis=vis) if selfcal_library['usermodel'] != '': print('Setting model column to user model') usermodel_wrapper(selfcal_library,sani_target+'_'+band, @@ -194,8 +205,10 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ flagmanager(vis=vis, mode = 'restore', versionname = versionname, comment = 'Flag states at start of reduction') if mode == "cocal": - flagmanager(vis=vis, mode = 'restore', versionname = 'selfcal_starting_flags', comment = 'Flag states at start of the reduction') - + if os.path.exists(vis+".flagversions/flags.selfcal_starting_flags"): + flagmanager(vis=vis, mode = 'restore', versionname = 'selfcal_starting_flags', comment = 'Flag states at start of the reduction') + else: + flagmanager(vis=vis,mode='save',versionname='selfcal_starting_flags') if not do_fallback_combinespw: # We need to redo saving the model now that we have potentially unflagged some data. if not do_fallback_calonly: diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index 4285c610..5b6fe7f6 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,parallel=parallel) +auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,targets='FZ_Tau,DL_Tau,DO_Tau',parallel=parallel) From f39b3a1a1a4b80986ccd0dc0fbea8f32e02d6ede Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 7 May 2026 10:50:31 -0400 Subject: [PATCH 24/43] Make sure to increment iteration for all possible endings of cocal solints. --- auto_selfcal/run_selfcal.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index fb347e8e..e9945106 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -93,14 +93,17 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ preapply_targets_own_inf_EB = False # If there was not a successful inf_EB solint, then this duplicates inf_fb1 so skip if "inf_EB" not in selfcal_library[vislist[0]]: + iteration += 1 continue elif not selfcal_library[vislist[0]]["inf_EB"]['Pass']: + iteration += 1 continue elif selfcal_plan['solints'][iteration] == "inf_fb3": calculate_inf_EB_fb_anyways = False preapply_targets_own_inf_EB = True # If there was no inf solint (e.g. because each source was observed only a single time, skip this as there are no gain tables to stick together. if "inf" not in selfcal_plan['solints']: + iteration += 1 continue if 'ap' in selfcal_plan['solints'][iteration]: @@ -779,6 +782,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ continue elif mode == "cocal" and "inf_fb" in solint: print('****************Cocal failed, attempting next inf_fb option*************') + iteration += 1 continue else: print('****************Aborting further self-calibration attempts for '+target+' '+band+'**************') From 88e82b453deed72f81a7024d4fc1bb0e677de76f Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 7 May 2026 15:20:12 -0400 Subject: [PATCH 25/43] Loop over all possible gaincal modes and look up the correct calibration files for the inf_fb3 solint. --- auto_selfcal/gaincal_wrapper.py | 78 ++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 70b224c1..238689f3 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -231,49 +231,57 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so ## If we want to pre-apply inf_EB solution from each calibrator to itself, all we do is combine all of thier ## individual inf tables, as these were pre-calculated in that way. ## - destination_table = sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'.g' - for t in include_targets.split(","): - if os.path.exists(sanitize_string(t)+'_'+vis+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '.pre-pass.g'): - table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '.pre-pass.g' - else: - table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+'.g' - #t_final_solint = selfcal_library[t][band]["final_phase_solint"] - #t_iteration = selfcal_library[t][band][vislist[0]][t_final_solint]["iteration"] - #table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+t_final_solint+'_'+str(t_iteration)+'_'+solmode[band][iteration]+'.g' + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] + + destination_table = sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' + for t in include_targets.split(","): + sani_t = sanitize_string(t) + if os.path.exists(sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ + '_'+filename_append+'.pre-pass.g'): + table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ + '_'+filename_append+'.pre-pass.g' + else: + table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ + '_'+filename_append+'.g' + #t_final_solint = selfcal_library[t][band]["final_phase_solint"] + #t_iteration = selfcal_library[t][band][vislist[0]][t_final_solint]["iteration"] + #table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+t_final_solint+'_'+str(t_iteration)+'_'+solmode[band][iteration]+'.g' - rerefant(vis, table_name, caltable="tmp0.g", refantmode="strict", refant=selfcal_library[vis]['refant']) + if not os.path.exists(table_name): + continue - tb.open("tmp0.g") - if not os.path.exists("tmp1.g"): - tb.copy("tmp1.g", deep=True) - else: - tb.copyrows("tmp1.g") - tb.close() + rerefant(vis, table_name, caltable="tmp0.g", refantmode="strict", refant=selfcal_library[vis]['refant']) - os.system("rm -rf tmp0.g") + tb.open("tmp0.g") + if not os.path.exists("tmp1.g"): + tb.copy("tmp1.g", deep=True) + else: + tb.copyrows("tmp1.g") + tb.close() - tb.open("tmp1.g") - subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") - copyt = subt.copy(destination_table, deep=True) - tb.close() - subt.close() - copyt.close() + os.system("rm -rf tmp0.g") + + tb.open("tmp1.g") + subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") + copyt = subt.copy(destination_table, deep=True) + tb.close() + subt.close() + copyt.close() - os.system("rm -rf tmp1.g") + os.system("rm -rf tmp1.g") - # Remove all of the scans that failed the triage above. - tb.open(destination_table, nomodify=False) - scans = tb.getcol("SCAN_NUMBER") + # Remove all of the scans that failed the triage above. + tb.open(destination_table, nomodify=False) + scans = tb.getcol("SCAN_NUMBER") - bad_scans = np.repeat(True, scans.size) - for scan in include_scans[0].split(","): - bad_scans[scans == int(scan)] = False + bad_scans = np.repeat(True, scans.size) + for scan in include_scans[0].split(","): + bad_scans[scans == int(scan)] = False - tb.removerows(rownrs=np.where(bad_scans)[0]) - tb.flush() - tb.close() + tb.removerows(rownrs=np.where(bad_scans)[0]) + tb.flush() + tb.close() else: # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. gaincal_solmode="" From 38f5711a8385376c23f620115c2171729646195e Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 7 May 2026 19:42:11 +0000 Subject: [PATCH 26/43] Track the sub-fields to selfcal/gaincal per-EB. Also store the original solint name in the solint settings to keep track of special solints like scan_inf and 300s_ap --- auto_selfcal/gaincal_wrapper.py | 18 +++--- auto_selfcal/mosaic_helpers.py | 69 +++++++++++---------- auto_selfcal/prepare_selfcal.py | 9 ++- auto_selfcal/run_selfcal.py | 81 ++++++++++++++++++------- auto_selfcal/selfcal_helpers.py | 4 +- auto_selfcal/tests/test_auto_selfcal.py | 7 ++- 6 files changed, 117 insertions(+), 71 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 3a9da755..7a3b3bfd 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -109,7 +109,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap include_targets = selfcal_library['bands_for_targets'][vis]['field_str'] include_scans = "" - if solint == "scan_inf": + if selfcal_plan[vis]['solint_settings'][solint]['sub-name'] == "scan_inf": if len(gaincalibrator_dict[vis]) > 0: print("Determining scan_inf from calibrator scans in full MS") scans = [] @@ -285,10 +285,10 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap for incl_scan in include_scans: scan_targets = [] for fid in [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ - np.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))] if incl_scan == '' else \ + np.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))] if incl_scan == '' else \ np.intersect1d(msmd.fieldsforscans(np.array(incl_scan.split(",")).astype(int)), \ [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ - numpy.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))]): + numpy.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))]): # Note: because of the msmd above getting actual fids from the MS, we just need to append fid below. scan_targets.append(fid) @@ -412,7 +412,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap remove_modes(selfcal_plan,vis,current_solint_index) - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): selfcal_library[fid][vis][solint]['final_mode']=preferred_mode+'' selfcal_library[fid][vis][solint]['spwmap']=applycal_spwmap.copy() selfcal_library[fid][vis][solint]['gaincal_combine']=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][preferred_mode]+'' @@ -430,7 +430,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap else: # this else is for ap selfcal os.system('rm -rf temp*.g') - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): gaincal_spwmap=[] gaincal_preapply_gaintable=[] gaincal_interpolate=[] @@ -537,7 +537,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap remove_modes(selfcal_plan,vis,current_solint_index) - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): selfcal_library[fid][vis][solint]['final_mode']=preferred_mode+'' selfcal_library[fid][vis][solint]['spwmap']=applycal_spwmap.copy() selfcal_library[fid][vis][solint]['gaincal_combine']=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][preferred_mode]+'' @@ -612,7 +612,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap refant=selfcal_library[vis]["refant"], refantmode=refantmode if 'inf_EB' not in solint else 'flex') selfcal_library[vis][solint]['fallback']=fallback+'' - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): selfcal_library[fid][vis][solint]['fallback']=fallback+'' # If iteration two, try restricting to just the antennas with enough unflagged data. @@ -656,7 +656,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, ap if (solint != "inf_EB" and not allow_gain_interpolation) or (allow_gain_interpolation and "inf" not in solint): # If a given field has > 25% of its solutions flagged then just flag the whole field because it will have too much # interpolation. - if solint == "scan_inf": + if selfcal_plan[vis]['solint_settings'][solint]['sub-name'] == "scan_inf": max_n_solutions = max([(scans == scan).sum() for scan in np.unique(scans)]) for scan in np.unique(scans): scan_n_solutions = (flags[0,0,scans == scan] == False).sum() @@ -714,7 +714,7 @@ def generate_settings_for_combinespw_fallback(selfcal_library, selfcal_plan, tar selfcal_library[vis][solint]['applycal_interpolate']=applycal_interpolate selfcal_library[vis][solint]['solmode']=selfcal_plan['solmode'][iteration]+'' selfcal_library[vis][solint]['gaincal_combine']=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][preferred_mode]+'' - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): selfcal_library[fid][vis][solint]['final_mode']=preferred_mode+'_fallback' selfcal_library[fid][vis][solint]['spwmap']=applycal_spwmap selfcal_library[fid][vis][solint]['gaincal_combine']=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][preferred_mode]+'' diff --git a/auto_selfcal/mosaic_helpers.py b/auto_selfcal/mosaic_helpers.py index 93543f2a..3301bfde 100644 --- a/auto_selfcal/mosaic_helpers.py +++ b/auto_selfcal/mosaic_helpers.py @@ -1,14 +1,14 @@ import numpy as np from .selfcal_helpers import * -def evaluate_subfields_to_gaincal(selfcal_library, target, band, solint, iteration, solmode, solints, selfcal_plan, +def evaluate_subfields_to_gaincal(vis, selfcal_library, target, band, solint, iteration, solmode, solints, selfcal_plan, minsnr_to_proceed, allow_gain_interpolation=False,): sani_target=sanitize_string(target) # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. new_fields_to_selfcal = [] - for fid in selfcal_library['sub-fields-to-selfcal']: + for fid in selfcal_library[vis]['sub-fields-to-selfcal']: os.system('rm -rf test*.mask') tmp_SNR_NF,tmp_RMS_NF=estimate_near_field_SNR(sani_target+'_field_'+str(fid)+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0', \ las=selfcal_library['LAS'], mosaic_sub_field=True, save_near_field_mask=False) @@ -41,8 +41,8 @@ def evaluate_subfields_to_gaincal(selfcal_library, target, band, solint, iterati if not checkmask(sani_target+'_field_'+str(fid)+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0'): print("Removing field "+str(fid)+" from gaincal because there is no signal within the primary beam.") skip_reason = "No signal" - elif selfcal_plan[fid]['solint_snr_per_field'][solints[iteration]] < minsnr_to_proceed and \ - solint not in ['inf_EB','scan_inf']: + elif selfcal_plan[fid][vis]['solint_snr_per_field'][solints[iteration]] < minsnr_to_proceed and \ + selfcal_plan[vis]['solint_settings'][solint]['sub-name'] not in ['inf_EB','scan_inf']: print("Removing field "+str(fid)+" from gaincal because the estimated solint snr is too low.") skip_reason = "Estimated SNR" elif updated_intflux > selfcal_library['flux_threshold'] * original_intflux: @@ -53,21 +53,20 @@ def evaluate_subfields_to_gaincal(selfcal_library, target, band, solint, iterati new_fields_to_selfcal.append(fid) if fid not in new_fields_to_selfcal and solint != "inf_EB" and not allow_gain_interpolation: - for vis in selfcal_library[fid]['vislist']: - #selfcal_library[fid][vis][solint]['interpolated_gains'] = True - #selfcal_library[fid]['Stop_Reason'] = "Gaincal solutions would be interpolated" - selfcal_library[fid]['Stop_Reason'] = skip_reason - selfcal_library[fid][vis][solint]['Pass'] = "None" - selfcal_library[fid][vis][solint]['Fail_Reason'] = skip_reason + #selfcal_library[fid][vis][solint]['interpolated_gains'] = True + #selfcal_library[fid]['Stop_Reason'] = "Gaincal solutions would be interpolated" + #selfcal_library[fid]['Stop_Reason'] = skip_reason + selfcal_library[fid][vis][solint]['Pass'] = "None" + selfcal_library[fid][vis][solint]['Fail_Reason'] = skip_reason return new_fields_to_selfcal -def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iteration, solmode, allow_gain_interpolation=False): +def evaluate_subfields_after_gaincal(vis, selfcal_library, selfcal_plan, target, band, solint, iteration, solmode, allow_gain_interpolation=False): - new_fields_to_selfcal = selfcal_library['sub-fields-to-selfcal'].copy() + new_fields_to_selfcal = selfcal_library[vis]['sub-fields-to-selfcal'].copy() sani_target=sanitize_string(target) @@ -75,23 +74,23 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter (allow_gain_interpolation and "inf" not in solint)): # With gaincal done and bad fields removed from gain tables if necessary, check whether any fields should no longer be selfcal'd # because they have too much interpolation. - for vis in selfcal_library['vislist']: + #for vis in selfcal_library['vislist']: + if True: ## If an EB had no fields to gaincal on, remove all fields in that EB from being selfcal'd as there is no calibration available ## in this EB. - if np.intersect1d(selfcal_library['sub-fields-to-gaincal'],\ + if np.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],\ list(selfcal_library['sub-fields-fid_map'][vis].keys())).size == 0: for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())): new_fields_to_selfcal.remove(fid) selfcal_library[fid]['Stop_Reason'] = 'No viable calibrator fields in at least 1 EB' - for v in selfcal_library[fid]['vislist']: - selfcal_library[fid][v][solint]['Pass'] = 'None' - if 'Fail_Reason' in selfcal_library[fid][v][solint]: - selfcal_library[fid][v][solint]['Fail_Reason'] += '; ' - else: - selfcal_library[fid][v][solint]['Fail_Reason'] = '' - selfcal_library[fid][v][solint]['Fail_Reason'] += 'No viable fields' - continue + selfcal_library[fid][vis][solint]['Pass'] = 'None' + if 'Fail_Reason' in selfcal_library[fid][vis][solint]: + selfcal_library[fid][vis][solint]['Fail_Reason'] += '; ' + else: + selfcal_library[fid][vis][solint]['Fail_Reason'] = '' + selfcal_library[fid][vis][solint]['Fail_Reason'] += 'No viable fields' + return new_fields_to_selfcal ## NEXT TO DO: check % of flagged solutions - DONE, see above ## After that enable option for interpolation through inf - DONE tb.open(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+solmode[iteration]+'_'+ @@ -100,7 +99,7 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter scans = tb.getcol("SCAN_NUMBER") for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())): - if solint == "scan_inf": + if selfcal_plan[vis]['solint_settings'][solint]['sub-name'] == "scan_inf": msmd.open(vis) cals_for_scan = [] total_cals_for_scan = [] @@ -125,29 +124,29 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter new_fields_to_selfcal.remove(fid) if fid not in new_fields_to_selfcal: - # We need to update all the EBs, not just the one that failed. - for v in selfcal_library[fid]['vislist']: - selfcal_library[fid][v][solint]['Pass'] = 'None' - if allow_gain_interpolation: - selfcal_library[fid][v][solint]['Fail_Reason'] = 'Interpolation beyond inf' - else: - selfcal_library[fid][v][solint]['Fail_Reason'] = 'Bad gaincal solutions' + selfcal_library[fid][vis][solint]['Pass'] = 'None' + if allow_gain_interpolation: + selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Interpolation beyond inf' + else: + selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Bad gaincal solutions' tb.close() elif selfcal_library['obstype'] == 'mosaic' and solint == "inf_EB": ## If an EB had no fields to gaincal on, remove all fields in that EB from being selfcal'd as there is no calibration available ## in this EB. - for vis in selfcal_library['vislist']: - if np.intersect1d(selfcal_library['sub-fields-to-gaincal'],\ + #for vis in selfcal_library['vislist']: + if True: + if np.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],\ list(selfcal_library['sub-fields-fid_map'][vis].keys())).size == 0: for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())): new_fields_to_selfcal.remove(fid) selfcal_library[fid]['Stop_Reason'] = 'No viable calibrator fields for inf_EB in at least 1 EB' - for v in selfcal_library[fid]['vislist']: - selfcal_library[fid][v][solint]['Pass'] = 'None' - selfcal_library[fid][v][solint]['Fail_Reason'] = 'No viable inf_EB fields' + selfcal_library[fid][vis][solint]['Pass'] = 'None' + selfcal_library[fid][vis][solint]['Fail_Reason'] = 'No viable inf_EB fields' + + print("new_fields_to_selfcal", new_fields_to_selfcal) return new_fields_to_selfcal diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index b294397a..1cb94dad 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -152,6 +152,8 @@ def prepare_selfcal(all_targets, bands, bands_for_targets, vislist, selfcal_library[target][band]['sub-fields'] = list(range(len(all_phasecenters))) selfcal_library[target][band]['sub-fields-to-selfcal'] = list(range(len(all_phasecenters))) + for vis in vislist: + selfcal_library[target][band][vis]['sub-fields-to-selfcal'] = list(range(len(all_phasecenters))) selfcal_library[target][band]['sub-fields-phasecenters'] = dict(zip(selfcal_library[target][band]['sub-fields'], all_phasecenters)) # Now we can start to create a sub-field selfcal_library entry for each sub-field. @@ -161,6 +163,8 @@ def prepare_selfcal(all_targets, bands, bands_for_targets, vislist, for vis in vislist: if not fid in selfcal_library[target][band]['sub-fields-fid_map'][vis]: + # If a sub-field is not in an EB, it shouldn't be considered for selfcal for that EB + selfcal_library[target][band][vis]['sub-fields-to-selfcal'].pop(fid) continue selfcal_library[target][band][fid][vis] = {} @@ -568,6 +572,7 @@ def default(self, obj): selfcal_plan[target][band][vis]['solint_settings'][solint_name]={} selfcal_plan[target][band][vis]['solint_settings'][solint_name]['interval'] = solint_interval + selfcal_plan[target][band][vis]['solint_settings'][solint_name]['sub-name'] = solint selfcal_plan[target][band]['applycal_mode']=[apply_cal_mode_default]*len(selfcal_plan[target][band]['solints']) @@ -690,11 +695,11 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T gaincal_combine='spw' filename_append='per_bb' selfcal_plan[target][band][vis]['solint_settings'][solint]['spwmap_for_mode']['per_bb']=selfcal_library[target][band][vis]['baseband_spwmap'] - if solint in ['inf_EB','inf_EB_delay','scan_inf','300s_ap']: + if selfcal_plan[target][band][vis]['solint_settings'][solint]['sub-name'] in ['inf_EB','inf_EB_delay','scan_inf','300s_ap']: if gaincal_combine!='': gaincal_combine+=',' gaincal_combine+='scan' - if solint in ['inf_EB','inf_EB_delay','scan_inf'] and selfcal_library[target][band]['obstype'] == 'mosaic': + if selfcal_plan[target][band][vis]['solint_settings'][solint]['sub-name'] in ['inf_EB','inf_EB_delay','scan_inf'] and selfcal_library[target][band]['obstype'] == 'mosaic': gaincal_combine+=',field' selfcal_plan[target][band][vis]['solint_settings'][solint]['gaincal_combine'][mode]=gaincal_combine selfcal_plan[target][band][vis]['solint_settings'][solint]['filename_append'][mode]=filename_append diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 67625997..0a3a00dc 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -66,6 +66,9 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ do_fallback_combinespw=False do_fallback_calonly=False print('Starting selfcal procedure on: '+target+' '+band) + print('selfcal_library["sub-fields-to-selfcal"] = ', selfcal_library['sub-fields-to-selfcal']) + for vis in selfcal_library['vislist']: + print('selfcal_library[vis]["sub-fields-to-selfcal"] = ', selfcal_library[vis]['sub-fields-to-selfcal']) while iteration < len(selfcal_plan['solints']): vislist=[vis for vis in selfcal_library['vislist'] if selfcal_plan['solints'][iteration] in selfcal_plan[vis]['solint_settings']] @@ -172,7 +175,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ selfcal_library[vis][solint]={} selfcal_library[vis][solint]['clean_threshold'] = selfcal_library['nsigma'][iteration]*selfcal_library['RMS_NF_curr'] selfcal_library[vis][solint]['nfrms_multiplier'] = selfcal_library['RMS_NF_curr'] / selfcal_library['RMS_curr'] - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): selfcal_library[fid][vis][solint]={} selfcal_library[fid][vis][solint]['clean_threshold'] = selfcal_library['nsigma'][iteration]*selfcal_library['RMS_NF_curr'] selfcal_library[fid][vis][solint]['nfrms_multiplier'] = selfcal_library['RMS_NF_curr'] / selfcal_library['RMS_curr'] @@ -195,7 +198,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ for vis in vislist: selfcal_library[vis][solint]['Pass'] = False selfcal_library[vis][solint]['Fail_Reason'] = 'Empty model for solint '+solint - for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): selfcal_library[fid][vis][solint]['Pass'] = False selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Empty model for solint '+solint break # breakout of loop because the model is empty and gaincal will therefore fail @@ -226,13 +229,22 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. if selfcal_library['obstype'] == 'mosaic': - selfcal_library['sub-fields-to-gaincal'] = evaluate_subfields_to_gaincal(selfcal_library, target, band, - solint, iteration, selfcal_plan['solmode'], selfcal_plan['solints'], selfcal_plan, minsnr_to_proceed, - allow_gain_interpolation=allow_gain_interpolation) + selfcal_library['sub-fields-to-gaincal'] = [] + for vis in vislist: + selfcal_library[vis]['sub-fields-to-gaincal'] = evaluate_subfields_to_gaincal(vis, selfcal_library, target, band, + solint, iteration, selfcal_plan['solmode'], selfcal_plan['solints'], selfcal_plan, minsnr_to_proceed, + allow_gain_interpolation=allow_gain_interpolation) + selfcal_library['sub-fields-to-gaincal'] = np.union1d(selfcal_library['sub-fields-to-gaincal'], selfcal_library[vis]['sub-fields-to-gaincal']).tolist() if solint != 'inf_EB' and not allow_gain_interpolation: selfcal_library['sub-fields-to-selfcal'] = selfcal_library['sub-fields-to-gaincal'] - print('Fields to gaincal: ',selfcal_library['sub-fields-to-gaincal']) + for vis in vislist: + selfcal_library['sub-fields-to-selfcal'] = selfcal_library['sub-fields-to-gaincal'] + + print('Fields to gaincal: ') + for vis in vislist: + print(f'{vis}:', selfcal_library[vis]['sub-fields-to-gaincal']) + if len(selfcal_library['sub-fields-to-gaincal']) == 0: print('No fields to selfcal, exiting solution interval an selfcal for current target') selfcal_library['Stop_Reason']='Missing_flux_in_all_sub-fields_for_solint '+selfcal_plan['solints'][iteration] @@ -253,13 +265,17 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ else: selfcal_library['sub-fields-to-gaincal'] = selfcal_library['sub-fields-to-selfcal'] + for vis in vislist: + selfcal_library[vis]['sub-fields-to-gaincal'] = selfcal_library[vis]['sub-fields-to-selfcal'] - + print('selfcal_library["sub-fields-to-selfcal"] = ', selfcal_library['sub-fields-to-selfcal']) + for vis in selfcal_library['vislist']: + print('selfcal_library[vis]["sub-fields-to-selfcal"] = ', selfcal_library[vis]['sub-fields-to-selfcal']) # Calculate the complex gains for vis in vislist: - if np.intersect1d(selfcal_library['sub-fields-to-gaincal'],\ + if np.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],\ list(selfcal_library['sub-fields-fid_map'][vis].keys())).size == 0: continue @@ -275,16 +291,23 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ # With gaincal done and bad fields removed from gain tables if necessary, check whether any fields should no longer be # selfcal'd because they have too much interpolation. if selfcal_library['obstype'] == 'mosaic': - selfcal_library['sub-fields-to-selfcal'] = evaluate_subfields_after_gaincal(selfcal_library, target, band, - solint, iteration, selfcal_plan['solmode'], allow_gain_interpolation=allow_gain_interpolation) + selfcal_library['sub-fields-to-selfcal'] = [] + for vis in vislist: + selfcal_library[vis]['sub-fields-to-selfcal'] = evaluate_subfields_after_gaincal(vis, selfcal_library, selfcal_plan, target, band, + solint, iteration, selfcal_plan['solmode'], allow_gain_interpolation=allow_gain_interpolation) + selfcal_library['sub-fields-to-selfcal'] = np.union1d(selfcal_library['sub-fields-to-selfcal'], selfcal_library[vis]['sub-fields-to-selfcal']).astype(int).tolist() + + print('selfcal_library["sub-fields-to-selfcal"] = ', selfcal_library['sub-fields-to-selfcal']) + for vis in selfcal_library['vislist']: + print('selfcal_library[vis]["sub-fields-to-selfcal"] = ', selfcal_library[vis]['sub-fields-to-selfcal']) ## ## Apply gain solutions per MS, target, solint, and band ## for vis in vislist: applycal_wrapper(vis, target, band, solint, selfcal_library, - current=lambda f: f in selfcal_library['sub-fields-to-selfcal'], - final=lambda f: f not in selfcal_library['sub-fields-to-selfcal'] and selfcal_library[f]['SC_success'], + current=lambda f: f in selfcal_library[vis]['sub-fields-to-selfcal'], + final=lambda f: f not in selfcal_library[vis]['sub-fields-to-selfcal'] and selfcal_library[f]['SC_success'], restore_flags='fb_selfcal_starting_flags_'+sani_target+'_'+band if mode == "cocal" else None) ## Create post self-cal image using the model as a startmodel to evaluate how much selfcal helped @@ -314,6 +337,9 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ mosaic_SNR, mosaic_RMS, mosaic_SNR_NF, mosaic_RMS_NF = {}, {}, {}, {} post_mosaic_SNR, post_mosaic_RMS, post_mosaic_SNR_NF, post_mosaic_RMS_NF = {}, {}, {}, {} + print('selfcal_library["sub-fields-to-selfcal"] = ', selfcal_library['sub-fields-to-selfcal']) + for vis in vislist: + print('selfcal_library[vis]["sub-fields-to-selfcal"] = ', selfcal_library[vis]['sub-fields-to-selfcal']) for fid in selfcal_library['sub-fields-to-selfcal']: if selfcal_library['obstype'] == 'mosaic': imagename = sani_target+'_field_'+str(fid)+'_'+band+'_'+solint+'_'+str(iteration) @@ -524,6 +550,10 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ selfcal_library[fid]['inf_EB_SNR_decrease']=False for vis in selfcal_library[fid]['vislist-to-gaincal']: + # I think the below might not be correct for mosaics - it would set the gaintable even if fid is not in [vis]['sub-fields-to-selfcal']. + # I think this needs: + # if fid not in selfcal_library[vis]['sub-fields-to-selfcal']: + # continue selfcal_library[fid][vis]['gaintable_final']=selfcal_library[fid][vis][solint]['gaintable'] selfcal_library[fid][vis]['spwmap_final']=selfcal_library[fid][vis][solint]['spwmap'].copy() selfcal_library[fid][vis]['applycal_mode_final']=selfcal_library[fid][vis][solint]['applycal_mode'] @@ -774,6 +804,12 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if mode == "selfcal" and iteration > 1 and selfcal_plan['solmode'][iteration] !='ap' and do_amp_selfcal: # if a solution interval shorter than inf for phase-only SC has passed, attempt amplitude selfcal iteration=selfcal_plan['solmode'].index('ap') selfcal_library['sub-fields-to-selfcal'] = selfcal_library['sub-fields'] + for vis in vislist: + selfcal_library[vis]['sub-fields-to-selfcal'] = selfcal_library['sub-fields'].copy() + for fid in selfcal_library[vis]['sub-fields-to-selfcal']: + if not fid in selfcal_library['sub-fields-fid_map'][vis]: + # If a sub-field is not in an EB, it shouldn't be considered for selfcal for that EB + selfcal_library[vis]['sub-fields-to-selfcal'].pop(fid) print('****************Selfcal halted for phase, attempting amplitude*************') continue elif mode == "cocal" and "inf_fb" in solint: @@ -784,14 +820,17 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ break # breakout of loops of successive solints since solutions are getting worse # Finally, update the list of fields to be self-calibrated now that we don't need to know the list at the beginning of this solint. - new_fields_to_selfcal = [] coarsest_solint=selfcal_plan['solints'][0] - for fid in selfcal_library['sub-fields']: - if mode == "cocal": - if ("inf_EB" in selfcal_library[fid]['vislist'][0] and selfcal_library[fid][selfcal_library[fid]['vislist'][0]][coarsest_solint]["Pass"]) or selfcal_library[fid][selfcal_library[fid]['vislist'][0]][coarsest_solint+"_fb"]["Pass"]: - new_fields_to_selfcal.append(fid) - else: - if selfcal_library[fid][selfcal_library[fid]['vislist'][0]][coarsest_solint]["Pass"]: - new_fields_to_selfcal.append(fid) + selfcal_library['sub-fields-to-selfcal'] = [] + for vis in vislist: + new_fields_to_selfcal = [] + for fid in selfcal_library['sub-fields']: + if mode == "cocal": + if ("inf_EB" in selfcal_library[fid]['vislist'][0] and selfcal_library[fid][vis][coarsest_solint]["Pass"]) or selfcal_library[fid][selfcal_library[fid]['vislist'][0]][coarsest_solint+"_fb"]["Pass"]: + new_fields_to_selfcal.append(fid) + else: + if selfcal_library[fid][vis][coarsest_solint]["Pass"]: + new_fields_to_selfcal.append(fid) - selfcal_library['sub-fields-to-selfcal'] = new_fields_to_selfcal + selfcal_library[vis]['sub-fields-to-selfcal'] = new_fields_to_selfcal + selfcal_library['sub-fields-to-selfcal'] = np.union1d(selfcal_library['sub-fields-to-selfcal'], selfcal_library[vis]['sub-fields-to-selfcal']).tolist() \ No newline at end of file diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 6fa9b89c..210a2c18 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -1906,7 +1906,7 @@ def get_SNR_self_individual(vislist,selfcal_library,n_ant,solints,solint_setting solint_snr_per_bb[solint][baseband]=mean_SNR_bb solint_snr[solint]=np.mean(SNR_self_EB) selfcal_library['per_EB_SNR']=np.mean(SNR_self_EB) - elif solint_settings[solint]['interval'] =='scan_inf': + elif solint_settings[solint]['sub-name'] =='scan_inf': selfcal_library['per_scan_SNR']=SNR/((n_ant-3)**0.5*(selfcal_library['Total_TOS']/selfcal_library[vislist[0]]['Median_scan_time'])**0.5) solint_snr[solint]=selfcal_library['per_scan_SNR'] for spw in selfcal_library['spw_map']: @@ -3799,7 +3799,7 @@ def select_best_gaincal_mode(selfcal_library,selfcal_plan,vis,gaintable_prefix,s for i in range(len(spwlist)): # use >= to not always map if an spw has flagged solutions for a given antenna if np.min(selfcal_plan[vis]['solint_settings'][solint]['delta_nflags']['per_spw'][i]) >= max_flagged_ants_spwmap or \ - selfcal_plan['solint_snr_per_spw'][coarsest_solint][str(selfcal_library['reverse_spw_map'][vis][int(spwlist[i])])] < minsnr_to_proceed or \ + selfcal_plan[vis]['solint_snr_per_spw'][coarsest_solint][str(selfcal_library['reverse_spw_map'][vis][int(spwlist[i])])] < minsnr_to_proceed or \ selfcal_plan[vis]['solint_settings'][solint]['fracflagged']['per_spw'][i] == 1.0: fallback='spwmap' spwmap[i]=1.0 diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 746ced34..d4c9eca9 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -120,7 +120,7 @@ def test_on_github(tmp_path, request, zip_file, link): if solint not in solint_map: solint_map[solint] = [] - mapped_solint = selfcal_plan[target][band][vis]['solint_settings'][solint]['interval'] + mapped_solint = selfcal_plan[target][band][vis]['solint_settings'][solint]['sub-name'] if solint == 'inf_EB': mapped_solint += '_EB' elif 'ap' in solint: @@ -130,7 +130,10 @@ def test_on_github(tmp_path, request, zip_file, link): print(solint_map) difference_count = compare_two_dictionaries(selfcal_library1, selfcal_library2, tolerance=0.001, key_map=solint_map, - exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal", "telescope"]) + exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal", "telescope", + "gaintable","sub-fields-to-gaincal", "sub-fields-to-selfcal", "am_dogrowprune", "am_growiterations", + "am_lownoisethreshold", "am_minbeamfrac", "am_noisethreshold", "am_sidelobethreshold", + "am_smoothfactor"]) assert difference_count == 0 From fb73814bd7183b89b0e5d8fb2451d6fe9a570693 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Thu, 7 May 2026 17:26:56 -0400 Subject: [PATCH 27/43] add sanitize string calls to construction of concat vislist --- auto_selfcal/auto_selfcal.py | 12 +++++++++--- auto_selfcal/run_selfcal.py | 2 +- bin/auto_selfcal.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index ef23b1d0..d1c928e9 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -775,9 +775,15 @@ def default(self, obj): selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] for vis in selfcal_library[target][band]['vislist']: selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]=selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + if "inf" in selfcal_plan[target][band][vis]['solint_settings'].keys(): + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + else: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index e9945106..530df8b0 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -61,7 +61,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ os.system('mv '+vis+'.flagversions '+vis.replace('.ms','_orig.ms.flagversions')) concatvislist=[] for cal_target in include_targets.split(','): - concatvislist.append(vis.replace(target,cal_target)) + concatvislist.append(vis.replace(sanitize_string(target),sanitize_string(cal_target))) concatvislist.append(vis.replace('.ms','_orig.ms')) print('Concat vislist: ',concatvislist) concat(vis=concatvislist,concatvis=vis) diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index 5b6fe7f6..4285c610 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,targets='FZ_Tau,DL_Tau,DO_Tau',parallel=parallel) +auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,parallel=parallel) From 3ac55bcfd4f0dc1eaec45c24d42740fc623e24b5 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 8 May 2026 11:03:12 -0400 Subject: [PATCH 28/43] remove mask check if doing cocal --- auto_selfcal/run_selfcal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 530df8b0..93e962c0 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -57,6 +57,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ return else: for vis in vislist: + clearcal(vis=vis,addmodel=True) os.system('mv '+vis+' '+vis.replace('.ms','_orig.ms')) os.system('mv '+vis+'.flagversions '+vis.replace('.ms','_orig.ms.flagversions')) concatvislist=[] @@ -183,7 +184,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ # Check that a mask was actually created, because if not the model will be empty and gaincal will do bad things and the # code will break. - if not checkmask(sani_target+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0'): + if not checkmask(sani_target+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0') and mode !="cocal": selfcal_library['Stop_Reason'] = 'Empty model for solint '+solint for fid in selfcal_library['sub-fields-to-selfcal']: selfcal_library[fid]['Stop_Reason'] = 'Empty model for solint '+solint From 4392918a2b422cf0e7068480ed8abeafabf302c8 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Mon, 11 May 2026 18:30:26 +0000 Subject: [PATCH 29/43] No need to add _ap or _EB for solint matching now that we are tracking the sub-name. --- auto_selfcal/tests/test_auto_selfcal.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index d4c9eca9..2fba8761 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -121,10 +121,6 @@ def test_on_github(tmp_path, request, zip_file, link): solint_map[solint] = [] mapped_solint = selfcal_plan[target][band][vis]['solint_settings'][solint]['sub-name'] - if solint == 'inf_EB': - mapped_solint += '_EB' - elif 'ap' in solint: - mapped_solint += '_ap' solint_map[solint].append(mapped_solint) print(solint_map) From e733a4aed5a7516e9bab9f4dedcb23d3b1f8cc64 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 12 May 2026 09:22:23 -0400 Subject: [PATCH 30/43] Make sure the ap solints actually use solmode ap --- auto_selfcal/prepare_selfcal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index 1cb94dad..ff1e7e64 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -671,7 +671,7 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval - if '_ap' in solint: + if 'ap' in solint: selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='ap' else: selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='p' From 5fe98a6a11f5fd75ab8caeaa05504f6c8edf8239 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 12 May 2026 10:03:40 -0400 Subject: [PATCH 31/43] sub-fields-to-selfcal was not being properly updated per-vis after evaluating which fields should be gaincal'd --- auto_selfcal/run_selfcal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 0a3a00dc..4ce4eb33 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -239,7 +239,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if solint != 'inf_EB' and not allow_gain_interpolation: selfcal_library['sub-fields-to-selfcal'] = selfcal_library['sub-fields-to-gaincal'] for vis in vislist: - selfcal_library['sub-fields-to-selfcal'] = selfcal_library['sub-fields-to-gaincal'] + selfcal_library[vis]['sub-fields-to-selfcal'] = selfcal_library['sub-fields-to-gaincal'] print('Fields to gaincal: ') for vis in vislist: @@ -833,4 +833,4 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ new_fields_to_selfcal.append(fid) selfcal_library[vis]['sub-fields-to-selfcal'] = new_fields_to_selfcal - selfcal_library['sub-fields-to-selfcal'] = np.union1d(selfcal_library['sub-fields-to-selfcal'], selfcal_library[vis]['sub-fields-to-selfcal']).tolist() \ No newline at end of file + selfcal_library['sub-fields-to-selfcal'] = np.union1d(selfcal_library['sub-fields-to-selfcal'], selfcal_library[vis]['sub-fields-to-selfcal']).tolist() From 7da3c3f5e5213640820555357d37f382595e92eb Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 12 May 2026 17:46:58 +0000 Subject: [PATCH 32/43] Track Stop_Reason per EB as well. --- auto_selfcal/mosaic_helpers.py | 6 ++--- auto_selfcal/run_selfcal.py | 29 ++++++++++++------------- auto_selfcal/tests/test_auto_selfcal.py | 2 +- auto_selfcal/weblog_creation.py | 16 ++++++++------ 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/auto_selfcal/mosaic_helpers.py b/auto_selfcal/mosaic_helpers.py index 3301bfde..65a470a4 100644 --- a/auto_selfcal/mosaic_helpers.py +++ b/auto_selfcal/mosaic_helpers.py @@ -55,7 +55,7 @@ def evaluate_subfields_to_gaincal(vis, selfcal_library, target, band, solint, it if fid not in new_fields_to_selfcal and solint != "inf_EB" and not allow_gain_interpolation: #selfcal_library[fid][vis][solint]['interpolated_gains'] = True #selfcal_library[fid]['Stop_Reason'] = "Gaincal solutions would be interpolated" - #selfcal_library[fid]['Stop_Reason'] = skip_reason + selfcal_library[fid][vis]['Stop_Reason'] = skip_reason selfcal_library[fid][vis][solint]['Pass'] = "None" selfcal_library[fid][vis][solint]['Fail_Reason'] = skip_reason @@ -83,7 +83,7 @@ def evaluate_subfields_after_gaincal(vis, selfcal_library, selfcal_plan, target, for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())): new_fields_to_selfcal.remove(fid) - selfcal_library[fid]['Stop_Reason'] = 'No viable calibrator fields in at least 1 EB' + selfcal_library[fid][vis]['Stop_Reason'] = 'No viable calibrator fields in at least 1 EB' selfcal_library[fid][vis][solint]['Pass'] = 'None' if 'Fail_Reason' in selfcal_library[fid][vis][solint]: selfcal_library[fid][vis][solint]['Fail_Reason'] += '; ' @@ -142,7 +142,7 @@ def evaluate_subfields_after_gaincal(vis, selfcal_library, selfcal_plan, target, for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())): new_fields_to_selfcal.remove(fid) - selfcal_library[fid]['Stop_Reason'] = 'No viable calibrator fields for inf_EB in at least 1 EB' + selfcal_library[fid][vis]['Stop_Reason'] = 'No viable calibrator fields for inf_EB in at least 1 EB' selfcal_library[fid][vis][solint]['Pass'] = 'None' selfcal_library[fid][vis][solint]['Fail_Reason'] = 'No viable inf_EB fields' diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 4ce4eb33..f69bbbed 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -126,9 +126,10 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ print('****************Attempting amplitude selfcal*************') continue - selfcal_library['Stop_Reason']='Estimated_SNR_too_low_for_solint '+selfcal_plan['solints'][iteration] - for fid in selfcal_library['sub-fields-to-selfcal']: - selfcal_library[fid]['Stop_Reason']='Estimated_SNR_too_low_for_solint '+selfcal_plan['solints'][iteration] + for vis in vislist: + selfcal_library[vis]['Stop_Reason']='Estimated_SNR_too_low_for_solint '+selfcal_plan['solints'][iteration] + for fid in selfcal_library['sub-fields-to-selfcal']: + selfcal_library[fid][vis]['Stop_Reason']='Estimated_SNR_too_low_for_solint '+selfcal_plan['solints'][iteration] break else: selfcal_library['vislist-to-gaincal'] = vislist @@ -191,14 +192,13 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ # Check that a mask was actually created, because if not the model will be empty and gaincal will do bad things and the # code will break. - if not checkmask(sani_target+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0'): - selfcal_library['Stop_Reason'] = 'Empty model for solint '+solint - for fid in selfcal_library['sub-fields-to-selfcal']: - selfcal_library[fid]['Stop_Reason'] = 'Empty model for solint '+solint + if not checkmask(sani_target+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0'): for vis in vislist: + selfcal_library[vis]['Stop_Reason'] = 'Empty model for solint '+solint selfcal_library[vis][solint]['Pass'] = False selfcal_library[vis][solint]['Fail_Reason'] = 'Empty model for solint '+solint for fid in np.intersect1d(selfcal_library[vis]['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): + selfcal_library[fid][vis]['Stop_Reason'] = 'Empty model for solint '+solint selfcal_library[fid][vis][solint]['Pass'] = False selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Empty model for solint '+solint break # breakout of loop because the model is empty and gaincal will therefore fail @@ -246,17 +246,16 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ print(f'{vis}:', selfcal_library[vis]['sub-fields-to-gaincal']) if len(selfcal_library['sub-fields-to-gaincal']) == 0: - print('No fields to selfcal, exiting solution interval an selfcal for current target') - selfcal_library['Stop_Reason']='Missing_flux_in_all_sub-fields_for_solint '+selfcal_plan['solints'][iteration] - for fid in list(selfcal_library['sub-fields-fid_map'][vis].keys()): - selfcal_library[fid]['Stop_Reason']='Missing_flux_in_all_sub-fields_for_solint '+selfcal_plan['solints'][iteration] + print('No fields to selfcal, exiting solution interval an selfcal for current target') for vis in vislist: + selfcal_library[vis]['Stop_Reason']='Missing_flux_in_all_sub-fields_for_solint '+selfcal_plan['solints'][iteration] selfcal_library[vis][solint]['Pass'] = 'None' selfcal_library[vis][solint]['Fail_Reason'] = 'Missing_flux_in_all_sub-fields_for_solint '+solint #for fid in np.intersect1d(selfcal_library['sub-fields-to-selfcal'],list(selfcal_library['sub-fields-fid_map'][vis].keys())): for fid in list(selfcal_library['sub-fields-fid_map'][vis].keys()): if solint in selfcal_library[fid][vis].keys(): + selfcal_library[fid][vis]['Stop_Reason']='Missing_flux_in_all_sub-fields_for_solint '+selfcal_plan['solints'][iteration] selfcal_library[fid][vis][solint]['Pass'] = 'None' selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Missing_flux_in_all_sub-fields_for_solint '+solint @@ -518,13 +517,13 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ continue selfcal_library['SC_success']=True - selfcal_library['Stop_Reason']='None' #keep track of whether inf_EB had a S/N decrease if (solint =='inf_EB' or 'd' in solint) and (((post_SNR-SNR)/SNR < 0.0) or ((post_SNR_NF - SNR_NF)/SNR_NF < 0.0)): selfcal_library['inf_EB_SNR_decrease']=True elif (solint =='inf_EB' or 'd' in solint) and (((post_SNR-SNR)/SNR >= 0.0) and ((post_SNR_NF - SNR_NF)/SNR_NF >= 0.0)): selfcal_library['inf_EB_SNR_decrease']=False for vis in vislist: + selfcal_library[vis]['Stop_Reason']='None' selfcal_library[vis]['gaintable_final']=selfcal_library[vis][solint]['gaintable'] selfcal_library[vis]['spwmap_final']=selfcal_library[vis][solint]['spwmap'].copy() selfcal_library[vis]['applycal_mode_final']=selfcal_library[vis][solint]['applycal_mode'] @@ -543,7 +542,6 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ for ind, fid in enumerate(selfcal_library['sub-fields-to-selfcal']): if field_by_field_success[ind]: selfcal_library[fid]['SC_success']=True - selfcal_library[fid]['Stop_Reason']='None' if (solint =='inf_EB') and not strict_field_by_field_success[ind]: selfcal_library[fid]['inf_EB_SNR_decrease']=True elif (solint =='inf_EB') and strict_field_by_field_success[ind]: @@ -554,6 +552,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ # I think this needs: # if fid not in selfcal_library[vis]['sub-fields-to-selfcal']: # continue + selfcal_library[fid][vis]['Stop_Reason']='None' selfcal_library[fid][vis]['gaintable_final']=selfcal_library[fid][vis][solint]['gaintable'] selfcal_library[fid][vis]['spwmap_final']=selfcal_library[fid][vis][solint]['spwmap'].copy() selfcal_library[fid][vis]['applycal_mode_final']=selfcal_library[fid][vis][solint]['applycal_mode'] @@ -668,8 +667,8 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if reason != '': reason=reason+'; ' reason=reason+'All sub-fields failed' - selfcal_library['Stop_Reason']=reason for vis in vislist: + selfcal_library['Stop_Reason']=reason #selfcal_library[vis][solint]['Pass']=False selfcal_library[vis][solint]['Fail_Reason']=reason @@ -698,8 +697,8 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ mosaic_reason[fid]=mosaic_reason[fid]+'NF RMS increase beyond 5%' if mosaic_reason[fid] == '': mosaic_reason[fid] = "Global selfcal failed" - selfcal_library[fid]['Stop_Reason']=mosaic_reason[fid] for vis in selfcal_library[fid]['vislist-to-gaincal']: + selfcal_library[fid][vis]['Stop_Reason']=mosaic_reason[fid] #selfcal_library[fid][vis][solint]['Pass']=False selfcal_library[fid][vis][solint]['Fail_Reason']=mosaic_reason[fid] diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index b419c288..57b506c7 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -140,7 +140,7 @@ def test_on_github(tmp_path, request, zip_file, link): exclude=["final_phase_solint", "final_solint", "gaintable_final", "per_EB_SNR", "vislist-to-gaincal", "telescope", "gaintable","sub-fields-to-gaincal", "sub-fields-to-selfcal", "am_dogrowprune", "am_growiterations", "am_lownoisethreshold", "am_minbeamfrac", "am_noisethreshold", "am_sidelobethreshold", - "am_smoothfactor"]) + "am_smoothfactor", "Stop_Reason"]) assert difference_count == 0 diff --git a/auto_selfcal/weblog_creation.py b/auto_selfcal/weblog_creation.py index 1fa0f01b..c7a631e0 100644 --- a/auto_selfcal/weblog_creation.py +++ b/auto_selfcal/weblog_creation.py @@ -79,15 +79,17 @@ def generate_weblog(sclib,selfcal_plan,directory='weblog'): htmlOut.writelines('Aligned EBs?: False\n') htmlOut.writelines('Selfcal Success?: '+str(sclib[target][band]['SC_success'])+'
\n') keylist=sclib[target][band].keys() - if 'Stop_Reason' not in keylist: + if np.all(['Stop_Reason' not in sclib[target][band][vis].keys() for vis in sclib[target][band]['vislist']]): htmlOut.writelines('Stop Reason: Estimated Selfcal S/N too low for solint

\n') if sclib[target][band]['SC_success']==False: render_summary_table(htmlOut,sclib,target,band,directory=directory) continue else: - htmlOut.writelines('Stop Reason: '+str(sclib[target][band]['Stop_Reason'])+'

\n') - print(target,band,sclib[target][band]['Stop_Reason']) - if (('Estimated_SNR_too_low_for_solint' in sclib[target][band]['Stop_Reason']) or ('Selfcal_Not_Attempted' in sclib[target][band]['Stop_Reason'])) and sclib[target][band]['final_solint']=='None': + for vis in sclib[target][band]['vislist']: + htmlOut.writelines(vis + 'Stop Reason: '+str(sclib[target][band][vis]['Stop_Reason'])+'

\n') + print(vis, target,band,sclib[target][band][vis]['Stop_Reason']) + if ((np.all(['Estimated_SNR_too_low_for_solint' in sclib[target][band][vis]['Stop_Reason'] for vis in sclib[target][band][vis]])) or \ + (np.all(['Selfcal_Not_Attempted' in sclib[target][band][vis]['Stop_Reason'] for vis in sclib[target][band]['vislist']]))) and sclib[target][band]['final_solint']=='None': render_summary_table(htmlOut,sclib,target,band,directory=directory) continue htmlOut.writelines('Final Successful solint: '+str(sclib[target][band]['final_solint'])+'

\n') @@ -115,7 +117,7 @@ def generate_weblog(sclib,selfcal_plan,directory='weblog'): htmlOut.writelines('Noise Characteristics
\n') # Solint summary table - if 'Empty model' not in sclib[target][band]['Stop_Reason']: + if 'Empty model' not in sclib[target][band][sclib[target][band]['vislist'][0]]['Stop_Reason']: render_selfcal_solint_summary_table(htmlOut,sclib,target,band,selfcal_plan) # PER SPW STATS TABLE @@ -128,7 +130,7 @@ def generate_weblog(sclib,selfcal_plan,directory='weblog'): htmlOut.close() # Pages for each solint - if 'Empty model' not in sclib[target][band]['Stop_Reason']: + if 'Empty model' not in sclib[target][band][sclib[target][band]['vislist'][0]]['Stop_Reason']: render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory=directory) @@ -566,7 +568,7 @@ def render_per_solint_QA_pages(sclib,selfcal_plan,bands,directory='weblog'): htmlOutSolint.writelines('

Passed: True

\n') else: htmlOutSolint.writelines('

Passed: False

\n') - if 'Empty model' in sclib[target][band]['Stop_Reason']: + if 'Empty model' in sclib[target][band][sclib[target][band]['vislist'][0]]['Stop_Reason']: htmlOutSolint.writelines('Empty model image, no gains solved
\n') htmlOutSolint.writelines('\n') htmlOutSolint.writelines('\n') From 2643132bf19305240bc12208cf45f071d2793c51 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Wed, 13 May 2026 13:02:56 +0000 Subject: [PATCH 33/43] vis should be vislist --- auto_selfcal/weblog_creation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/weblog_creation.py b/auto_selfcal/weblog_creation.py index c7a631e0..61102b7b 100644 --- a/auto_selfcal/weblog_creation.py +++ b/auto_selfcal/weblog_creation.py @@ -88,7 +88,7 @@ def generate_weblog(sclib,selfcal_plan,directory='weblog'): for vis in sclib[target][band]['vislist']: htmlOut.writelines(vis + 'Stop Reason: '+str(sclib[target][band][vis]['Stop_Reason'])+'

\n') print(vis, target,band,sclib[target][band][vis]['Stop_Reason']) - if ((np.all(['Estimated_SNR_too_low_for_solint' in sclib[target][band][vis]['Stop_Reason'] for vis in sclib[target][band][vis]])) or \ + if ((np.all(['Estimated_SNR_too_low_for_solint' in sclib[target][band][vis]['Stop_Reason'] for vis in sclib[target][band]['vislist']])) or \ (np.all(['Selfcal_Not_Attempted' in sclib[target][band][vis]['Stop_Reason'] for vis in sclib[target][band]['vislist']]))) and sclib[target][band]['final_solint']=='None': render_summary_table(htmlOut,sclib,target,band,directory=directory) continue From f81270daac5f4710b7ac9a822a307049b31b9a62 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 14 May 2026 15:05:09 -0400 Subject: [PATCH 34/43] Update cocal to work better with setting the solint_settings for the fb solints. This includes removing the stapling together of existing gaintables for inf_fb3, and also splitting cocal preparation into prepare_cocal. --- auto_selfcal/auto_selfcal.py | 116 +-------- auto_selfcal/gaincal_wrapper.py | 419 ++++++++------------------------ auto_selfcal/mosaic_helpers.py | 118 +++++++++ auto_selfcal/prepare_cocal.py | 143 +++++++++++ auto_selfcal/prepare_selfcal.py | 43 ++-- auto_selfcal/selfcal_helpers.py | 8 +- 6 files changed, 403 insertions(+), 444 deletions(-) create mode 100644 auto_selfcal/prepare_cocal.py diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index ef23b1d0..1db79af9 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -8,6 +8,7 @@ import sys import pickle import pprint +import copy from .selfcal_helpers import * from .run_selfcal import run_selfcal @@ -15,6 +16,7 @@ from .weblog_creation import * from .prepare_selfcal import prepare_selfcal, set_clean_thresholds, plan_selfcal_per_solint from .original_ms_helpers import applycal_to_orig_MSes, uvcontsub_orig_MSes +from .prepare_cocal import prepare_cocal try: from .alignment_helpers import align_measurement_sets except: @@ -729,116 +731,7 @@ def default(self, obj): if allow_cocal: - ## - ## Save the flags following the main iteration of self-calibration since we will need to revert to the beginning for the fallback mode. - ## - # PS: I don't need this anymore? - for vis in selfcal_library[target][band]['vislist']: - if not os.path.exists(vis+'.flagversions/flags.fb_selfcal_starting_flags'): - flagmanager(vis=vis,mode='save',versionname='fb_selfcal_starting_flags') - else: - flagmanager(vis=vis,mode='restore',versionname='fb_selfcal_starting_flags') - - ## - ## For sources that self-calibration failed, try to use the inf_EB and the inf solutions from the sources that - ## were successful. - - for target in selfcal_library.keys(): - for band in selfcal_library[target].keys(): - print(target, selfcal_library[target][band]["final_solint"]) - - inf_EB_fields = {} - inf_fields = {} - fallback_fields = {} - calibrators = {} - for band in vis_for_targets[target]['Bands']: - # Initialize the lists for this band. - inf_EB_fields[band] = [] - inf_fields[band] = [] - fallback_fields[band] = [] - - # Loop through and identify which sources belong where. - for target in selfcal_library.keys(): - if selfcal_library[target][band]['SC_success'] and 'fb' not in selfcal_library[target][band]['final_solint']: - inf_EB_fields[band].append(target) - if selfcal_library[target][band]['final_solint'] != 'inf_EB': - inf_fields[band].append(target) - elif 'inf' in selfcal_plan[target][band]['solints']: - fallback_fields[band].append(target) - else: - fallback_fields[band].append(target) - - # Update the relevant lists if we are going to do a fallback mode. - if len(fallback_fields[band]) > 0: - selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] - selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] - selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] - for vis in selfcal_library[target][band]['vislist']: - selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]=selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] - selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] - #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] - calibrators[band] = [inf_EB_fields[band], inf_fields[band], inf_fields[band], inf_fields[band]] - selfcal_library[target][band]["nsigma"] = np.concatenate((selfcal_library[target][band]["nsigma"],[selfcal_library[target][band]["nsigma"][0], \ - selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1]])) - - print(inf_EB_fields) - print(inf_fields) - print(fallback_fields) - - ## - ## Reset the inf_EB informational dictionaries. - ## - - for target in selfcal_library: - for band in selfcal_library[target].keys(): - # If the target had a successful inf_EB solution, no need to reset. - if target in inf_EB_fields[band]: - continue - - for vis in selfcal_library[target][band]['vislist']: - selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']=inf_EB_gaincal_combine #'scan' - if selfcal_library[target][band]['obstype']=='mosaic': - selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']+=',field' - selfcal_plan[target][band][vis]['inf_EB_gaintype']=inf_EB_gaintype #G - selfcal_plan[target][band][vis]['inf_EB_fallback_mode']='' #'scan' - - - calculate_inf_EB_fb_anyways = True - preapply_targets_own_inf_EB = False - - ## The below sets the calibrations back to what they were prior to starting the fallback mode. It should not be needed - ## for the final version of the codue, but is used for testing. - - - for target in selfcal_library: - sani_target=sanitize_string(target) - for band in selfcal_library[target].keys(): - if target not in fallback_fields[band]: - continue - if 'gaintable_final' in selfcal_library[target][band]['vislist'][0]: - print('****************Reapplying previous solint solutions*************') - for vis in selfcal_library[target][band]['vislist']: - print('****************Applying '+str(selfcal_library[target][band][vis]['gaintable_final'])+' to '+target+' '+band+'*************') - ## NOTE: should this be selfcal_starting_flags instead of fb_selfcal_starting_flags ??? - flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) - applycal(vis=vis,\ - gaintable=selfcal_library[target][band][vis]['gaintable_final'],\ - interp=selfcal_library[target][band][vis]['applycal_interpolate_final'],\ - calwt=True,spwmap=selfcal_library[target][band][vis]['spwmap_final'],\ - applymode=selfcal_library[target][band][vis]['applycal_mode_final'],\ - field=target,spw=selfcal_library[target][band][vis]['spws']) - else: - print('****************Removing all calibrations for '+target+' '+band+'**************') - for vis in selfcal_library[target][band]['vislist']: - flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) - clearcal(vis=vis,field=target,spw=selfcal_library[target][band][vis]['spws']) - ## END - - + fallback_fields, calibrators = prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_gaintype) ## ## Begin fallback self-cal loops ## @@ -852,8 +745,7 @@ def default(self, obj): inf_EB_gaincal_combine=inf_EB_gaincal_combine, inf_EB_gaintype=inf_EB_gaintype, unflag_only_lbants=unflag_only_lbants, \ unflag_only_lbants_onlyap=unflag_only_lbants_onlyap, calonly_max_flagged=calonly_max_flagged, \ second_iter_solmode=second_iter_solmode, unflag_fb_to_prev_solint=unflag_fb_to_prev_solint, rerank_refants=rerank_refants, \ - mode="cocal", calibrators=calibrators, calculate_inf_EB_fb_anyways=calculate_inf_EB_fb_anyways, \ - preapply_targets_own_inf_EB=preapply_targets_own_inf_EB, gaincalibrator_dict=gaincalibrator_dict, allow_gain_interpolation=True) + mode="cocal", calibrators=calibrators, gaincalibrator_dict=gaincalibrator_dict, allow_gain_interpolation=True) if debug: print(json.dumps(selfcal_library, indent=4, cls=NpEncoder)) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 238689f3..fae8cc42 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -1,5 +1,6 @@ import numpy as np from .selfcal_helpers import * +from .mosaic_helpers import scan_inf_scan_combine def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, solint_interval, applymode, iteration, gaincal_minsnr, gaincal_unflag_minsnr=5.0, minsnr_to_proceed=3.0, rerank_refants=False, unflag_only_lbants=False, unflag_only_lbants_onlyap=False, @@ -31,6 +32,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so ## Reset the gaincal return dictionaries, in case this is a repeat of the current solution interval. for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode] = [] + ## ## Set gaincal parameters depending on which iteration and whether to use combine=spw for inf_EB or not ## Defaults should assume combine='scan' and gaintpe='G' will fallback to combine='scan,spw' if too much flagging @@ -39,58 +41,31 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so current_solint_index=selfcal_plan['solints'].index(solint) selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'] = {} if selfcal_plan['solmode'][iteration] == 'p': - if mode == "cocal": - if 'inf_EB' in selfcal_library[vis]: - #gaincal_preapply_gaintable[vis]=[sani_target+'_'+vis+'_'+band+'_inf_EB_0_p.g'] - if calculate_inf_EB_fb_anyways or not selfcal_library[vis]["inf_EB"]["Pass"]: - previous_solint = "inf_EB_fb" - else: - previous_solint = "inf_EB" - else: - #gaincal_preapply_gaintable[vis]=[sani_target+'_'+vis+'_'+band+'_inf_EB_fb_'+str(iteration-1)+'_p.g'] - previous_solint = "inf_EB_fb" - else: - # not entirely sure why this if/else is necessary might not be with new selfcal plan - if selfcal_plan['solmode'][iteration]=='p': - previous_solint = "inf_EB" - else: - previous_solint = selfcal_library['final_phase_solint'] gaincal_spwmap=[] gaincal_preapply_gaintable=[] gaincal_interpolate=[] applycal_spwmap=[] applycal_interpolate=[] applycal_gaintable=[] - for j in range(current_solint_index): - if selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['preapply_this_gaintable']: - gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['accepted_gaintable']) - gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_spwmap']) - gaincal_interpolate.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_interpolate']) - applycal_spwmap.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_spwmap']) - applycal_interpolate.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_interpolate']) - applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['accepted_gaintable']) + + if mode == "cocal": + include_solints = selfcal_plan[vis]['solint_settings'][solint]['preapply_solints'] + else: + include_solints = [selfcal_plan['solints'][j] for j in range(current_solint_index) if + selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['preapply_this_gaintable']] + + for incl_solint in include_solints: + gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) + gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) + gaincal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) + applycal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) + applycal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) + applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) if solint != 'inf_EB': gaincal_solmode = "" if not do_fallback_calonly or second_iter_solmode == "GSPLINE" else second_iter_solmode - """ - if 'spw' in selfcal_plan[vis]['inf_EB_gaincal_combine']: - applycal_spwmap[vis]=[selfcal_library[vis]['spwmap'],selfcal_library[vis]['spwmap']] - gaincal_spwmap[vis]=[selfcal_library[vis]['spwmap']] - elif selfcal_plan[vis]['inf_EB_fallback_mode']=='spwmap': - applycal_spwmap[vis]=selfcal_library[vis]['inf_EB']['spwmap'] + [selfcal_library[vis]['spwmap']] - gaincal_spwmap[vis]=selfcal_library[vis]['inf_EB']['spwmap'] - else: - applycal_spwmap[vis]=[[],selfcal_library[vis]['spwmap']] - gaincal_spwmap[vis]=[] - """ - # Revert back to applying the inf_EB solution if calculate_inf_EB_fb_anyways, i.e. we just use the inf_EB_fb solution - # for gaincal. - if mode == "cocal": - if selfcal_library['final_solint'] == 'inf_EB' and calculate_inf_EB_fb_anyways: - previous_solint = "inf_EB" - fallback='' if selfcal_plan['solmode'][iteration] == 'ap': solnorm=True @@ -109,293 +84,106 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print('Include targets: ', include_targets) if solint == "scan_inf": - if len(gaincalibrator_dict[vis]) > 0: - print("Determining scan_inf from calibrator scans in full MS") - scans = [] - intents = [] - times = [] - for t in gaincalibrator_dict[vis].keys(): - scans += [gaincalibrator_dict[vis][t]["scans"]] - intents += [np.repeat(gaincalibrator_dict[vis][t]["intent"],gaincalibrator_dict[vis][t]["scans"].size)] - times += [gaincalibrator_dict[vis][t]["times"]] - - times = np.concatenate(times) - order = np.argsort(times) - times = times[order] - - scans = np.concatenate(scans)[order] - intents = np.concatenate(intents)[order] - - is_gaincalibrator = intents == "phase" - scans = scans[is_gaincalibrator] - - msmd.open(vis) - if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': - scan_ids_for_target = msmd.scansforfield(target) - elif 'VLA' in selfcal_library['telescope']: - scan_ids_for_target=np.array([],dtype=int) - for fid in selfcal_library['sub-fields']: - if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): - field_id=selfcal_library['sub-fields-fid_map'][vis][fid] - scan_ids_for_target=np.append(scan_ids_for_target,msmd.scansforfield(field_id)) - - scan_ids_for_target.sort() # sort scans since they will be out of order - include_scans = [] - for iscan in range(scans.size-1): - scan_group = np.intersect1d(scan_ids_for_target, - np.array(list(range(scans[iscan]+1, scans[iscan+1])))).astype(str) - if scan_group.size > 0: - include_scans.append(",".join(scan_group)) - # PIPE-2741: if there are any scans before the first gain calibrator scan or after the last gain calibrator scan, catch them. - if scans.size > 0: - extra_scans = scan_ids_for_target[scan_ids_for_target > max(scans)] - if extra_scans.size > 0: - include_scans.append(','.join(extra_scans.astype(str))) - extra_scans = scan_ids_for_target[scan_ids_for_target < min(scans)] - if extra_scans.size > 0: - include_scans.append(','.join(extra_scans.astype(str))) - msmd.close() - elif guess_scan_combine: - print("Determining scan_inf from guessing where the calibrator scans were") - msmd.open(vis) - include_scans = [] - - #to guess at scan_inf combination for VLA look for breaks in the consecutive - #scan numbers and assume that the break is due to a calibrator scan - #Fetch scans for scan inf by collecting the field ids and running msmd.scansforfield - if 'VLA' in selfcal_library['telescope']: - scans=np.array([],dtype=int) - for fid in selfcal_library['sub-fields']: - if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): - field_id=selfcal_library['sub-fields-fid_map'][vis][fid] - scans=np.append(scans,msmd.scansforfield(field_id)) - - scans.sort() # sort scans since they will be out of order - scan_group='' - for iscan in range(scans.size): - if len(include_scans) > 0: - if str(scans[iscan]) in include_scans[-1]: - continue - if scan_group == '': - scan_group = str(int(scans[iscan])) - - if iscan < scans.size-1: - if scans[iscan+1] == scans[iscan]+1: - scan_group += ","+str(int(scans[iscan+1])) - else: - include_scans.append(scan_group) - scan_group='' - else: #write out the last scan group to include_scans - if scan_group != '': - include_scans.append(scan_group) - scan_group='' - - #guess scan_inf combination by getting all the scans for targets and do a simple grouping - if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': - scans = msmd.scansforfield(target) - - for iscan in range(scans.size): - if len(include_scans) > 0: - if str(scans[iscan]) in include_scans[-1]: - continue - - scan_group = str(scans[iscan]) - - if iscan < scans.size-1: - if msmd.fieldsforscan(scans[iscan+1]).size < msmd.fieldsforscan(scans[iscan]).size/3: - scan_group += ","+str(scans[iscan+1]) - - include_scans.append(scan_group) - - msmd.close() - else: - print("Not guessing where calibration scans are and justincluding all scans") - msmd.open(vis) - if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': - include_scans = [str(scan) for scan in msmd.scansforfield(target)] - elif 'VLA' in selfcal_library['telescope']: - scans=np.array([],dtype=int) - for fid in selfcal_library['sub-fields']: - if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): - field_id=selfcal_library['sub-fields-fid_map'][vis][fid] - scans=np.append(scans,msmd.scansforfield(field_id)) - - scans.sort() # sort scans since they will be out of order - include_scans = [str(scan) for scan in scans] - msmd.close() + include_scans = scan_inf_scan_combine(selfcal_library, vis, target, gaincalibrator_dict, guess_scan_combine=guess_scan_combine) else: include_scans = [include_scans] - if mode == "cocal" and preapply_targets_own_inf_EB and "inf_fb" in solint and "inf" in selfcal_plan['solints']: - ## - ## If we want to pre-apply inf_EB solution from each calibrator to itself, all we do is combine all of thier - ## individual inf tables, as these were pre-calculated in that way. - ## - for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] - - destination_table = sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - for t in include_targets.split(","): - sani_t = sanitize_string(t) - if os.path.exists(sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '_'+filename_append+'.pre-pass.g'): - table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '_'+filename_append+'.pre-pass.g' - else: - table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '_'+filename_append+'.g' - #t_final_solint = selfcal_library[t][band]["final_phase_solint"] - #t_iteration = selfcal_library[t][band][vislist[0]][t_final_solint]["iteration"] - #table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+t_final_solint+'_'+str(t_iteration)+'_'+solmode[band][iteration]+'.g' - - if not os.path.exists(table_name): - continue + + # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. + gaincal_solmode="" + if selfcal_library['obstype'] == 'mosaic': + msmd.open(vis) + include_targets = [] + remove = [] + for incl_scan in include_scans: + scan_targets = [] + for fid in [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ + np.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))] if incl_scan == '' else \ + np.intersect1d(msmd.fieldsforscans(np.array(incl_scan.split(",")).astype(int)), \ + [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ + numpy.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))]): + # Note: because of the msmd above getting actual fids from the MS, we just need to append fid below. + scan_targets.append(fid) + + if len(scan_targets) > 0: + include_targets.append(','.join(np.array(scan_targets).astype(str))) + else: + remove.append(incl_scan) - rerefant(vis, table_name, caltable="tmp0.g", refantmode="strict", refant=selfcal_library[vis]['refant']) + for incl_scan in remove: + include_scans.remove(incl_scan) - tb.open("tmp0.g") - if not os.path.exists("tmp1.g"): - tb.copy("tmp1.g", deep=True) - else: - tb.copyrows("tmp1.g") - tb.close() + msmd.close() + elif solint == "inf_fb3": + include_targets = include_targets.split(',') + include_scans = include_scans * len(include_targets) + else: + include_targets = [include_targets] - os.system("rm -rf tmp0.g") + selfcal_library[vis][solint]["include_scans"] = include_scans + selfcal_library[vis][solint]["include_targets"] = include_targets + print(solint,'Include scans: ', include_scans) + print(solint,'Include targets: ', include_targets) + print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) + for incl_scans, incl_targets in zip(include_scans, include_targets): + if solint == "inf_fb3": + gaincal_preapply_gaintable = [selfcal_plan[vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][incl_targets]] - tb.open("tmp1.g") - subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") - copyt = subt.copy(destination_table, deep=True) - tb.close() - subt.close() - copyt.close() + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + print(incl_targets) + print(incl_scans) + print(vis,solint,gc_mode) + print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) + gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] - os.system("rm -rf tmp1.g") + if 'spw' in gaincal_combine: + if selfcal_library['spws_set'][vis].ndim == 1: + nspw_sets=1 + else: + nspw_sets=selfcal_library['spws_set'][vis].shape[0] + else: #only necessary to loop over gain cal when in inf_EB to avoid inf_EB solving for all spws + nspw_sets=1 - # Remove all of the scans that failed the triage above. - tb.open(destination_table, nomodify=False) - scans = tb.getcol("SCAN_NUMBER") + for i in range(nspw_sets): # run gaincal on each spw set to handle spectral scans + if 'spw' in gaincal_combine: + if nspw_sets == 1 and selfcal_library['spws_set'][vis].ndim == 1: + spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis].tolist()) + else: + spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis][i].tolist()) + else: + spwselect=selfcal_library[vis]['spws'] + gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' + print('prior to gaincal',gaintable_name, gc_mode) - bad_scans = np.repeat(True, scans.size) - for scan in include_scans[0].split(","): - bad_scans[scans == int(scan)] = False + if gc_mode != 'per_bb': + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) + else: + for baseband in selfcal_library[vis]['baseband'].keys(): + spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - tb.removerows(rownrs=np.where(bad_scans)[0]) - tb.flush() - tb.close() - else: - # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. - gaincal_solmode="" - if selfcal_library['obstype'] == 'mosaic': - msmd.open(vis) - include_targets = [] - remove = [] - for incl_scan in include_scans: - scan_targets = [] - for fid in [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ - np.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))] if incl_scan == '' else \ - np.intersect1d(msmd.fieldsforscans(np.array(incl_scan.split(",")).astype(int)), \ - [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ - numpy.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))]): - # Note: because of the msmd above getting actual fids from the MS, we just need to append fid below. - scan_targets.append(fid) - - if len(scan_targets) > 0: - include_targets.append(','.join(np.array(scan_targets).astype(str))) - else: - remove.append(incl_scan) + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name - for incl_scan in remove: - include_scans.remove(incl_scan) + # restricted gaincal table comparisons to only inf_EB prior to changes + # commenting because we want to do comparisons for other solints as well + #if 'inf_EB' not in solint: + # break - msmd.close() - else: - include_targets = [include_targets] - - selfcal_library[vis][solint]["include_scans"] = include_scans - selfcal_library[vis][solint]["include_targets"] = include_targets - print(solint,'Include scans: ', include_scans) - print(solint,'Include targets: ', include_targets) - print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) - for incl_scans, incl_targets in zip(include_scans, include_targets): - for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - print(incl_targets) - print(incl_scans) - print(vis,solint,gc_mode) - print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) - gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] - - if 'spw' in gaincal_combine: - if selfcal_library['spws_set'][vis].ndim == 1: - nspw_sets=1 - else: - nspw_sets=selfcal_library['spws_set'][vis].shape[0] - else: #only necessary to loop over gain cal when in inf_EB to avoid inf_EB solving for all spws - nspw_sets=1 - for i in range(nspw_sets): # run gaincal on each spw set to handle spectral scans - if 'spw' in gaincal_combine: - if nspw_sets == 1 and selfcal_library['spws_set'][vis].ndim == 1: - spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis].tolist()) - else: - spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis][i].tolist()) - else: - spwselect=selfcal_library[vis]['spws'] - gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - print('prior to gaincal',gaintable_name, gc_mode) - if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': - print('in cocal if statement') - for incl_target in include_targets[0].split(','): - #vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') - vis_to_gaincal = vis - print('Vis to gaincal: ',vis_to_gaincal) - if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - else: - for baseband in selfcal_library[vis]['baseband'].keys(): - spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - else: - if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - else: - for baseband in selfcal_library[vis]['baseband'].keys(): - spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - - selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name - - # restricted gaincal table comparisons to only inf_EB prior to changes - # commenting because we want to do comparisons for other solints as well - #if 'inf_EB' not in solint: - # break gaintable_prefix=sani_target+'_'+vis+'_'+band+'_' # assume that if there is only one mode to attempt, that it is combinespw and don't bother checking. if 'delay' not in solint: @@ -457,6 +245,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so selfcal_library[fid][vis][solint]['applycal_mode']=selfcal_plan['applycal_mode'][iteration]+'' selfcal_library[fid][vis][solint]['applycal_interpolate']=applycal_interpolate.copy() selfcal_library[fid][vis][solint]['solmode']=selfcal_plan['solmode'][iteration]+'' + selfcal_plan[vis]['solint_settings'][solint]['accepted_gaintable']=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+preferred_mode+'.g' if 'combinespw' not in preferred_mode: # preapply all non spwcombine gain tables selfcal_plan[vis]['solint_settings'][solint]['preapply_this_gaintable']=True diff --git a/auto_selfcal/mosaic_helpers.py b/auto_selfcal/mosaic_helpers.py index 93543f2a..1db762f6 100644 --- a/auto_selfcal/mosaic_helpers.py +++ b/auto_selfcal/mosaic_helpers.py @@ -151,3 +151,121 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter return new_fields_to_selfcal + +def scan_inf_scan_combine(selfcal_library, vis, target, gaincalibrator_dict, guess_scan_combine=False): + if len(gaincalibrator_dict[vis]) > 0: + print("Determining scan_inf from calibrator scans in full MS") + scans = [] + intents = [] + times = [] + for t in gaincalibrator_dict[vis].keys(): + scans += [gaincalibrator_dict[vis][t]["scans"]] + intents += [np.repeat(gaincalibrator_dict[vis][t]["intent"],gaincalibrator_dict[vis][t]["scans"].size)] + times += [gaincalibrator_dict[vis][t]["times"]] + + times = np.concatenate(times) + order = np.argsort(times) + times = times[order] + + scans = np.concatenate(scans)[order] + intents = np.concatenate(intents)[order] + + is_gaincalibrator = intents == "phase" + scans = scans[is_gaincalibrator] + + msmd.open(vis) + if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': + scan_ids_for_target = msmd.scansforfield(target) + elif 'VLA' in selfcal_library['telescope']: + scan_ids_for_target=np.array([],dtype=int) + for fid in selfcal_library['sub-fields']: + if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): + field_id=selfcal_library['sub-fields-fid_map'][vis][fid] + scan_ids_for_target=np.append(scan_ids_for_target,msmd.scansforfield(field_id)) + + scan_ids_for_target.sort() # sort scans since they will be out of order + include_scans = [] + for iscan in range(scans.size-1): + scan_group = np.intersect1d(scan_ids_for_target, + np.array(list(range(scans[iscan]+1, scans[iscan+1])))).astype(str) + if scan_group.size > 0: + include_scans.append(",".join(scan_group)) + # PIPE-2741: if there are any scans before the first gain calibrator scan or after the last gain calibrator scan, catch them. + if scans.size > 0: + extra_scans = scan_ids_for_target[scan_ids_for_target > max(scans)] + if extra_scans.size > 0: + include_scans.append(','.join(extra_scans.astype(str))) + extra_scans = scan_ids_for_target[scan_ids_for_target < min(scans)] + if extra_scans.size > 0: + include_scans.append(','.join(extra_scans.astype(str))) + msmd.close() + elif guess_scan_combine: + print("Determining scan_inf from guessing where the calibrator scans were") + msmd.open(vis) + include_scans = [] + + #to guess at scan_inf combination for VLA look for breaks in the consecutive + #scan numbers and assume that the break is due to a calibrator scan + #Fetch scans for scan inf by collecting the field ids and running msmd.scansforfield + if 'VLA' in selfcal_library['telescope']: + scans=np.array([],dtype=int) + for fid in selfcal_library['sub-fields']: + if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): + field_id=selfcal_library['sub-fields-fid_map'][vis][fid] + scans=np.append(scans,msmd.scansforfield(field_id)) + + scans.sort() # sort scans since they will be out of order + scan_group='' + for iscan in range(scans.size): + if len(include_scans) > 0: + if str(scans[iscan]) in include_scans[-1]: + continue + if scan_group == '': + scan_group = str(int(scans[iscan])) + + if iscan < scans.size-1: + if scans[iscan+1] == scans[iscan]+1: + scan_group += ","+str(int(scans[iscan+1])) + else: + include_scans.append(scan_group) + scan_group='' + else: #write out the last scan group to include_scans + if scan_group != '': + include_scans.append(scan_group) + scan_group='' + + #guess scan_inf combination by getting all the scans for targets and do a simple grouping + if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': + scans = msmd.scansforfield(target) + + for iscan in range(scans.size): + if len(include_scans) > 0: + if str(scans[iscan]) in include_scans[-1]: + continue + + scan_group = str(scans[iscan]) + + if iscan < scans.size-1: + if msmd.fieldsforscan(scans[iscan+1]).size < msmd.fieldsforscan(scans[iscan]).size/3: + scan_group += ","+str(scans[iscan+1]) + + include_scans.append(scan_group) + + msmd.close() + else: + print("Not guessing where calibration scans are and justincluding all scans") + msmd.open(vis) + if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': + include_scans = [str(scan) for scan in msmd.scansforfield(target)] + elif 'VLA' in selfcal_library['telescope']: + scans=np.array([],dtype=int) + for fid in selfcal_library['sub-fields']: + if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): + field_id=selfcal_library['sub-fields-fid_map'][vis][fid] + scans=np.append(scans,msmd.scansforfield(field_id)) + + scans.sort() # sort scans since they will be out of order + include_scans = [str(scan) for scan in scans] + msmd.close() + + return include_scans \ No newline at end of file diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py new file mode 100644 index 00000000..bd3b872e --- /dev/null +++ b/auto_selfcal/prepare_cocal.py @@ -0,0 +1,143 @@ +from casatasks import flagmanager, applycal, clearcal +from .prepare_selfcal import plan_selfcal_per_solint +import numpy as np +import copy +import os + +from .selfcal_helpers import sanitize_string + +def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_gaintype): + for target in selfcal_library: + for band in selfcal_library[target]: + for vis in selfcal_library[target][band]['vislist']: + if not os.path.exists(vis+'.flagversions/flags.fb_selfcal_starting_flags'): + flagmanager(vis=vis,mode='save',versionname='fb_selfcal_starting_flags') + else: + flagmanager(vis=vis,mode='restore',versionname='fb_selfcal_starting_flags') + + ## + ## For sources that self-calibration failed, try to use the inf_EB and the inf solutions from the sources that + ## were successful. + + for target in selfcal_library.keys(): + for band in selfcal_library[target].keys(): + print(target, selfcal_library[target][band]["final_solint"]) + + inf_EB_fields = {} + inf_fields = {} + fallback_fields = {} + calibrators = {} + + # Collect the potential targets and calibrators + for target in selfcal_library: + for band in selfcal_library[target]: + if band not in inf_EB_fields: + # Initialize the lists for this band. + inf_EB_fields[band] = [] + inf_fields[band] = [] + fallback_fields[band] = [] + + if selfcal_library[target][band]['SC_success'] and 'fb' not in selfcal_library[target][band]['final_solint']: + inf_EB_fields[band].append(target) + if selfcal_library[target][band]['final_solint'] != 'inf_EB' or \ + (selfcal_library[target][band]['final_solint'] == 'inf_EB' and + 'inf' not in selfcal_plan[target][band]['solints']): + inf_fields[band].append(target) + else: + fallback_fields[band].append(target) + else: + fallback_fields[band].append(target) + + + if len(fallback_fields[band]) > 0: + for target in selfcal_library: + for band in selfcal_library[target]: + # Update the relevant lists if we are going to do a fallback mode. + selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] + selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] + selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] + + plan_selfcal_per_solint(selfcal_library, + selfcal_plan, + optimize_spw_combine=False, + solints=["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"]) + + for target in selfcal_library: + for band in selfcal_library[target]: + for vis in selfcal_library[target][band]['vislist']: + """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"] = {} + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf"]) + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]) + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"])""" + + selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["preapply_solints"] = [] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["preapply_solints"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["preapply_solints"] = ["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_solints"] = ["inf_EB"] + + """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["modes_to_attempt"] = ["combinespw","combinespwpol"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["modes_to_attempt"] = ["combinespw"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["modes_to_attempt"] = ["combinespw"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["modes_to_attempt"] = ["combinespw"]""" + + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} + for cal_target in inf_fields[band]: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf']['accepted_gaintable'] + + selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] + selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] + + calibrators[band] = [inf_EB_fields[band], inf_fields[band], inf_fields[band], inf_fields[band]] + + selfcal_library[target][band]["nsigma"] = np.concatenate((selfcal_library[target][band]["nsigma"],[selfcal_library[target][band]["nsigma"][0], \ + selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1]])) + + print(inf_EB_fields) + print(inf_fields) + print(fallback_fields) + + ## + ## Reset the inf_EB informational dictionaries. + ## + + for target in selfcal_library: + for band in selfcal_library[target].keys(): + # If the target had a successful inf_EB solution, no need to reset. + if target in inf_EB_fields[band]: + continue + + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']=inf_EB_gaincal_combine #'scan' + if selfcal_library[target][band]['obstype']=='mosaic': + selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']+=',field' + selfcal_plan[target][band][vis]['inf_EB_gaintype']=inf_EB_gaintype #G + selfcal_plan[target][band][vis]['inf_EB_fallback_mode']='' #'scan' + + ## The below sets the calibrations back to what they were prior to starting the fallback mode. It should not be needed + ## for the final version of the codue, but is used for testing. + + + for target in selfcal_library: + sani_target=sanitize_string(target) + for band in selfcal_library[target].keys(): + if target not in fallback_fields[band]: + continue + if 'gaintable_final' in selfcal_library[target][band]['vislist'][0]: + print('****************Reapplying previous solint solutions*************') + for vis in selfcal_library[target][band]['vislist']: + print('****************Applying '+str(selfcal_library[target][band][vis]['gaintable_final'])+' to '+target+' '+band+'*************') + ## NOTE: should this be selfcal_starting_flags instead of fb_selfcal_starting_flags ??? + flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) + applycal(vis=vis,\ + gaintable=selfcal_library[target][band][vis]['gaintable_final'],\ + interp=selfcal_library[target][band][vis]['applycal_interpolate_final'],\ + calwt=True,spwmap=selfcal_library[target][band][vis]['spwmap_final'],\ + applymode=selfcal_library[target][band][vis]['applycal_mode_final'],\ + field=target,spw=selfcal_library[target][band][vis]['spws']) + else: + print('****************Removing all calibrations for '+target+' '+band+'**************') + for vis in selfcal_library[target][band]['vislist']: + flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) + clearcal(vis=vis,field=target,spw=selfcal_library[target][band][vis]['spws']) + + return fallback_fields, calibrators \ No newline at end of file diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index b6616315..f36fd497 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -534,7 +534,7 @@ def default(self, obj): return selfcal_library, selfcal_plan, gaincalibrator_dict -def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=True): +def plan_selfcal_per_solint(selfcal_library, selfcal_plan, optimize_spw_combine=True, solints=None): # there are some extra keys in this dictionary that stem from how my thinking was orginally and how it evolved # the current philosophy is that for each solint it will specify how to apply each gain table, and if it should # be pre-applied for gaincal solves. Then in gaincal wrapper, the parameters for preapplying all tables and for applying @@ -559,8 +559,13 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T if selfcal_library[target][band][vis]['baseband'][baseband]['nspws']> maxspws_per_bb: maxspws_per_bb=selfcal_library[target][band][vis]['baseband'][baseband]['nspws']+0.0 - selfcal_plan[target][band][vis]['solint_settings']={} - for solint in selfcal_plan[target][band]['solints']: + if solints is not None: + use_solints = solints + else: + selfcal_plan[target][band][vis]['solint_settings']={} + use_solints = selfcal_plan[target][band]['solints'] + + for solint in use_solints: gaincal_combine='' filename_append='' selfcal_plan[target][band][vis]['solint_settings'][solint]={} @@ -582,8 +587,7 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T selfcal_plan[target][band][vis]['solint_settings'][solint]['final_mode']='' selfcal_plan[target][band][vis]['solint_settings'][solint]['accepted_gaintable']='' selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=[] - min_SNR_spw=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_spw'][solint]) - min_SNR_bb=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_bb'][solint]) + print('Nspws: {}, spws per BB: {}, basebands: {}'.format(nspws,maxspws_per_bb,n_basebands)) if selfcal_library[target][band]['telescope'] == 'VLBA' and 'delay' in solint and maxspws_per_bb > 1.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') @@ -591,11 +595,14 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('combinespw') if 'delay' in solint and n_basebands > 1: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') - if solint == 'inf_EB': + if "inf_EB" in solint: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('combinespwpol') - selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True + if solint == 'inf_EB': + selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True print('solint',solint,'N basebands: ',n_basebands, 'modes to attempt: ',selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']) - if 'spw' not in selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']: + if 'spw' not in selfcal_plan[target][band][vis]['inf_EB_gaincal_combine'] and 'fb' not in solint: + min_SNR_spw=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_spw'][solint]) + min_SNR_bb=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_bb'][solint]) if min_SNR_spw > 2.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval @@ -603,12 +610,15 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval - if '_ap' in solint: - selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='ap' - else: - selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='p' - if solint != 'inf_EB' and optimize_spw_combine==False: - selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=['combinespw'] + + if '_ap' in solint: + selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='ap' + else: + selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='p' + + if 'inf_EB' not in solint and optimize_spw_combine==False: + selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=['combinespw'] + for mode in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: gaincal_combine='' if mode =='combinespw': @@ -627,11 +637,12 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T gaincal_combine='spw' filename_append='per_bb' selfcal_plan[target][band][vis]['solint_settings'][solint]['spwmap_for_mode']['per_bb']=selfcal_library[target][band][vis]['baseband_spwmap'] - if solint in ['inf_EB','inf_EB_delay','scan_inf','300s_ap']: + if solint in ['inf_EB','inf_EB_delay','scan_inf','300s_ap','inf_EB_fb']: if gaincal_combine!='': gaincal_combine+=',' gaincal_combine+='scan' - if solint in ['inf_EB','inf_EB_delay','scan_inf'] and selfcal_library[target][band]['obstype'] == 'mosaic': + if (solint in ['inf_EB','inf_EB_delay','scan_inf'] and selfcal_library[target][band]['obstype'] == 'mosaic') or \ + solint == "inf_EB_fb": gaincal_combine+=',field' selfcal_plan[target][band][vis]['solint_settings'][solint]['gaincal_combine'][mode]=gaincal_combine selfcal_plan[target][band][vis]['solint_settings'][solint]['filename_append'][mode]=filename_append diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index ddf3d06f..3eb1d7e4 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -3058,6 +3058,7 @@ def plot_ants_flagging_colored(filename,vis,gaintable): def get_flagged_solns_per_ant_from_dict(gc_dict_list,spwlist,vis): msmd.open(vis) antids=[] + print("len(gc_dict_list)", len(gc_dict_list)) for ant in [idant for idant in gc_dict_list[0]['solvestats']['spw'+str(spwlist[0])].keys() if idant.startswith('ant')]: antids.append(int(ant.replace('ant',''))) antids.sort() @@ -3124,6 +3125,8 @@ def plot_ants_flagging_colored_from_dict(filename,selfcal_library,selfcal_plan,s spwlist_pass=spwlist_bb.copy() + print(selfcal_plan.keys()) + print("plot_ants_flagging_colored_from_dict", solint, final_mode, len(selfcal_plan['solint_settings'][solint]['gaincal_return_dict'][final_mode])) names, offset_x, offset_y, apriori_flagged, nflagged, nunflagged, ntotal, fracflagged, nflagged_non_apriori, ntotal_non_apriori_flagged, fracflagged_non_apriori=get_flagged_solns_per_ant_from_dict(selfcal_plan['solint_settings'][solint]['gaincal_return_dict'][final_mode],spwlist_pass,vis) fracflagged=fracflagged_non_apriori print(fracflagged) @@ -3628,8 +3631,11 @@ def get_gaincalmode_flagging_stats(selfcal_library,selfcal_plan,vis,gaintable_pr selfcal_plan[vis]['solint_settings'][solint]['nflags_apriori'][mode],selfcal_plan[vis]['solint_settings'][solint]['nflags'][mode],selfcal_plan[vis]['solint_settings'][solint]['nunflagged'][mode],selfcal_plan[vis]['solint_settings'][solint]['ntotal'][mode],selfcal_plan[vis]['solint_settings'][solint]['fracflagged'][mode],selfcal_plan[vis]['solint_settings'][solint]['nflags_non_apriori'][mode],selfcal_plan[vis]['solint_settings'][solint]['ntotal_non_apriori'][mode],selfcal_plan[vis]['solint_settings'][solint]['fracflagged_non_apriori'][mode]=get_gaintable_flagging_stats(selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode],spwlist_bb) else: baseband_scale=1.0 - if solint == 'inf_EB': + if 'inf_EB' in solint: n_solutions=1.0 + elif 'inf_EB_fb' in selfcal_plan[vis]['solint_settings'].keys(): + n_antennas=selfcal_plan[vis]['solint_settings']['inf_EB_fb']['ntotal_non_apriori'][coarsest_mode][0]/selfcal_plan[vis]['solint_settings'][solint]['polscale'][mode] + n_solutions=(selfcal_plan[vis]['solint_settings'][solint]['nflags_non_apriori'][coarsest_mode][0]+selfcal_plan[vis]['solint_settings'][solint]['nunflagged'][coarsest_mode][0])/n_antennas elif 'inf_EB' in selfcal_plan[vis]['solint_settings'].keys(): n_antennas=selfcal_plan[vis]['solint_settings']['inf_EB']['ntotal_non_apriori'][coarsest_mode][0]/selfcal_plan[vis]['solint_settings'][solint]['polscale'][mode] n_solutions=(selfcal_plan[vis]['solint_settings'][solint]['nflags_non_apriori'][coarsest_mode][0]+selfcal_plan[vis]['solint_settings'][solint]['nunflagged'][coarsest_mode][0])/n_antennas From 6697b711f1dd99ab4b4e0595980f7581c9f55596 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 14 May 2026 16:39:43 -0400 Subject: [PATCH 35/43] Preapply inf_EB for inf_fb3, not inf. Also, delete some unused, commented out code. --- auto_selfcal/prepare_cocal.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py index bd3b872e..40372bac 100644 --- a/auto_selfcal/prepare_cocal.py +++ b/auto_selfcal/prepare_cocal.py @@ -65,24 +65,14 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ for target in selfcal_library: for band in selfcal_library[target]: for vis in selfcal_library[target][band]['vislist']: - """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"] = {} - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf"]) - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]) - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"])""" - selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["preapply_solints"] = [] selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["preapply_solints"] = ["inf_EB_fb"] selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["preapply_solints"] = ["inf_EB"] selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_solints"] = ["inf_EB"] - """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["modes_to_attempt"] = ["combinespw","combinespwpol"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["modes_to_attempt"] = ["combinespw"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["modes_to_attempt"] = ["combinespw"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["modes_to_attempt"] = ["combinespw"]""" - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} for cal_target in inf_fields[band]: - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf']['accepted_gaintable'] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf_EB']['accepted_gaintable'] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] From f6abd08ae579ae3dce650b19ad892e3b166c170f Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Fri, 15 May 2026 10:22:12 -0400 Subject: [PATCH 36/43] We need a manual override of using the same pre-apply gaintable for gaincal as is used for applycal when doing the inf_fb* modes, because in those cases the same table might not be used for both. --- auto_selfcal/gaincal_wrapper.py | 11 ++++++++++- auto_selfcal/prepare_cocal.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index fae8cc42..96401eb9 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -58,9 +58,18 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) gaincal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) + + # If we manually specify that applycal should use a different solint reference than gaincal (cocal), change + # to that solint here. + if "applycal_solint" in selfcal_plan[vis]['solint_settings'][incl_solint]: + incl_solint = selfcal_plan[vis]['solint_settings'][incl_solint]["applycal_solint"] + applycal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) applycal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) - applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) + if solint == "inf_fb3": + applycal_gaintable.append(selfcal_plan[vis]['solint_settings']["inf_EB_fb"]['accepted_gaintable']) + else: + applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) if solint != 'inf_EB': diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py index 40372bac..2ae6f73c 100644 --- a/auto_selfcal/prepare_cocal.py +++ b/auto_selfcal/prepare_cocal.py @@ -70,6 +70,15 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["preapply_solints"] = ["inf_EB"] selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_solints"] = ["inf_EB"] + if selfcal_library[target][band]['SC_success']: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["applycal_solint"] = ["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["applycal_solint"] = ["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["applycal_solint"] = ["inf_EB"] + else: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["applycal_solint"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["applycal_solint"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["applycal_solint"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} for cal_target in inf_fields[band]: selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf_EB']['accepted_gaintable'] From 2d97ecf5de2295249b6cee69748c954d2af4c631 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 15 May 2026 10:56:12 -0400 Subject: [PATCH 37/43] add sanitize_string() to target and cal_target when doing a .replace on a filename --- auto_selfcal/prepare_cocal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py index 40372bac..dca8fa7d 100644 --- a/auto_selfcal/prepare_cocal.py +++ b/auto_selfcal/prepare_cocal.py @@ -72,7 +72,7 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} for cal_target in inf_fields[band]: - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf_EB']['accepted_gaintable'] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(sanitize_string(target), sanitize_string(cal_target))]['solint_settings']['inf_EB']['accepted_gaintable'] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] @@ -130,4 +130,4 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) clearcal(vis=vis,field=target,spw=selfcal_library[target][band][vis]['spws']) - return fallback_fields, calibrators \ No newline at end of file + return fallback_fields, calibrators From 8337db497be4b4e95571c5c6961f09da50ff303f Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 15 May 2026 16:09:39 -0400 Subject: [PATCH 38/43] write selfcal_plan.pickle before starting selfcal and ensure single_spw data have a solution populated for all solints --- auto_selfcal/auto_selfcal.py | 4 +++- auto_selfcal/prepare_selfcal.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index 1db79af9..4c386434 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -632,6 +632,7 @@ def auto_selfcal( with open('selfcal_library.pickle', 'wb') as handle: pickle.dump(selfcal_library, handle, protocol=pickle.HIGHEST_PROTOCOL) + import json class NpEncoder(json.JSONEncoder): @@ -711,7 +712,8 @@ def default(self, obj): with open('selfcal_library.pickle', 'wb') as handle: pickle.dump(selfcal_library, handle, protocol=pickle.HIGHEST_PROTOCOL) - + with open('selfcal_plan.pickle', 'wb') as handle: + pickle.dump(selfcal_plan, handle, protocol=pickle.HIGHEST_PROTOCOL) ## ## Begin Self-cal loops diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index f36fd497..2cc058ef 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -606,6 +606,8 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan, optimize_spw_combine= if min_SNR_spw > 2.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval + elif n_spws == 1: # make sure we populate when only single spectral window, otherwise can errorr + selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') if min_SNR_bb > 2.0 and maxspws_per_bb > 1.0 and selfcal_library[target][band]['spectral_scan']==False and n_basebands > 1: # only do the per baseband solutions if there are more than 1 spw and more than 1 baseband if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') From 1d8cab56ef887ed891f77743e370d18d978caf6d Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Mon, 18 May 2026 16:49:22 -0400 Subject: [PATCH 39/43] Add dataset to test cocal online. --- auto_selfcal/tests/test_auto_selfcal.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 5f0aafcb..26f63104 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -87,7 +87,8 @@ def test_benchmark(tmp_path, dataset): pytest.param("2018.1.01284.S_HOPS-384.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCvDRlo5TabTZ8qoP-tscZnAdJZOuCKQew3ewIH5bZzZF0?e=XWOGxH&download=1', id="2018.1.01284.S_HOPS-384"), pytest.param("Band8-7m-2.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAnFUlx_SR0SLdj_vX8MVUWAbMu9BveHCsb9NB07-OD3bo?e=TNBTXq&download=1', id="Band8-7m-2"), pytest.param("M82-C-conf-C-band_small.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCT-CkbAW7YT5LIev5CrDSOAZt5uC_tUIReMwlA14Tu9y4?e=2zF126&download=1', id="M82-C-conf-C-band_small"), - pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic') + pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic'), + pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=KHkjlK', id='Band8-7m-cocal') ] ) def test_on_github(tmp_path, request, zip_file, link): @@ -100,7 +101,7 @@ def test_on_github(tmp_path, request, zip_file, link): os.system(f'tar xf {zip_file}') os.system(f'rm -rf {zip_file}') - auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True) + auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True, allow_cocal=True) os.system('rm -rf *.ms*') # Delete MS files as space is limited on GitHub. From 4c5f8d6eeca94108d8d891e40edbf995965ae80f Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 19 May 2026 15:57:50 +0000 Subject: [PATCH 40/43] Fix 'n_spws' => 'nspws' bug; update cocal test link --- auto_selfcal/prepare_selfcal.py | 2 +- auto_selfcal/tests/test_auto_selfcal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index 2cc058ef..22226f12 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -606,7 +606,7 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan, optimize_spw_combine= if min_SNR_spw > 2.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval - elif n_spws == 1: # make sure we populate when only single spectral window, otherwise can errorr + elif nspws == 1: # make sure we populate when only single spectral window, otherwise can errorr selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') if min_SNR_bb > 2.0 and maxspws_per_bb > 1.0 and selfcal_library[target][band]['spectral_scan']==False and n_basebands > 1: # only do the per baseband solutions if there are more than 1 spw and more than 1 baseband if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 6b1a8be8..cda0e8ad 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -88,7 +88,7 @@ def test_benchmark(tmp_path, dataset): pytest.param("Band8-7m-2.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAnFUlx_SR0SLdj_vX8MVUWAbMu9BveHCsb9NB07-OD3bo?e=TNBTXq&download=1', id="Band8-7m-2"), pytest.param("M82-C-conf-C-band_small.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCT-CkbAW7YT5LIev5CrDSOAZt5uC_tUIReMwlA14Tu9y4?e=2zF126&download=1', id="M82-C-conf-C-band_small"), pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic'), - pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=KHkjlK', id='Band8-7m-cocal'), + pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=M7gnSx&download=1', id='Band8-7m-cocal'), pytest.param("2019.1.00691.S_SB.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAoeNLrwb1aSrWCfo8ekE6NAXVJ4Qlpps0gdAeY1U9Axn0?e=MMXCaT&download=1', id='2019.1.00691.S_SB') ] ) From e7d5d7c5aff7f58dd908242fa95cc8d2d2d50fe5 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 19 May 2026 16:45:55 -0500 Subject: [PATCH 41/43] Add reference values/weblog for online cocal test --- auto_selfcal/tests/test_auto_selfcal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index cda0e8ad..e19b4719 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -88,7 +88,7 @@ def test_benchmark(tmp_path, dataset): pytest.param("Band8-7m-2.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAnFUlx_SR0SLdj_vX8MVUWAbMu9BveHCsb9NB07-OD3bo?e=TNBTXq&download=1', id="Band8-7m-2"), pytest.param("M82-C-conf-C-band_small.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCT-CkbAW7YT5LIev5CrDSOAZt5uC_tUIReMwlA14Tu9y4?e=2zF126&download=1', id="M82-C-conf-C-band_small"), pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic'), - pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=M7gnSx&download=1', id='Band8-7m-cocal'), + pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=rX9LaN&download=1', id='Band8-7m-cocal'), pytest.param("2019.1.00691.S_SB.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAoeNLrwb1aSrWCfo8ekE6NAXVJ4Qlpps0gdAeY1U9Axn0?e=MMXCaT&download=1', id='2019.1.00691.S_SB') ] ) From 833ed1756d57ae47a33c3a8d8b52bec2a9281714 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 22 May 2026 14:02:44 -0400 Subject: [PATCH 42/43] re-enable cocal on github tests --- auto_selfcal/tests/test_auto_selfcal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 025083c8..89102473 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -122,7 +122,7 @@ def test_on_github(tmp_path, request, zip_file, link): auto_selfcal(iscalibrator=True, refant='FD,NL,PT', do_delay_cal=True, shorter_amp_solints=True, targets='J1154+6022', applytargets='J1203+6031', imsize=640, cell='0.0002arcsec') else: - auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True, usermask=usermask, usermodel=usermodel) + auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True, allow_cocal=True, usermask=usermask, usermodel=usermodel) os.system('rm -rf *.ms*') # Delete MS files as space is limited on GitHub. From 9eb76be714f158f41c3bacd17b374910255b6a43 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 2 Jun 2026 21:29:28 +0000 Subject: [PATCH 43/43] Remove print statements that are superseded by updates on this branch. --- auto_selfcal/run_selfcal.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 8a8f9812..fa2bf8f9 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -82,8 +82,6 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ for vis in selfcal_library['vislist']: print('selfcal_library[vis]["sub-fields-to-selfcal"] = ', selfcal_library[vis]['sub-fields-to-selfcal']) while iteration < len(selfcal_plan['solints']): - print(selfcal_plan['solints']) - print(selfcal_plan['solint_interval'][iteration]) vislist=[vis for vis in selfcal_library['vislist'] if selfcal_plan['solints'][iteration] in selfcal_plan[vis]['solint_settings']] print("Solving for solint="+selfcal_plan['solints'][iteration]+' with intervals:')