From 8c1438df813b8d9555827c7992a3cd30e0d8a7c2 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 09:23:46 -0400 Subject: [PATCH 01/24] fixes to get auto_selfcal to run cleanly with allow_cocal=True --- auto_selfcal/auto_selfcal.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index a475e76c..e5993cbe 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -751,7 +751,7 @@ def default(self, obj): inf_fields = {} fallback_fields = {} calibrators = {} - for band in bands: + for band in vis_for_targets[target]['Bands']: # Initialize the lists for this band. inf_EB_fields[band] = [] inf_fields[band] = [] @@ -773,7 +773,8 @@ def default(self, obj): selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] - applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] + selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] + #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] calibrators[band] = [inf_EB_fields[band], inf_fields[band], inf_fields[band], inf_fields[band]] selfcal_library[target][band]["nsigma"] = np.concatenate((selfcal_library[target][band]["nsigma"],[selfcal_library[target][band]["nsigma"][0], \ selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1]])) @@ -787,7 +788,7 @@ def default(self, obj): ## for target in selfcal_library: - for band in solint_snr[target].keys(): + for band in selfcal_library[target].keys(): # If the target had a successful inf_EB solution, no need to reset. if target in inf_EB_fields[band]: continue @@ -812,7 +813,7 @@ def default(self, obj): for band in selfcal_library[target].keys(): if target not in fallback_fields[band]: continue - if 'gaintable_final' in selfcal_library[target][band][vislist[0]]: + if 'gaintable_final' in selfcal_library[target][band]['vislist'][0]: print('****************Reapplying previous solint solutions*************') for vis in selfcal_library[target][band]['vislist']: print('****************Applying '+str(selfcal_library[target][band][vis]['gaintable_final'])+' to '+target+' '+band+'*************') From 9e9712f117177f4137d20e401ef0d5c999210608 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 10:02:12 -0400 Subject: [PATCH 02/24] changes to get cocal to look at original targets ms for possible calibrators --- auto_selfcal/run_selfcal.py | 2 +- auto_selfcal/selfcal_helpers.py | 20 ++++++++++++++++---- bin/auto_selfcal.py | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index f0b9165e..bb86f9ec 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -48,7 +48,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if mode == "cocal": # Check whether there are suitable calibrators, otherwise skip this target/band. - include_targets, include_scans = triage_calibrators(vislist[0], target, calibrators[band][0]) + include_targets, include_scans = triage_calibrators(vislist[0], target, band, calibrators[band][0]) if include_targets == "": print("No suitable calibrators found, skipping "+target) selfcal_library['Stop_Reason'] += '; No suitable co-calibrators' diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 2a7f6da5..2e302416 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -516,7 +516,7 @@ def tclean_wrapper(selfcal_library, imagename, band, scales=[0], smallscalebias uvrange=selfcal_library['uvrange'], reffreq = reffreq, threshold=threshold, - parallel=parallel, + parallel=False, phasecenter=phasecenter,spw=spws_per_vis,wprojplanes=wprojplanes) elif savemodel=='modelcolumn' and selfcal_library['usermodel'] !='': @@ -4643,11 +4643,23 @@ def unflag_failed_antennas(vis, caltable, gaincal_return, telescope, flagged_fra -def triage_calibrators(vis, target, potential_calibrators, max_distance=10.0, max_time=600.): +def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10.0, max_time=600.): gaincalibrator_dict = {} + # account for possible different naming conventions in original visibilities + sani_target=sanitize_string(target) + orig_vis='' + if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): + orig_vis=os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) + elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): + orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) - if os.path.exists(vis.replace("_target.selfcal.ms",".ms")): - msmd.open(vis.replace("_target.selfcal.ms",".ms")) + # original visibilities, with all sources have a different filename now + + orig_targets_vis=vis.replace(".selfcal.ms",".ms").replace(sani_target+'_'+band+'_') + vis=orig_targets_vis + + if orig_vis !='': + msmd.open(orig_vis) for field in msmd.fieldsforintent("*CALIBRATE_PHASE*"): scans_for_field = msmd.scansforfield(field) diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index b6fbde6c..79976d9a 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, parallel=parallel) +auto_selfcal(vislist, allow_cocal=True,parallel=parallel) From e5cf7d0d37c1e3eb4e8d91d568ae6e22eca07469 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 10:06:51 -0400 Subject: [PATCH 03/24] fix missing argument for a replace() call --- auto_selfcal/selfcal_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 2e302416..38827cba 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -4655,7 +4655,7 @@ def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10 # original visibilities, with all sources have a different filename now - orig_targets_vis=vis.replace(".selfcal.ms",".ms").replace(sani_target+'_'+band+'_') + orig_targets_vis=vis.replace(".selfcal.ms",".ms").replace(sani_target+'_'+band+'_','') vis=orig_targets_vis if orig_vis !='': From 05e0202b4f34ce586abf5372ff2dcc5b226d731b Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 12:10:40 -0400 Subject: [PATCH 04/24] fix another missing second argument on a replace() --- auto_selfcal/selfcal_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 38827cba..16c8bcbf 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -4648,7 +4648,7 @@ def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10 # account for possible different naming conventions in original visibilities sani_target=sanitize_string(target) orig_vis='' - if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): + if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')): orig_vis=os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) From f4db2a1b54cab5dc613e723528ef5b03cca78fbe Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 14:16:14 -0400 Subject: [PATCH 05/24] missing additions to selfcal_plan solint_interval --- auto_selfcal/auto_selfcal.py | 1 + auto_selfcal/gaincal_wrapper.py | 66 +++++++++++++++++++++++---------- auto_selfcal/run_selfcal.py | 3 +- auto_selfcal/selfcal_helpers.py | 4 +- bin/auto_selfcal.py | 2 +- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index e5993cbe..be8f4464 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -772,6 +772,7 @@ def default(self, obj): if len(fallback_fields[band]) > 0: selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] + selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index dab9e4cf..c6fff6d6 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -101,7 +101,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so # Check which targets are acceptable to use as calibrators. targets = calibrators[band][iteration - len(selfcal_plan['solints'])] - include_targets, include_scans = triage_calibrators(vis, target, targets) + include_targets, include_scans = triage_calibrators(vis, target, band, targets) else: #include_targets = str(selfcal_library['sub-fields-fid_map'][vis][0]) include_targets = selfcal_library['bands_for_targets'][vis]['field_str'] @@ -309,6 +309,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) for incl_scans, incl_targets in zip(include_scans, include_targets): for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + print(vis,solint,mode) print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][mode] @@ -331,27 +332,52 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so spwselect=selfcal_library[vis]['spws'] gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' print('prior to gaincal',gaintable_name, mode) - if mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': + vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + for incl_target in incl_targets: + vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + if mode != 'per_bb': + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + else: + for baseband in selfcal_library[vis]['baseband'].keys(): + spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) else: - for baseband in selfcal_library[vis]['baseband'].keys(): - spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + if mode != 'per_bb': + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) - + else: + for baseband in selfcal_library[vis]['baseband'].keys(): + spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][mode] = gaintable_name # restricted gaincal table comparisons to only inf_EB prior to changes diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index bb86f9ec..41cd4872 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -69,7 +69,8 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ do_fallback_calonly=False print('Starting selfcal procedure on: '+target+' '+band) while iteration < len(selfcal_plan['solints']): - + print(selfcal_plan['solints']) + print(selfcal_plan['solint_interval'][iteration]) print("Solving for solint="+selfcal_plan['solints'][iteration]+' with interval '+selfcal_plan['solint_interval'][iteration]) # Set some cocal parameters. diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index 16c8bcbf..ddf3d06f 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -4650,8 +4650,8 @@ def triage_calibrators(vis, target, band, potential_calibrators, max_distance=10 orig_vis='' if os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')): orig_vis=os.path.exists(vis.replace("_target.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) - elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')): - orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_')) + elif os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')): + orig_vis=os.path.exists(vis.replace("_targets.selfcal.ms",".ms").replace(sani_target+'_'+band+'_','')) # original visibilities, with all sources have a different filename now diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index 79976d9a..a73cb692 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True,parallel=parallel) +auto_selfcal(vislist, allow_cocal=True,do_amp_selfcal=False,parallel=parallel) From 4e005e0d0123073c89822951ff384d450030d12a Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 1 May 2026 16:44:06 -0400 Subject: [PATCH 06/24] more progress toward getting cocal to run --- auto_selfcal/auto_selfcal.py | 5 +++++ auto_selfcal/run_selfcal.py | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index be8f4464..ef23b1d0 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -773,6 +773,11 @@ def default(self, obj): selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]=selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 41cd4872..034784ef 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -442,9 +442,12 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ marginal_inf_EB_will_attempt_next_solint = False else: marginal_inf_EB_will_attempt_next_solint = True - - RMS_change_acceptable = (post_RMS/RMS < 1.05 and post_RMS_NF/RMS_NF < 1.05) or \ - ((post_RMS/RMS > 1.05 or post_RMS_NF/RMS_NF > 1.05) and selfcal_plan['solint_snr'][solint] > 5) + RMS_change_acceptable= False + if mode != 'cocal': + RMS_change_acceptable = (post_RMS/RMS < 1.05 and post_RMS_NF/RMS_NF < 1.05) or \ + ((post_RMS/RMS > 1.05 or post_RMS_NF/RMS_NF > 1.05) and selfcal_plan['solint_snr'][solint] > 5) + else: + RMS_change_acceptable = (post_RMS/RMS < 1.05 and post_RMS_NF/RMS_NF < 1.05) if (((post_SNR >= SNR) and (post_SNR_NF >= SNR_NF) and (delta_beamarea < delta_beam_thresh)) or ((('inf_EB' in solint) or 'delay' in solint) and marginal_inf_EB_will_attempt_next_solint and ((post_SNR-SNR)/SNR > -0.02) and ((post_SNR_NF - SNR_NF)/SNR_NF > -0.02) and (delta_beamarea < delta_beam_thresh))) and np.any(field_by_field_success) and RMS_change_acceptable: From c64e5799e6e2706b4e7fcc972c95e2de4d7814c0 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Mon, 4 May 2026 14:55:22 -0400 Subject: [PATCH 07/24] fix variable name used twice and another fail statement invalid for co-cal --- auto_selfcal/gaincal_wrapper.py | 27 ++++++++++++++------------- auto_selfcal/run_selfcal.py | 28 ++++++++++++++++++++-------- bin/auto_selfcal.py | 2 +- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index c6fff6d6..13bc59bd 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -106,6 +106,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so #include_targets = str(selfcal_library['sub-fields-fid_map'][vis][0]) include_targets = selfcal_library['bands_for_targets'][vis]['field_str'] include_scans = "" + print('Include targets: ', include_targets) if solint == "scan_inf": if len(gaincalibrator_dict[vis]) > 0: @@ -308,12 +309,12 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print(solint,'Include targets: ', include_targets) print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) for incl_scans, incl_targets in zip(include_scans, include_targets): - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - - print(vis,solint,mode) + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + print(incl_targets) + print(vis,solint,gc_mode) print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) - gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][mode] - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][mode] + gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] if 'spw' in gaincal_combine: if selfcal_library['spws_set'][vis].ndim == 1: @@ -331,12 +332,12 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: spwselect=selfcal_library[vis]['spws'] gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - print('prior to gaincal',gaintable_name, mode) + print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') for incl_target in incl_targets: vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') - if mode != 'per_bb': + if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ @@ -344,7 +345,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] @@ -355,9 +356,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: - if mode != 'per_bb': + if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ @@ -365,7 +366,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] @@ -376,9 +377,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][mode] = gaintable_name + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name # restricted gaincal table comparisons to only inf_EB prior to changes # commenting because we want to do comparisons for other solints as well diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 034784ef..5737d8f9 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -608,14 +608,26 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if reason !='': reason=reason+'; ' reason=reason+'Beam change beyond '+str(delta_beam_thresh) - if (post_RMS/RMS > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): - if reason != '': - reason=reason+'; ' - reason=reason+'RMS increase beyond 5%' - if (post_RMS_NF/RMS_NF > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): - if reason != '': - reason=reason+'; ' - reason=reason+'NF RMS increase beyond 5%' + + if mode != 'cocal': + if (post_RMS/RMS > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): + if reason != '': + reason=reason+'; ' + reason=reason+'RMS increase beyond 5%' + if (post_RMS_NF/RMS_NF > 1.05 and selfcal_plan['solint_snr'][solint] <= 5): + if reason != '': + reason=reason+'; ' + reason=reason+'NF RMS increase beyond 5%' + else: + if (post_RMS/RMS > 1.05): + if reason != '': + reason=reason+'; ' + reason=reason+'RMS increase beyond 5%' + if (post_RMS_NF/RMS_NF > 1.05): + if reason != '': + reason=reason+'; ' + reason=reason+'NF RMS increase beyond 5%' + if not np.any(field_by_field_success): if reason != '': reason=reason+'; ' diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index a73cb692..4285c610 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True,do_amp_selfcal=False,parallel=parallel) +auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,parallel=parallel) From 96dab1389bb50cd8ad32e5785ad36ffc6216c960 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Mon, 4 May 2026 15:31:47 -0400 Subject: [PATCH 08/24] fix more double use of mode variable --- auto_selfcal/gaincal_wrapper.py | 41 +++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 13bc59bd..2c37f5d0 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -29,8 +29,8 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so os.system('rm -rf '+sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'*.g') ## Reset the gaincal return dictionaries, in case this is a repeat of the current solution interval. - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode] = [] + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode] = [] ## ## Set gaincal parameters depending on which iteration and whether to use combine=spw for inf_EB or not ## Defaults should assume combine='scan' and gaintpe='G' will fallback to combine='scan,spw' if too much flagging @@ -334,11 +334,12 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': - vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + print('in cocal if statement') for incl_target in incl_targets: vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + print('Vis to gaincal: ',vis_to_gaincal) if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -349,7 +350,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -359,7 +360,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -370,7 +371,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ @@ -478,9 +479,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: solnorm=False - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][mode] - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][mode] + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] if 'spw' in gaincal_combine: if selfcal_library['spws_set'][vis].ndim == 1: @@ -498,28 +499,28 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so else: spwselect=selfcal_library[vis]['spws'] gaintable_name='temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - if mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect, + if gc_mode != 'per_bb': + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=str(selfcal_library['sub-fields-fid_map'][vis][fid]),gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(gaintable_name)) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) else: for baseband in selfcal_library[vis]['baseband'].keys(): spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][mode], spw=spwselect_bb, + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ field=str(selfcal_library['sub-fields-fid_map'][vis][fid]),gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(gaintable_name)) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode].append(gcdict.copy()) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][mode] = gaintable_name + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name gaintable_prefix='temp_' if len(selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) > 1: @@ -575,18 +576,18 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][preferred_mode] - for mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: ##### send chosen subtable to this routine for final copying to the gain table we want. - tb.open('temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+mode+'.g') + tb.open('temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+gc_mode+'.g') subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") tb.close() - subt.copy(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+mode+'.g', deep=True) + subt.copy(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+gc_mode+'.g', deep=True) subt.close() # commented so that we keep all the gain tables around # once well-tested, we might remove the tables for the non-chosen modes to avoid generating too many useless files. - os.system('rm -rf '+'temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+mode+'.g') + os.system('rm -rf '+'temp_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+gc_mode+'.g') if rerank_refants: selfcal_library[vis]["refant"] = rank_refants(vis, selfcal_library['telescope'], caltable=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+selfcal_library[vis][solint]['final_mode']+'.g') From 8aea04864f1a8774b63c8cc0615851460d18df51 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Mon, 4 May 2026 16:43:17 -0400 Subject: [PATCH 09/24] pass preapply_targets_own_inf_EB to gaincal_wrapper --- auto_selfcal/gaincal_wrapper.py | 2 +- auto_selfcal/run_selfcal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 2c37f5d0..dbf1b5ce 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -5,7 +5,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so gaincal_minsnr, gaincal_unflag_minsnr=5.0, minsnr_to_proceed=3.0, rerank_refants=False, unflag_only_lbants=False, unflag_only_lbants_onlyap=False, calonly_max_flagged=0.0, second_iter_solmode="", unflag_fb_to_prev_solint=False, \ refantmode="flex", mode="selfcal", calibrators="", gaincalibrator_dict={}, allow_gain_interpolation=False,spectral_solution_fraction=0.3, - guess_scan_combine=False, do_fallback_calonly=False): + guess_scan_combine=False, do_fallback_calonly=False, preapply_targets_own_inf_EB=False): """ This function runs gaincal for a given target, band, and solint, and updates the selfcal_library and selfcal_plan dictionaries with the results. It also handles the pre-application of inf_EB solutions if necessary. diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 5737d8f9..9e892367 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -251,7 +251,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ second_iter_solmode=second_iter_solmode, unflag_fb_to_prev_solint=unflag_fb_to_prev_solint, \ refantmode=refantmode, mode=mode, calibrators=calibrators, gaincalibrator_dict=gaincalibrator_dict, allow_gain_interpolation=allow_gain_interpolation,spectral_solution_fraction=spectral_solution_fraction, - do_fallback_calonly=do_fallback_calonly, guess_scan_combine=guess_scan_combine) + do_fallback_calonly=do_fallback_calonly, guess_scan_combine=guess_scan_combine, preapply_targets_own_inf_EB=preapply_targets_own_inf_EB) # With gaincal done and bad fields removed from gain tables if necessary, check whether any fields should no longer be # selfcal'd because they have too much interpolation. From cd3feb9d0a2bf057183ea4dff35aa13520a88d65 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Tue, 5 May 2026 10:49:46 -0400 Subject: [PATCH 10/24] fix the target and vis file specification for cocal gaincal calls --- auto_selfcal/gaincal_wrapper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index dbf1b5ce..c3a97f55 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -311,6 +311,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so for incl_scans, incl_targets in zip(include_scans, include_targets): for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: print(incl_targets) + print(incl_scans) print(vis,solint,gc_mode) print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] @@ -335,15 +336,15 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': print('in cocal if statement') - for incl_target in incl_targets: - vis_to_gaincal = sanitize_target(incl_target)+vis.replace(sanitize_target(target),'') + for incl_target in include_targets: + vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') print('Vis to gaincal: ',vis_to_gaincal) if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) @@ -354,7 +355,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) From d55e0d63a25f44e38d9721c12d3501b568e8a3c5 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Tue, 5 May 2026 16:21:08 -0400 Subject: [PATCH 11/24] add a concat step to add co-calibrators to the MS of the source needing cocal --- auto_selfcal/gaincal_wrapper.py | 5 +++-- auto_selfcal/run_selfcal.py | 19 ++++++++++++++++--- bin/auto_selfcal.py | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index c3a97f55..70b224c1 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -336,8 +336,9 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print('prior to gaincal',gaintable_name, gc_mode) if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': print('in cocal if statement') - for incl_target in include_targets: - vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') + for incl_target in include_targets[0].split(','): + #vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') + vis_to_gaincal = vis print('Vis to gaincal: ',vis_to_gaincal) if gc_mode != 'per_bb': gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 9e892367..fb347e8e 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -49,11 +49,22 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ if mode == "cocal": # Check whether there are suitable calibrators, otherwise skip this target/band. include_targets, include_scans = triage_calibrators(vislist[0], target, band, calibrators[band][0]) + print('Co-calibrators: ',include_targets) + print('Co-calibrator scans: ',include_scans) if include_targets == "": print("No suitable calibrators found, skipping "+target) selfcal_library['Stop_Reason'] += '; No suitable co-calibrators' return - + else: + for vis in vislist: + os.system('mv '+vis+' '+vis.replace('.ms','_orig.ms')) + os.system('mv '+vis+'.flagversions '+vis.replace('.ms','_orig.ms.flagversions')) + concatvislist=[] + for cal_target in include_targets.split(','): + concatvislist.append(vis.replace(target,cal_target)) + concatvislist.append(vis.replace('.ms','_orig.ms')) + print('Concat vislist: ',concatvislist) + concat(vis=concatvislist,concatvis=vis) if selfcal_library['usermodel'] != '': print('Setting model column to user model') usermodel_wrapper(selfcal_library,sani_target+'_'+band, @@ -194,8 +205,10 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ flagmanager(vis=vis, mode = 'restore', versionname = versionname, comment = 'Flag states at start of reduction') if mode == "cocal": - flagmanager(vis=vis, mode = 'restore', versionname = 'selfcal_starting_flags', comment = 'Flag states at start of the reduction') - + if os.path.exists(vis+".flagversions/flags.selfcal_starting_flags"): + flagmanager(vis=vis, mode = 'restore', versionname = 'selfcal_starting_flags', comment = 'Flag states at start of the reduction') + else: + flagmanager(vis=vis,mode='save',versionname='selfcal_starting_flags') if not do_fallback_combinespw: # We need to redo saving the model now that we have potentially unflagged some data. if not do_fallback_calonly: diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index 4285c610..5b6fe7f6 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,parallel=parallel) +auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,targets='FZ_Tau,DL_Tau,DO_Tau',parallel=parallel) From f39b3a1a1a4b80986ccd0dc0fbea8f32e02d6ede Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 7 May 2026 10:50:31 -0400 Subject: [PATCH 12/24] Make sure to increment iteration for all possible endings of cocal solints. --- auto_selfcal/run_selfcal.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index fb347e8e..e9945106 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -93,14 +93,17 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ preapply_targets_own_inf_EB = False # If there was not a successful inf_EB solint, then this duplicates inf_fb1 so skip if "inf_EB" not in selfcal_library[vislist[0]]: + iteration += 1 continue elif not selfcal_library[vislist[0]]["inf_EB"]['Pass']: + iteration += 1 continue elif selfcal_plan['solints'][iteration] == "inf_fb3": calculate_inf_EB_fb_anyways = False preapply_targets_own_inf_EB = True # If there was no inf solint (e.g. because each source was observed only a single time, skip this as there are no gain tables to stick together. if "inf" not in selfcal_plan['solints']: + iteration += 1 continue if 'ap' in selfcal_plan['solints'][iteration]: @@ -779,6 +782,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ continue elif mode == "cocal" and "inf_fb" in solint: print('****************Cocal failed, attempting next inf_fb option*************') + iteration += 1 continue else: print('****************Aborting further self-calibration attempts for '+target+' '+band+'**************') From 88e82b453deed72f81a7024d4fc1bb0e677de76f Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 7 May 2026 15:20:12 -0400 Subject: [PATCH 13/24] Loop over all possible gaincal modes and look up the correct calibration files for the inf_fb3 solint. --- auto_selfcal/gaincal_wrapper.py | 78 ++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 70b224c1..238689f3 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -231,49 +231,57 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so ## If we want to pre-apply inf_EB solution from each calibrator to itself, all we do is combine all of thier ## individual inf tables, as these were pre-calculated in that way. ## - destination_table = sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'.g' - for t in include_targets.split(","): - if os.path.exists(sanitize_string(t)+'_'+vis+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '.pre-pass.g'): - table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '.pre-pass.g' - else: - table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+'.g' - #t_final_solint = selfcal_library[t][band]["final_phase_solint"] - #t_iteration = selfcal_library[t][band][vislist[0]][t_final_solint]["iteration"] - #table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+t_final_solint+'_'+str(t_iteration)+'_'+solmode[band][iteration]+'.g' + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] + + destination_table = sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' + for t in include_targets.split(","): + sani_t = sanitize_string(t) + if os.path.exists(sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ + '_'+filename_append+'.pre-pass.g'): + table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ + '_'+filename_append+'.pre-pass.g' + else: + table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ + '_'+filename_append+'.g' + #t_final_solint = selfcal_library[t][band]["final_phase_solint"] + #t_iteration = selfcal_library[t][band][vislist[0]][t_final_solint]["iteration"] + #table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+t_final_solint+'_'+str(t_iteration)+'_'+solmode[band][iteration]+'.g' - rerefant(vis, table_name, caltable="tmp0.g", refantmode="strict", refant=selfcal_library[vis]['refant']) + if not os.path.exists(table_name): + continue - tb.open("tmp0.g") - if not os.path.exists("tmp1.g"): - tb.copy("tmp1.g", deep=True) - else: - tb.copyrows("tmp1.g") - tb.close() + rerefant(vis, table_name, caltable="tmp0.g", refantmode="strict", refant=selfcal_library[vis]['refant']) - os.system("rm -rf tmp0.g") + tb.open("tmp0.g") + if not os.path.exists("tmp1.g"): + tb.copy("tmp1.g", deep=True) + else: + tb.copyrows("tmp1.g") + tb.close() - tb.open("tmp1.g") - subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") - copyt = subt.copy(destination_table, deep=True) - tb.close() - subt.close() - copyt.close() + os.system("rm -rf tmp0.g") + + tb.open("tmp1.g") + subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") + copyt = subt.copy(destination_table, deep=True) + tb.close() + subt.close() + copyt.close() - os.system("rm -rf tmp1.g") + os.system("rm -rf tmp1.g") - # Remove all of the scans that failed the triage above. - tb.open(destination_table, nomodify=False) - scans = tb.getcol("SCAN_NUMBER") + # Remove all of the scans that failed the triage above. + tb.open(destination_table, nomodify=False) + scans = tb.getcol("SCAN_NUMBER") - bad_scans = np.repeat(True, scans.size) - for scan in include_scans[0].split(","): - bad_scans[scans == int(scan)] = False + bad_scans = np.repeat(True, scans.size) + for scan in include_scans[0].split(","): + bad_scans[scans == int(scan)] = False - tb.removerows(rownrs=np.where(bad_scans)[0]) - tb.flush() - tb.close() + tb.removerows(rownrs=np.where(bad_scans)[0]) + tb.flush() + tb.close() else: # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. gaincal_solmode="" From fb73814bd7183b89b0e5d8fb2451d6fe9a570693 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Thu, 7 May 2026 17:26:56 -0400 Subject: [PATCH 14/24] add sanitize string calls to construction of concat vislist --- auto_selfcal/auto_selfcal.py | 12 +++++++++--- auto_selfcal/run_selfcal.py | 2 +- bin/auto_selfcal.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index ef23b1d0..d1c928e9 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -775,9 +775,15 @@ def default(self, obj): selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] for vis in selfcal_library[target][band]['vislist']: selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]=selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + if "inf" in selfcal_plan[target][band][vis]['solint_settings'].keys(): + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] + else: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] + selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index e9945106..530df8b0 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -61,7 +61,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ os.system('mv '+vis+'.flagversions '+vis.replace('.ms','_orig.ms.flagversions')) concatvislist=[] for cal_target in include_targets.split(','): - concatvislist.append(vis.replace(target,cal_target)) + concatvislist.append(vis.replace(sanitize_string(target),sanitize_string(cal_target))) concatvislist.append(vis.replace('.ms','_orig.ms')) print('Concat vislist: ',concatvislist) concat(vis=concatvislist,concatvis=vis) diff --git a/bin/auto_selfcal.py b/bin/auto_selfcal.py index 5b6fe7f6..4285c610 100644 --- a/bin/auto_selfcal.py +++ b/bin/auto_selfcal.py @@ -13,5 +13,5 @@ vislist = [] # Edit manually, or leave and let auto_selfcal automatically detect. -auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,targets='FZ_Tau,DL_Tau,DO_Tau',parallel=parallel) +auto_selfcal(vislist, allow_cocal=True, delta_beam_thresh=0.2,do_amp_selfcal=False,parallel=parallel) From 3ac55bcfd4f0dc1eaec45c24d42740fc623e24b5 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 8 May 2026 11:03:12 -0400 Subject: [PATCH 15/24] remove mask check if doing cocal --- auto_selfcal/run_selfcal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/run_selfcal.py b/auto_selfcal/run_selfcal.py index 530df8b0..93e962c0 100644 --- a/auto_selfcal/run_selfcal.py +++ b/auto_selfcal/run_selfcal.py @@ -57,6 +57,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ return else: for vis in vislist: + clearcal(vis=vis,addmodel=True) os.system('mv '+vis+' '+vis.replace('.ms','_orig.ms')) os.system('mv '+vis+'.flagversions '+vis.replace('.ms','_orig.ms.flagversions')) concatvislist=[] @@ -183,7 +184,7 @@ def run_selfcal(selfcal_library, selfcal_plan, target, band, n_ants, \ # Check that a mask was actually created, because if not the model will be empty and gaincal will do bad things and the # code will break. - if not checkmask(sani_target+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0'): + if not checkmask(sani_target+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0') and mode !="cocal": selfcal_library['Stop_Reason'] = 'Empty model for solint '+solint for fid in selfcal_library['sub-fields-to-selfcal']: selfcal_library[fid]['Stop_Reason'] = 'Empty model for solint '+solint From f81270daac5f4710b7ac9a822a307049b31b9a62 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 14 May 2026 15:05:09 -0400 Subject: [PATCH 16/24] Update cocal to work better with setting the solint_settings for the fb solints. This includes removing the stapling together of existing gaintables for inf_fb3, and also splitting cocal preparation into prepare_cocal. --- auto_selfcal/auto_selfcal.py | 116 +-------- auto_selfcal/gaincal_wrapper.py | 419 ++++++++------------------------ auto_selfcal/mosaic_helpers.py | 118 +++++++++ auto_selfcal/prepare_cocal.py | 143 +++++++++++ auto_selfcal/prepare_selfcal.py | 43 ++-- auto_selfcal/selfcal_helpers.py | 8 +- 6 files changed, 403 insertions(+), 444 deletions(-) create mode 100644 auto_selfcal/prepare_cocal.py diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index ef23b1d0..1db79af9 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -8,6 +8,7 @@ import sys import pickle import pprint +import copy from .selfcal_helpers import * from .run_selfcal import run_selfcal @@ -15,6 +16,7 @@ from .weblog_creation import * from .prepare_selfcal import prepare_selfcal, set_clean_thresholds, plan_selfcal_per_solint from .original_ms_helpers import applycal_to_orig_MSes, uvcontsub_orig_MSes +from .prepare_cocal import prepare_cocal try: from .alignment_helpers import align_measurement_sets except: @@ -729,116 +731,7 @@ def default(self, obj): if allow_cocal: - ## - ## Save the flags following the main iteration of self-calibration since we will need to revert to the beginning for the fallback mode. - ## - # PS: I don't need this anymore? - for vis in selfcal_library[target][band]['vislist']: - if not os.path.exists(vis+'.flagversions/flags.fb_selfcal_starting_flags'): - flagmanager(vis=vis,mode='save',versionname='fb_selfcal_starting_flags') - else: - flagmanager(vis=vis,mode='restore',versionname='fb_selfcal_starting_flags') - - ## - ## For sources that self-calibration failed, try to use the inf_EB and the inf solutions from the sources that - ## were successful. - - for target in selfcal_library.keys(): - for band in selfcal_library[target].keys(): - print(target, selfcal_library[target][band]["final_solint"]) - - inf_EB_fields = {} - inf_fields = {} - fallback_fields = {} - calibrators = {} - for band in vis_for_targets[target]['Bands']: - # Initialize the lists for this band. - inf_EB_fields[band] = [] - inf_fields[band] = [] - fallback_fields[band] = [] - - # Loop through and identify which sources belong where. - for target in selfcal_library.keys(): - if selfcal_library[target][band]['SC_success'] and 'fb' not in selfcal_library[target][band]['final_solint']: - inf_EB_fields[band].append(target) - if selfcal_library[target][band]['final_solint'] != 'inf_EB': - inf_fields[band].append(target) - elif 'inf' in selfcal_plan[target][band]['solints']: - fallback_fields[band].append(target) - else: - fallback_fields[band].append(target) - - # Update the relevant lists if we are going to do a fallback mode. - if len(fallback_fields[band]) > 0: - selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] - selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] - selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] - for vis in selfcal_library[target][band]['vislist']: - selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]=selfcal_plan[target][band][vis]['solint_settings']["inf_EB"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]= selfcal_plan[target][band][vis]['solint_settings']["inf"] - selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] - selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] - #applycal_mode[band][target] += [applycal_mode[band][target][0], applycal_mode[band][target][1], applycal_mode[band][target][1], applycal_mode[band][target][1]] - calibrators[band] = [inf_EB_fields[band], inf_fields[band], inf_fields[band], inf_fields[band]] - selfcal_library[target][band]["nsigma"] = np.concatenate((selfcal_library[target][band]["nsigma"],[selfcal_library[target][band]["nsigma"][0], \ - selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1]])) - - print(inf_EB_fields) - print(inf_fields) - print(fallback_fields) - - ## - ## Reset the inf_EB informational dictionaries. - ## - - for target in selfcal_library: - for band in selfcal_library[target].keys(): - # If the target had a successful inf_EB solution, no need to reset. - if target in inf_EB_fields[band]: - continue - - for vis in selfcal_library[target][band]['vislist']: - selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']=inf_EB_gaincal_combine #'scan' - if selfcal_library[target][band]['obstype']=='mosaic': - selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']+=',field' - selfcal_plan[target][band][vis]['inf_EB_gaintype']=inf_EB_gaintype #G - selfcal_plan[target][band][vis]['inf_EB_fallback_mode']='' #'scan' - - - calculate_inf_EB_fb_anyways = True - preapply_targets_own_inf_EB = False - - ## The below sets the calibrations back to what they were prior to starting the fallback mode. It should not be needed - ## for the final version of the codue, but is used for testing. - - - for target in selfcal_library: - sani_target=sanitize_string(target) - for band in selfcal_library[target].keys(): - if target not in fallback_fields[band]: - continue - if 'gaintable_final' in selfcal_library[target][band]['vislist'][0]: - print('****************Reapplying previous solint solutions*************') - for vis in selfcal_library[target][band]['vislist']: - print('****************Applying '+str(selfcal_library[target][band][vis]['gaintable_final'])+' to '+target+' '+band+'*************') - ## NOTE: should this be selfcal_starting_flags instead of fb_selfcal_starting_flags ??? - flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) - applycal(vis=vis,\ - gaintable=selfcal_library[target][band][vis]['gaintable_final'],\ - interp=selfcal_library[target][band][vis]['applycal_interpolate_final'],\ - calwt=True,spwmap=selfcal_library[target][band][vis]['spwmap_final'],\ - applymode=selfcal_library[target][band][vis]['applycal_mode_final'],\ - field=target,spw=selfcal_library[target][band][vis]['spws']) - else: - print('****************Removing all calibrations for '+target+' '+band+'**************') - for vis in selfcal_library[target][band]['vislist']: - flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) - clearcal(vis=vis,field=target,spw=selfcal_library[target][band][vis]['spws']) - ## END - - + fallback_fields, calibrators = prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_gaintype) ## ## Begin fallback self-cal loops ## @@ -852,8 +745,7 @@ def default(self, obj): inf_EB_gaincal_combine=inf_EB_gaincal_combine, inf_EB_gaintype=inf_EB_gaintype, unflag_only_lbants=unflag_only_lbants, \ unflag_only_lbants_onlyap=unflag_only_lbants_onlyap, calonly_max_flagged=calonly_max_flagged, \ second_iter_solmode=second_iter_solmode, unflag_fb_to_prev_solint=unflag_fb_to_prev_solint, rerank_refants=rerank_refants, \ - mode="cocal", calibrators=calibrators, calculate_inf_EB_fb_anyways=calculate_inf_EB_fb_anyways, \ - preapply_targets_own_inf_EB=preapply_targets_own_inf_EB, gaincalibrator_dict=gaincalibrator_dict, allow_gain_interpolation=True) + mode="cocal", calibrators=calibrators, gaincalibrator_dict=gaincalibrator_dict, allow_gain_interpolation=True) if debug: print(json.dumps(selfcal_library, indent=4, cls=NpEncoder)) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index 238689f3..fae8cc42 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -1,5 +1,6 @@ import numpy as np from .selfcal_helpers import * +from .mosaic_helpers import scan_inf_scan_combine def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, solint_interval, applymode, iteration, gaincal_minsnr, gaincal_unflag_minsnr=5.0, minsnr_to_proceed=3.0, rerank_refants=False, unflag_only_lbants=False, unflag_only_lbants_onlyap=False, @@ -31,6 +32,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so ## Reset the gaincal return dictionaries, in case this is a repeat of the current solution interval. for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode] = [] + ## ## Set gaincal parameters depending on which iteration and whether to use combine=spw for inf_EB or not ## Defaults should assume combine='scan' and gaintpe='G' will fallback to combine='scan,spw' if too much flagging @@ -39,58 +41,31 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so current_solint_index=selfcal_plan['solints'].index(solint) selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'] = {} if selfcal_plan['solmode'][iteration] == 'p': - if mode == "cocal": - if 'inf_EB' in selfcal_library[vis]: - #gaincal_preapply_gaintable[vis]=[sani_target+'_'+vis+'_'+band+'_inf_EB_0_p.g'] - if calculate_inf_EB_fb_anyways or not selfcal_library[vis]["inf_EB"]["Pass"]: - previous_solint = "inf_EB_fb" - else: - previous_solint = "inf_EB" - else: - #gaincal_preapply_gaintable[vis]=[sani_target+'_'+vis+'_'+band+'_inf_EB_fb_'+str(iteration-1)+'_p.g'] - previous_solint = "inf_EB_fb" - else: - # not entirely sure why this if/else is necessary might not be with new selfcal plan - if selfcal_plan['solmode'][iteration]=='p': - previous_solint = "inf_EB" - else: - previous_solint = selfcal_library['final_phase_solint'] gaincal_spwmap=[] gaincal_preapply_gaintable=[] gaincal_interpolate=[] applycal_spwmap=[] applycal_interpolate=[] applycal_gaintable=[] - for j in range(current_solint_index): - if selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['preapply_this_gaintable']: - gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['accepted_gaintable']) - gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_spwmap']) - gaincal_interpolate.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_interpolate']) - applycal_spwmap.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_spwmap']) - applycal_interpolate.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['applycal_interpolate']) - applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['accepted_gaintable']) + + if mode == "cocal": + include_solints = selfcal_plan[vis]['solint_settings'][solint]['preapply_solints'] + else: + include_solints = [selfcal_plan['solints'][j] for j in range(current_solint_index) if + selfcal_plan[vis]['solint_settings'][selfcal_plan['solints'][j]]['preapply_this_gaintable']] + + for incl_solint in include_solints: + gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) + gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) + gaincal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) + applycal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) + applycal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) + applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) if solint != 'inf_EB': gaincal_solmode = "" if not do_fallback_calonly or second_iter_solmode == "GSPLINE" else second_iter_solmode - """ - if 'spw' in selfcal_plan[vis]['inf_EB_gaincal_combine']: - applycal_spwmap[vis]=[selfcal_library[vis]['spwmap'],selfcal_library[vis]['spwmap']] - gaincal_spwmap[vis]=[selfcal_library[vis]['spwmap']] - elif selfcal_plan[vis]['inf_EB_fallback_mode']=='spwmap': - applycal_spwmap[vis]=selfcal_library[vis]['inf_EB']['spwmap'] + [selfcal_library[vis]['spwmap']] - gaincal_spwmap[vis]=selfcal_library[vis]['inf_EB']['spwmap'] - else: - applycal_spwmap[vis]=[[],selfcal_library[vis]['spwmap']] - gaincal_spwmap[vis]=[] - """ - # Revert back to applying the inf_EB solution if calculate_inf_EB_fb_anyways, i.e. we just use the inf_EB_fb solution - # for gaincal. - if mode == "cocal": - if selfcal_library['final_solint'] == 'inf_EB' and calculate_inf_EB_fb_anyways: - previous_solint = "inf_EB" - fallback='' if selfcal_plan['solmode'][iteration] == 'ap': solnorm=True @@ -109,293 +84,106 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so print('Include targets: ', include_targets) if solint == "scan_inf": - if len(gaincalibrator_dict[vis]) > 0: - print("Determining scan_inf from calibrator scans in full MS") - scans = [] - intents = [] - times = [] - for t in gaincalibrator_dict[vis].keys(): - scans += [gaincalibrator_dict[vis][t]["scans"]] - intents += [np.repeat(gaincalibrator_dict[vis][t]["intent"],gaincalibrator_dict[vis][t]["scans"].size)] - times += [gaincalibrator_dict[vis][t]["times"]] - - times = np.concatenate(times) - order = np.argsort(times) - times = times[order] - - scans = np.concatenate(scans)[order] - intents = np.concatenate(intents)[order] - - is_gaincalibrator = intents == "phase" - scans = scans[is_gaincalibrator] - - msmd.open(vis) - if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': - scan_ids_for_target = msmd.scansforfield(target) - elif 'VLA' in selfcal_library['telescope']: - scan_ids_for_target=np.array([],dtype=int) - for fid in selfcal_library['sub-fields']: - if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): - field_id=selfcal_library['sub-fields-fid_map'][vis][fid] - scan_ids_for_target=np.append(scan_ids_for_target,msmd.scansforfield(field_id)) - - scan_ids_for_target.sort() # sort scans since they will be out of order - include_scans = [] - for iscan in range(scans.size-1): - scan_group = np.intersect1d(scan_ids_for_target, - np.array(list(range(scans[iscan]+1, scans[iscan+1])))).astype(str) - if scan_group.size > 0: - include_scans.append(",".join(scan_group)) - # PIPE-2741: if there are any scans before the first gain calibrator scan or after the last gain calibrator scan, catch them. - if scans.size > 0: - extra_scans = scan_ids_for_target[scan_ids_for_target > max(scans)] - if extra_scans.size > 0: - include_scans.append(','.join(extra_scans.astype(str))) - extra_scans = scan_ids_for_target[scan_ids_for_target < min(scans)] - if extra_scans.size > 0: - include_scans.append(','.join(extra_scans.astype(str))) - msmd.close() - elif guess_scan_combine: - print("Determining scan_inf from guessing where the calibrator scans were") - msmd.open(vis) - include_scans = [] - - #to guess at scan_inf combination for VLA look for breaks in the consecutive - #scan numbers and assume that the break is due to a calibrator scan - #Fetch scans for scan inf by collecting the field ids and running msmd.scansforfield - if 'VLA' in selfcal_library['telescope']: - scans=np.array([],dtype=int) - for fid in selfcal_library['sub-fields']: - if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): - field_id=selfcal_library['sub-fields-fid_map'][vis][fid] - scans=np.append(scans,msmd.scansforfield(field_id)) - - scans.sort() # sort scans since they will be out of order - scan_group='' - for iscan in range(scans.size): - if len(include_scans) > 0: - if str(scans[iscan]) in include_scans[-1]: - continue - if scan_group == '': - scan_group = str(int(scans[iscan])) - - if iscan < scans.size-1: - if scans[iscan+1] == scans[iscan]+1: - scan_group += ","+str(int(scans[iscan+1])) - else: - include_scans.append(scan_group) - scan_group='' - else: #write out the last scan group to include_scans - if scan_group != '': - include_scans.append(scan_group) - scan_group='' - - #guess scan_inf combination by getting all the scans for targets and do a simple grouping - if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': - scans = msmd.scansforfield(target) - - for iscan in range(scans.size): - if len(include_scans) > 0: - if str(scans[iscan]) in include_scans[-1]: - continue - - scan_group = str(scans[iscan]) - - if iscan < scans.size-1: - if msmd.fieldsforscan(scans[iscan+1]).size < msmd.fieldsforscan(scans[iscan]).size/3: - scan_group += ","+str(scans[iscan+1]) - - include_scans.append(scan_group) - - msmd.close() - else: - print("Not guessing where calibration scans are and justincluding all scans") - msmd.open(vis) - if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': - include_scans = [str(scan) for scan in msmd.scansforfield(target)] - elif 'VLA' in selfcal_library['telescope']: - scans=np.array([],dtype=int) - for fid in selfcal_library['sub-fields']: - if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): - field_id=selfcal_library['sub-fields-fid_map'][vis][fid] - scans=np.append(scans,msmd.scansforfield(field_id)) - - scans.sort() # sort scans since they will be out of order - include_scans = [str(scan) for scan in scans] - msmd.close() + include_scans = scan_inf_scan_combine(selfcal_library, vis, target, gaincalibrator_dict, guess_scan_combine=guess_scan_combine) else: include_scans = [include_scans] - if mode == "cocal" and preapply_targets_own_inf_EB and "inf_fb" in solint and "inf" in selfcal_plan['solints']: - ## - ## If we want to pre-apply inf_EB solution from each calibrator to itself, all we do is combine all of thier - ## individual inf tables, as these were pre-calculated in that way. - ## - for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] - - destination_table = sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - for t in include_targets.split(","): - sani_t = sanitize_string(t) - if os.path.exists(sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '_'+filename_append+'.pre-pass.g'): - table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '_'+filename_append+'.pre-pass.g' - else: - table_name = sani_t+'_'+vis.replace(sani_target, sani_t)+'_'+band+'_'+solint.replace('_fb1','').replace('_fb2','').replace('_fb3','')+'_'+str(1)+'_'+selfcal_plan['solmode'][iteration]+\ - '_'+filename_append+'.g' - #t_final_solint = selfcal_library[t][band]["final_phase_solint"] - #t_iteration = selfcal_library[t][band][vislist[0]][t_final_solint]["iteration"] - #table_name = sanitize_string(t)+'_'+vis+'_'+band+'_'+t_final_solint+'_'+str(t_iteration)+'_'+solmode[band][iteration]+'.g' - - if not os.path.exists(table_name): - continue + + # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. + gaincal_solmode="" + if selfcal_library['obstype'] == 'mosaic': + msmd.open(vis) + include_targets = [] + remove = [] + for incl_scan in include_scans: + scan_targets = [] + for fid in [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ + np.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))] if incl_scan == '' else \ + np.intersect1d(msmd.fieldsforscans(np.array(incl_scan.split(",")).astype(int)), \ + [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ + numpy.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))]): + # Note: because of the msmd above getting actual fids from the MS, we just need to append fid below. + scan_targets.append(fid) + + if len(scan_targets) > 0: + include_targets.append(','.join(np.array(scan_targets).astype(str))) + else: + remove.append(incl_scan) - rerefant(vis, table_name, caltable="tmp0.g", refantmode="strict", refant=selfcal_library[vis]['refant']) + for incl_scan in remove: + include_scans.remove(incl_scan) - tb.open("tmp0.g") - if not os.path.exists("tmp1.g"): - tb.copy("tmp1.g", deep=True) - else: - tb.copyrows("tmp1.g") - tb.close() + msmd.close() + elif solint == "inf_fb3": + include_targets = include_targets.split(',') + include_scans = include_scans * len(include_targets) + else: + include_targets = [include_targets] - os.system("rm -rf tmp0.g") + selfcal_library[vis][solint]["include_scans"] = include_scans + selfcal_library[vis][solint]["include_targets"] = include_targets + print(solint,'Include scans: ', include_scans) + print(solint,'Include targets: ', include_targets) + print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) + for incl_scans, incl_targets in zip(include_scans, include_targets): + if solint == "inf_fb3": + gaincal_preapply_gaintable = [selfcal_plan[vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][incl_targets]] - tb.open("tmp1.g") - subt = tb.query("OBSERVATION_ID==0", sortlist="TIME,ANTENNA1") - copyt = subt.copy(destination_table, deep=True) - tb.close() - subt.close() - copyt.close() + for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: + print(incl_targets) + print(incl_scans) + print(vis,solint,gc_mode) + print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) + gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] + filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] - os.system("rm -rf tmp1.g") + if 'spw' in gaincal_combine: + if selfcal_library['spws_set'][vis].ndim == 1: + nspw_sets=1 + else: + nspw_sets=selfcal_library['spws_set'][vis].shape[0] + else: #only necessary to loop over gain cal when in inf_EB to avoid inf_EB solving for all spws + nspw_sets=1 - # Remove all of the scans that failed the triage above. - tb.open(destination_table, nomodify=False) - scans = tb.getcol("SCAN_NUMBER") + for i in range(nspw_sets): # run gaincal on each spw set to handle spectral scans + if 'spw' in gaincal_combine: + if nspw_sets == 1 and selfcal_library['spws_set'][vis].ndim == 1: + spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis].tolist()) + else: + spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis][i].tolist()) + else: + spwselect=selfcal_library[vis]['spws'] + gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' + print('prior to gaincal',gaintable_name, gc_mode) - bad_scans = np.repeat(True, scans.size) - for scan in include_scans[0].split(","): - bad_scans[scans == int(scan)] = False + if gc_mode != 'per_bb': + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) + else: + for baseband in selfcal_library[vis]['baseband'].keys(): + spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] + gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, + refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, + solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ + minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ + field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ + interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ + append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) + selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - tb.removerows(rownrs=np.where(bad_scans)[0]) - tb.flush() - tb.close() - else: - # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad. - gaincal_solmode="" - if selfcal_library['obstype'] == 'mosaic': - msmd.open(vis) - include_targets = [] - remove = [] - for incl_scan in include_scans: - scan_targets = [] - for fid in [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ - np.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))] if incl_scan == '' else \ - np.intersect1d(msmd.fieldsforscans(np.array(incl_scan.split(",")).astype(int)), \ - [selfcal_library['sub-fields-fid_map'][vis][fid] for fid in \ - numpy.intersect1d(selfcal_library['sub-fields-to-gaincal'],list(selfcal_library['sub-fields-fid_map'][vis].keys()))]): - # Note: because of the msmd above getting actual fids from the MS, we just need to append fid below. - scan_targets.append(fid) - - if len(scan_targets) > 0: - include_targets.append(','.join(np.array(scan_targets).astype(str))) - else: - remove.append(incl_scan) + selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name - for incl_scan in remove: - include_scans.remove(incl_scan) + # restricted gaincal table comparisons to only inf_EB prior to changes + # commenting because we want to do comparisons for other solints as well + #if 'inf_EB' not in solint: + # break - msmd.close() - else: - include_targets = [include_targets] - - selfcal_library[vis][solint]["include_scans"] = include_scans - selfcal_library[vis][solint]["include_targets"] = include_targets - print(solint,'Include scans: ', include_scans) - print(solint,'Include targets: ', include_targets) - print(solint,'Modes to attempt: ',selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']) - for incl_scans, incl_targets in zip(include_scans, include_targets): - for gc_mode in selfcal_plan[vis]['solint_settings'][solint]['modes_to_attempt']: - print(incl_targets) - print(incl_scans) - print(vis,solint,gc_mode) - print(selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine']) - gaincal_combine=selfcal_plan[vis]['solint_settings'][solint]['gaincal_combine'][gc_mode] - filename_append=selfcal_plan[vis]['solint_settings'][solint]['filename_append'][gc_mode] - - if 'spw' in gaincal_combine: - if selfcal_library['spws_set'][vis].ndim == 1: - nspw_sets=1 - else: - nspw_sets=selfcal_library['spws_set'][vis].shape[0] - else: #only necessary to loop over gain cal when in inf_EB to avoid inf_EB solving for all spws - nspw_sets=1 - for i in range(nspw_sets): # run gaincal on each spw set to handle spectral scans - if 'spw' in gaincal_combine: - if nspw_sets == 1 and selfcal_library['spws_set'][vis].ndim == 1: - spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis].tolist()) - else: - spwselect=','.join(str(spw) for spw in selfcal_library['spws_set'][vis][i].tolist()) - else: - spwselect=selfcal_library[vis]['spws'] - gaintable_name=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g' - print('prior to gaincal',gaintable_name, gc_mode) - if mode == "cocal" and selfcal_library['obstype'] != 'mosaic': - print('in cocal if statement') - for incl_target in include_targets[0].split(','): - #vis_to_gaincal = sanitize_string(incl_target)+vis.replace(sanitize_string(target),'') - vis_to_gaincal = vis - print('Vis to gaincal: ',vis_to_gaincal) - if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - else: - for baseband in selfcal_library[vis]['baseband'].keys(): - spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis_to_gaincal, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_target,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - else: - if gc_mode != 'per_bb': - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - else: - for baseband in selfcal_library[vis]['baseband'].keys(): - spwselect_bb=selfcal_library[vis]['baseband'][baseband]['spwstring'] - gcdict=call_gaincal(vis=vis, caltable=gaintable_name, gaintype=selfcal_plan[vis]['solint_settings'][solint]['gaincal_gaintype'][gc_mode], spw=spwselect_bb, - refant=selfcal_library[vis]['refant'], calmode=selfcal_plan['solmode'][iteration], solnorm=solnorm if not do_fallback_calonly else False, - solint=solint_interval.replace('_EB','').replace('_ap','').replace('scan_','').replace('_fb1','').replace('_fb2','').replace('_fb3',''),\ - minsnr=gaincal_minsnr if not do_fallback_calonly else max(gaincal_minsnr,gaincal_unflag_minsnr), minblperant=4,combine=gaincal_combine,\ - field=incl_targets,scan=incl_scans,gaintable=gaincal_preapply_gaintable,spwmap=gaincal_spwmap,uvrange=selfcal_library['uvrange'],\ - interp=gaincal_interpolate, solmode=gaincal_solmode, refantmode='flex',\ - append=os.path.exists(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+filename_append+'.g')) - selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][gc_mode].append(gcdict.copy()) - - selfcal_plan[vis]['solint_settings'][solint]['computed_gaintable'][gc_mode] = gaintable_name - - # restricted gaincal table comparisons to only inf_EB prior to changes - # commenting because we want to do comparisons for other solints as well - #if 'inf_EB' not in solint: - # break gaintable_prefix=sani_target+'_'+vis+'_'+band+'_' # assume that if there is only one mode to attempt, that it is combinespw and don't bother checking. if 'delay' not in solint: @@ -457,6 +245,7 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so selfcal_library[fid][vis][solint]['applycal_mode']=selfcal_plan['applycal_mode'][iteration]+'' selfcal_library[fid][vis][solint]['applycal_interpolate']=applycal_interpolate.copy() selfcal_library[fid][vis][solint]['solmode']=selfcal_plan['solmode'][iteration]+'' + selfcal_plan[vis]['solint_settings'][solint]['accepted_gaintable']=sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+selfcal_plan['solmode'][iteration]+'_'+preferred_mode+'.g' if 'combinespw' not in preferred_mode: # preapply all non spwcombine gain tables selfcal_plan[vis]['solint_settings'][solint]['preapply_this_gaintable']=True diff --git a/auto_selfcal/mosaic_helpers.py b/auto_selfcal/mosaic_helpers.py index 93543f2a..1db762f6 100644 --- a/auto_selfcal/mosaic_helpers.py +++ b/auto_selfcal/mosaic_helpers.py @@ -151,3 +151,121 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter return new_fields_to_selfcal + +def scan_inf_scan_combine(selfcal_library, vis, target, gaincalibrator_dict, guess_scan_combine=False): + if len(gaincalibrator_dict[vis]) > 0: + print("Determining scan_inf from calibrator scans in full MS") + scans = [] + intents = [] + times = [] + for t in gaincalibrator_dict[vis].keys(): + scans += [gaincalibrator_dict[vis][t]["scans"]] + intents += [np.repeat(gaincalibrator_dict[vis][t]["intent"],gaincalibrator_dict[vis][t]["scans"].size)] + times += [gaincalibrator_dict[vis][t]["times"]] + + times = np.concatenate(times) + order = np.argsort(times) + times = times[order] + + scans = np.concatenate(scans)[order] + intents = np.concatenate(intents)[order] + + is_gaincalibrator = intents == "phase" + scans = scans[is_gaincalibrator] + + msmd.open(vis) + if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': + scan_ids_for_target = msmd.scansforfield(target) + elif 'VLA' in selfcal_library['telescope']: + scan_ids_for_target=np.array([],dtype=int) + for fid in selfcal_library['sub-fields']: + if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): + field_id=selfcal_library['sub-fields-fid_map'][vis][fid] + scan_ids_for_target=np.append(scan_ids_for_target,msmd.scansforfield(field_id)) + + scan_ids_for_target.sort() # sort scans since they will be out of order + include_scans = [] + for iscan in range(scans.size-1): + scan_group = np.intersect1d(scan_ids_for_target, + np.array(list(range(scans[iscan]+1, scans[iscan+1])))).astype(str) + if scan_group.size > 0: + include_scans.append(",".join(scan_group)) + # PIPE-2741: if there are any scans before the first gain calibrator scan or after the last gain calibrator scan, catch them. + if scans.size > 0: + extra_scans = scan_ids_for_target[scan_ids_for_target > max(scans)] + if extra_scans.size > 0: + include_scans.append(','.join(extra_scans.astype(str))) + extra_scans = scan_ids_for_target[scan_ids_for_target < min(scans)] + if extra_scans.size > 0: + include_scans.append(','.join(extra_scans.astype(str))) + msmd.close() + elif guess_scan_combine: + print("Determining scan_inf from guessing where the calibrator scans were") + msmd.open(vis) + include_scans = [] + + #to guess at scan_inf combination for VLA look for breaks in the consecutive + #scan numbers and assume that the break is due to a calibrator scan + #Fetch scans for scan inf by collecting the field ids and running msmd.scansforfield + if 'VLA' in selfcal_library['telescope']: + scans=np.array([],dtype=int) + for fid in selfcal_library['sub-fields']: + if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): + field_id=selfcal_library['sub-fields-fid_map'][vis][fid] + scans=np.append(scans,msmd.scansforfield(field_id)) + + scans.sort() # sort scans since they will be out of order + scan_group='' + for iscan in range(scans.size): + if len(include_scans) > 0: + if str(scans[iscan]) in include_scans[-1]: + continue + if scan_group == '': + scan_group = str(int(scans[iscan])) + + if iscan < scans.size-1: + if scans[iscan+1] == scans[iscan]+1: + scan_group += ","+str(int(scans[iscan+1])) + else: + include_scans.append(scan_group) + scan_group='' + else: #write out the last scan group to include_scans + if scan_group != '': + include_scans.append(scan_group) + scan_group='' + + #guess scan_inf combination by getting all the scans for targets and do a simple grouping + if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': + scans = msmd.scansforfield(target) + + for iscan in range(scans.size): + if len(include_scans) > 0: + if str(scans[iscan]) in include_scans[-1]: + continue + + scan_group = str(scans[iscan]) + + if iscan < scans.size-1: + if msmd.fieldsforscan(scans[iscan+1]).size < msmd.fieldsforscan(scans[iscan]).size/3: + scan_group += ","+str(scans[iscan+1]) + + include_scans.append(scan_group) + + msmd.close() + else: + print("Not guessing where calibration scans are and justincluding all scans") + msmd.open(vis) + if selfcal_library['telescope'] == 'ALMA' or selfcal_library['telescope'] == 'ACA': + include_scans = [str(scan) for scan in msmd.scansforfield(target)] + elif 'VLA' in selfcal_library['telescope']: + scans=np.array([],dtype=int) + for fid in selfcal_library['sub-fields']: + if fid in selfcal_library['sub-fields-fid_map'][vis].keys(): + field_id=selfcal_library['sub-fields-fid_map'][vis][fid] + scans=np.append(scans,msmd.scansforfield(field_id)) + + scans.sort() # sort scans since they will be out of order + include_scans = [str(scan) for scan in scans] + msmd.close() + + return include_scans \ No newline at end of file diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py new file mode 100644 index 00000000..bd3b872e --- /dev/null +++ b/auto_selfcal/prepare_cocal.py @@ -0,0 +1,143 @@ +from casatasks import flagmanager, applycal, clearcal +from .prepare_selfcal import plan_selfcal_per_solint +import numpy as np +import copy +import os + +from .selfcal_helpers import sanitize_string + +def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_gaintype): + for target in selfcal_library: + for band in selfcal_library[target]: + for vis in selfcal_library[target][band]['vislist']: + if not os.path.exists(vis+'.flagversions/flags.fb_selfcal_starting_flags'): + flagmanager(vis=vis,mode='save',versionname='fb_selfcal_starting_flags') + else: + flagmanager(vis=vis,mode='restore',versionname='fb_selfcal_starting_flags') + + ## + ## For sources that self-calibration failed, try to use the inf_EB and the inf solutions from the sources that + ## were successful. + + for target in selfcal_library.keys(): + for band in selfcal_library[target].keys(): + print(target, selfcal_library[target][band]["final_solint"]) + + inf_EB_fields = {} + inf_fields = {} + fallback_fields = {} + calibrators = {} + + # Collect the potential targets and calibrators + for target in selfcal_library: + for band in selfcal_library[target]: + if band not in inf_EB_fields: + # Initialize the lists for this band. + inf_EB_fields[band] = [] + inf_fields[band] = [] + fallback_fields[band] = [] + + if selfcal_library[target][band]['SC_success'] and 'fb' not in selfcal_library[target][band]['final_solint']: + inf_EB_fields[band].append(target) + if selfcal_library[target][band]['final_solint'] != 'inf_EB' or \ + (selfcal_library[target][band]['final_solint'] == 'inf_EB' and + 'inf' not in selfcal_plan[target][band]['solints']): + inf_fields[band].append(target) + else: + fallback_fields[band].append(target) + else: + fallback_fields[band].append(target) + + + if len(fallback_fields[band]) > 0: + for target in selfcal_library: + for band in selfcal_library[target]: + # Update the relevant lists if we are going to do a fallback mode. + selfcal_plan[target][band]['solints'] += ["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"] + selfcal_plan[target][band]['solmode'] += ["p","p","p","p"] + selfcal_plan[target][band]['solint_interval'] += ["inf","inf","inf","inf"] + + plan_selfcal_per_solint(selfcal_library, + selfcal_plan, + optimize_spw_combine=False, + solints=["inf_EB_fb","inf_fb1","inf_fb2","inf_fb3"]) + + for target in selfcal_library: + for band in selfcal_library[target]: + for vis in selfcal_library[target][band]['vislist']: + """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"] = {} + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf"]) + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]) + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"])""" + + selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["preapply_solints"] = [] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["preapply_solints"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["preapply_solints"] = ["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_solints"] = ["inf_EB"] + + """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["modes_to_attempt"] = ["combinespw","combinespwpol"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["modes_to_attempt"] = ["combinespw"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["modes_to_attempt"] = ["combinespw"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["modes_to_attempt"] = ["combinespw"]""" + + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} + for cal_target in inf_fields[band]: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf']['accepted_gaintable'] + + selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] + selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] + + calibrators[band] = [inf_EB_fields[band], inf_fields[band], inf_fields[band], inf_fields[band]] + + selfcal_library[target][band]["nsigma"] = np.concatenate((selfcal_library[target][band]["nsigma"],[selfcal_library[target][band]["nsigma"][0], \ + selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1], selfcal_library[target][band]["nsigma"][1]])) + + print(inf_EB_fields) + print(inf_fields) + print(fallback_fields) + + ## + ## Reset the inf_EB informational dictionaries. + ## + + for target in selfcal_library: + for band in selfcal_library[target].keys(): + # If the target had a successful inf_EB solution, no need to reset. + if target in inf_EB_fields[band]: + continue + + for vis in selfcal_library[target][band]['vislist']: + selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']=inf_EB_gaincal_combine #'scan' + if selfcal_library[target][band]['obstype']=='mosaic': + selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']+=',field' + selfcal_plan[target][band][vis]['inf_EB_gaintype']=inf_EB_gaintype #G + selfcal_plan[target][band][vis]['inf_EB_fallback_mode']='' #'scan' + + ## The below sets the calibrations back to what they were prior to starting the fallback mode. It should not be needed + ## for the final version of the codue, but is used for testing. + + + for target in selfcal_library: + sani_target=sanitize_string(target) + for band in selfcal_library[target].keys(): + if target not in fallback_fields[band]: + continue + if 'gaintable_final' in selfcal_library[target][band]['vislist'][0]: + print('****************Reapplying previous solint solutions*************') + for vis in selfcal_library[target][band]['vislist']: + print('****************Applying '+str(selfcal_library[target][band][vis]['gaintable_final'])+' to '+target+' '+band+'*************') + ## NOTE: should this be selfcal_starting_flags instead of fb_selfcal_starting_flags ??? + flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) + applycal(vis=vis,\ + gaintable=selfcal_library[target][band][vis]['gaintable_final'],\ + interp=selfcal_library[target][band][vis]['applycal_interpolate_final'],\ + calwt=True,spwmap=selfcal_library[target][band][vis]['spwmap_final'],\ + applymode=selfcal_library[target][band][vis]['applycal_mode_final'],\ + field=target,spw=selfcal_library[target][band][vis]['spws']) + else: + print('****************Removing all calibrations for '+target+' '+band+'**************') + for vis in selfcal_library[target][band]['vislist']: + flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) + clearcal(vis=vis,field=target,spw=selfcal_library[target][band][vis]['spws']) + + return fallback_fields, calibrators \ No newline at end of file diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index b6616315..f36fd497 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -534,7 +534,7 @@ def default(self, obj): return selfcal_library, selfcal_plan, gaincalibrator_dict -def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=True): +def plan_selfcal_per_solint(selfcal_library, selfcal_plan, optimize_spw_combine=True, solints=None): # there are some extra keys in this dictionary that stem from how my thinking was orginally and how it evolved # the current philosophy is that for each solint it will specify how to apply each gain table, and if it should # be pre-applied for gaincal solves. Then in gaincal wrapper, the parameters for preapplying all tables and for applying @@ -559,8 +559,13 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T if selfcal_library[target][band][vis]['baseband'][baseband]['nspws']> maxspws_per_bb: maxspws_per_bb=selfcal_library[target][band][vis]['baseband'][baseband]['nspws']+0.0 - selfcal_plan[target][band][vis]['solint_settings']={} - for solint in selfcal_plan[target][band]['solints']: + if solints is not None: + use_solints = solints + else: + selfcal_plan[target][band][vis]['solint_settings']={} + use_solints = selfcal_plan[target][band]['solints'] + + for solint in use_solints: gaincal_combine='' filename_append='' selfcal_plan[target][band][vis]['solint_settings'][solint]={} @@ -582,8 +587,7 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T selfcal_plan[target][band][vis]['solint_settings'][solint]['final_mode']='' selfcal_plan[target][band][vis]['solint_settings'][solint]['accepted_gaintable']='' selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=[] - min_SNR_spw=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_spw'][solint]) - min_SNR_bb=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_bb'][solint]) + print('Nspws: {}, spws per BB: {}, basebands: {}'.format(nspws,maxspws_per_bb,n_basebands)) if selfcal_library[target][band]['telescope'] == 'VLBA' and 'delay' in solint and maxspws_per_bb > 1.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') @@ -591,11 +595,14 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('combinespw') if 'delay' in solint and n_basebands > 1: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') - if solint == 'inf_EB': + if "inf_EB" in solint: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('combinespwpol') - selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True + if solint == 'inf_EB': + selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True print('solint',solint,'N basebands: ',n_basebands, 'modes to attempt: ',selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']) - if 'spw' not in selfcal_plan[target][band][vis]['inf_EB_gaincal_combine']: + if 'spw' not in selfcal_plan[target][band][vis]['inf_EB_gaincal_combine'] and 'fb' not in solint: + min_SNR_spw=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_spw'][solint]) + min_SNR_bb=get_min_SNR_spw(selfcal_plan[target][band]['solint_snr_per_bb'][solint]) if min_SNR_spw > 2.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval @@ -603,12 +610,15 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval - if '_ap' in solint: - selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='ap' - else: - selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='p' - if solint != 'inf_EB' and optimize_spw_combine==False: - selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=['combinespw'] + + if '_ap' in solint: + selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='ap' + else: + selfcal_plan[target][band][vis]['solint_settings'][solint]['solmode']='p' + + if 'inf_EB' not in solint and optimize_spw_combine==False: + selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']=['combinespw'] + for mode in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: gaincal_combine='' if mode =='combinespw': @@ -627,11 +637,12 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan,optimize_spw_combine=T gaincal_combine='spw' filename_append='per_bb' selfcal_plan[target][band][vis]['solint_settings'][solint]['spwmap_for_mode']['per_bb']=selfcal_library[target][band][vis]['baseband_spwmap'] - if solint in ['inf_EB','inf_EB_delay','scan_inf','300s_ap']: + if solint in ['inf_EB','inf_EB_delay','scan_inf','300s_ap','inf_EB_fb']: if gaincal_combine!='': gaincal_combine+=',' gaincal_combine+='scan' - if solint in ['inf_EB','inf_EB_delay','scan_inf'] and selfcal_library[target][band]['obstype'] == 'mosaic': + if (solint in ['inf_EB','inf_EB_delay','scan_inf'] and selfcal_library[target][band]['obstype'] == 'mosaic') or \ + solint == "inf_EB_fb": gaincal_combine+=',field' selfcal_plan[target][band][vis]['solint_settings'][solint]['gaincal_combine'][mode]=gaincal_combine selfcal_plan[target][band][vis]['solint_settings'][solint]['filename_append'][mode]=filename_append diff --git a/auto_selfcal/selfcal_helpers.py b/auto_selfcal/selfcal_helpers.py index ddf3d06f..3eb1d7e4 100644 --- a/auto_selfcal/selfcal_helpers.py +++ b/auto_selfcal/selfcal_helpers.py @@ -3058,6 +3058,7 @@ def plot_ants_flagging_colored(filename,vis,gaintable): def get_flagged_solns_per_ant_from_dict(gc_dict_list,spwlist,vis): msmd.open(vis) antids=[] + print("len(gc_dict_list)", len(gc_dict_list)) for ant in [idant for idant in gc_dict_list[0]['solvestats']['spw'+str(spwlist[0])].keys() if idant.startswith('ant')]: antids.append(int(ant.replace('ant',''))) antids.sort() @@ -3124,6 +3125,8 @@ def plot_ants_flagging_colored_from_dict(filename,selfcal_library,selfcal_plan,s spwlist_pass=spwlist_bb.copy() + print(selfcal_plan.keys()) + print("plot_ants_flagging_colored_from_dict", solint, final_mode, len(selfcal_plan['solint_settings'][solint]['gaincal_return_dict'][final_mode])) names, offset_x, offset_y, apriori_flagged, nflagged, nunflagged, ntotal, fracflagged, nflagged_non_apriori, ntotal_non_apriori_flagged, fracflagged_non_apriori=get_flagged_solns_per_ant_from_dict(selfcal_plan['solint_settings'][solint]['gaincal_return_dict'][final_mode],spwlist_pass,vis) fracflagged=fracflagged_non_apriori print(fracflagged) @@ -3628,8 +3631,11 @@ def get_gaincalmode_flagging_stats(selfcal_library,selfcal_plan,vis,gaintable_pr selfcal_plan[vis]['solint_settings'][solint]['nflags_apriori'][mode],selfcal_plan[vis]['solint_settings'][solint]['nflags'][mode],selfcal_plan[vis]['solint_settings'][solint]['nunflagged'][mode],selfcal_plan[vis]['solint_settings'][solint]['ntotal'][mode],selfcal_plan[vis]['solint_settings'][solint]['fracflagged'][mode],selfcal_plan[vis]['solint_settings'][solint]['nflags_non_apriori'][mode],selfcal_plan[vis]['solint_settings'][solint]['ntotal_non_apriori'][mode],selfcal_plan[vis]['solint_settings'][solint]['fracflagged_non_apriori'][mode]=get_gaintable_flagging_stats(selfcal_plan[vis]['solint_settings'][solint]['gaincal_return_dict'][mode],spwlist_bb) else: baseband_scale=1.0 - if solint == 'inf_EB': + if 'inf_EB' in solint: n_solutions=1.0 + elif 'inf_EB_fb' in selfcal_plan[vis]['solint_settings'].keys(): + n_antennas=selfcal_plan[vis]['solint_settings']['inf_EB_fb']['ntotal_non_apriori'][coarsest_mode][0]/selfcal_plan[vis]['solint_settings'][solint]['polscale'][mode] + n_solutions=(selfcal_plan[vis]['solint_settings'][solint]['nflags_non_apriori'][coarsest_mode][0]+selfcal_plan[vis]['solint_settings'][solint]['nunflagged'][coarsest_mode][0])/n_antennas elif 'inf_EB' in selfcal_plan[vis]['solint_settings'].keys(): n_antennas=selfcal_plan[vis]['solint_settings']['inf_EB']['ntotal_non_apriori'][coarsest_mode][0]/selfcal_plan[vis]['solint_settings'][solint]['polscale'][mode] n_solutions=(selfcal_plan[vis]['solint_settings'][solint]['nflags_non_apriori'][coarsest_mode][0]+selfcal_plan[vis]['solint_settings'][solint]['nunflagged'][coarsest_mode][0])/n_antennas From 6697b711f1dd99ab4b4e0595980f7581c9f55596 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Thu, 14 May 2026 16:39:43 -0400 Subject: [PATCH 17/24] Preapply inf_EB for inf_fb3, not inf. Also, delete some unused, commented out code. --- auto_selfcal/prepare_cocal.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py index bd3b872e..40372bac 100644 --- a/auto_selfcal/prepare_cocal.py +++ b/auto_selfcal/prepare_cocal.py @@ -65,24 +65,14 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ for target in selfcal_library: for band in selfcal_library[target]: for vis in selfcal_library[target][band]['vislist']: - """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"] = {} - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf"]) - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]) - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"] = copy.deepcopy(selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"])""" - selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["preapply_solints"] = [] selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["preapply_solints"] = ["inf_EB_fb"] selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["preapply_solints"] = ["inf_EB"] selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_solints"] = ["inf_EB"] - """selfcal_plan[target][band][vis]['solint_settings']["inf_EB_fb"]["modes_to_attempt"] = ["combinespw","combinespwpol"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["modes_to_attempt"] = ["combinespw"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["modes_to_attempt"] = ["combinespw"] - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["modes_to_attempt"] = ["combinespw"]""" - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} for cal_target in inf_fields[band]: - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf']['accepted_gaintable'] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf_EB']['accepted_gaintable'] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] From f6abd08ae579ae3dce650b19ad892e3b166c170f Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Fri, 15 May 2026 10:22:12 -0400 Subject: [PATCH 18/24] We need a manual override of using the same pre-apply gaintable for gaincal as is used for applycal when doing the inf_fb* modes, because in those cases the same table might not be used for both. --- auto_selfcal/gaincal_wrapper.py | 11 ++++++++++- auto_selfcal/prepare_cocal.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/gaincal_wrapper.py b/auto_selfcal/gaincal_wrapper.py index fae8cc42..96401eb9 100644 --- a/auto_selfcal/gaincal_wrapper.py +++ b/auto_selfcal/gaincal_wrapper.py @@ -58,9 +58,18 @@ def gaincal_wrapper(selfcal_library, selfcal_plan, target, band, vis, solint, so gaincal_preapply_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) gaincal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) gaincal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) + + # If we manually specify that applycal should use a different solint reference than gaincal (cocal), change + # to that solint here. + if "applycal_solint" in selfcal_plan[vis]['solint_settings'][incl_solint]: + incl_solint = selfcal_plan[vis]['solint_settings'][incl_solint]["applycal_solint"] + applycal_spwmap.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_spwmap']) applycal_interpolate.append(selfcal_plan[vis]['solint_settings'][incl_solint]['applycal_interpolate']) - applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) + if solint == "inf_fb3": + applycal_gaintable.append(selfcal_plan[vis]['solint_settings']["inf_EB_fb"]['accepted_gaintable']) + else: + applycal_gaintable.append(selfcal_plan[vis]['solint_settings'][incl_solint]['accepted_gaintable']) if solint != 'inf_EB': diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py index 40372bac..2ae6f73c 100644 --- a/auto_selfcal/prepare_cocal.py +++ b/auto_selfcal/prepare_cocal.py @@ -70,6 +70,15 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["preapply_solints"] = ["inf_EB"] selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_solints"] = ["inf_EB"] + if selfcal_library[target][band]['SC_success']: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["applycal_solint"] = ["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["applycal_solint"] = ["inf_EB"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["applycal_solint"] = ["inf_EB"] + else: + selfcal_plan[target][band][vis]['solint_settings']["inf_fb1"]["applycal_solint"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb2"]["applycal_solint"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["applycal_solint"] = ["inf_EB_fb"] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} for cal_target in inf_fields[band]: selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf_EB']['accepted_gaintable'] From 2d97ecf5de2295249b6cee69748c954d2af4c631 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 15 May 2026 10:56:12 -0400 Subject: [PATCH 19/24] add sanitize_string() to target and cal_target when doing a .replace on a filename --- auto_selfcal/prepare_cocal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/prepare_cocal.py b/auto_selfcal/prepare_cocal.py index 40372bac..dca8fa7d 100644 --- a/auto_selfcal/prepare_cocal.py +++ b/auto_selfcal/prepare_cocal.py @@ -72,7 +72,7 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"] = {} for cal_target in inf_fields[band]: - selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(target, cal_target)]['solint_settings']['inf_EB']['accepted_gaintable'] + selfcal_plan[target][band][vis]['solint_settings']["inf_fb3"]["preapply_gaintable_dict"][cal_target] = selfcal_plan[cal_target][band][vis.replace(sanitize_string(target), sanitize_string(cal_target))]['solint_settings']['inf_EB']['accepted_gaintable'] selfcal_plan[target][band]['gaincal_combine'] += [selfcal_plan[target][band]['gaincal_combine'][0], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1], selfcal_plan[target][band]['gaincal_combine'][1]] selfcal_plan[target][band]['applycal_mode'] += [selfcal_plan[target][band]['applycal_mode'][0], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1], selfcal_plan[target][band]['applycal_mode'][1]] @@ -130,4 +130,4 @@ def prepare_cocal(selfcal_library, selfcal_plan, inf_EB_gaincal_combine, inf_EB_ flagmanager(vis=vis,mode='delete',versionname='fb_selfcal_starting_flags_'+sani_target) clearcal(vis=vis,field=target,spw=selfcal_library[target][band][vis]['spws']) - return fallback_fields, calibrators \ No newline at end of file + return fallback_fields, calibrators From 8337db497be4b4e95571c5c6961f09da50ff303f Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 15 May 2026 16:09:39 -0400 Subject: [PATCH 20/24] write selfcal_plan.pickle before starting selfcal and ensure single_spw data have a solution populated for all solints --- auto_selfcal/auto_selfcal.py | 4 +++- auto_selfcal/prepare_selfcal.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/auto_selfcal/auto_selfcal.py b/auto_selfcal/auto_selfcal.py index 1db79af9..4c386434 100644 --- a/auto_selfcal/auto_selfcal.py +++ b/auto_selfcal/auto_selfcal.py @@ -632,6 +632,7 @@ def auto_selfcal( with open('selfcal_library.pickle', 'wb') as handle: pickle.dump(selfcal_library, handle, protocol=pickle.HIGHEST_PROTOCOL) + import json class NpEncoder(json.JSONEncoder): @@ -711,7 +712,8 @@ def default(self, obj): with open('selfcal_library.pickle', 'wb') as handle: pickle.dump(selfcal_library, handle, protocol=pickle.HIGHEST_PROTOCOL) - + with open('selfcal_plan.pickle', 'wb') as handle: + pickle.dump(selfcal_plan, handle, protocol=pickle.HIGHEST_PROTOCOL) ## ## Begin Self-cal loops diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index f36fd497..2cc058ef 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -606,6 +606,8 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan, optimize_spw_combine= if min_SNR_spw > 2.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval + elif n_spws == 1: # make sure we populate when only single spectral window, otherwise can errorr + selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') if min_SNR_bb > 2.0 and maxspws_per_bb > 1.0 and selfcal_library[target][band]['spectral_scan']==False and n_basebands > 1: # only do the per baseband solutions if there are more than 1 spw and more than 1 baseband if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_bb') From 1d8cab56ef887ed891f77743e370d18d978caf6d Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Mon, 18 May 2026 16:49:22 -0400 Subject: [PATCH 21/24] Add dataset to test cocal online. --- auto_selfcal/tests/test_auto_selfcal.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 5f0aafcb..26f63104 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -87,7 +87,8 @@ def test_benchmark(tmp_path, dataset): pytest.param("2018.1.01284.S_HOPS-384.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCvDRlo5TabTZ8qoP-tscZnAdJZOuCKQew3ewIH5bZzZF0?e=XWOGxH&download=1', id="2018.1.01284.S_HOPS-384"), pytest.param("Band8-7m-2.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAnFUlx_SR0SLdj_vX8MVUWAbMu9BveHCsb9NB07-OD3bo?e=TNBTXq&download=1', id="Band8-7m-2"), pytest.param("M82-C-conf-C-band_small.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCT-CkbAW7YT5LIev5CrDSOAZt5uC_tUIReMwlA14Tu9y4?e=2zF126&download=1', id="M82-C-conf-C-band_small"), - pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic') + pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic'), + pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=KHkjlK', id='Band8-7m-cocal') ] ) def test_on_github(tmp_path, request, zip_file, link): @@ -100,7 +101,7 @@ def test_on_github(tmp_path, request, zip_file, link): os.system(f'tar xf {zip_file}') os.system(f'rm -rf {zip_file}') - auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True) + auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True, allow_cocal=True) os.system('rm -rf *.ms*') # Delete MS files as space is limited on GitHub. From 4c5f8d6eeca94108d8d891e40edbf995965ae80f Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 19 May 2026 15:57:50 +0000 Subject: [PATCH 22/24] Fix 'n_spws' => 'nspws' bug; update cocal test link --- auto_selfcal/prepare_selfcal.py | 2 +- auto_selfcal/tests/test_auto_selfcal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_selfcal/prepare_selfcal.py b/auto_selfcal/prepare_selfcal.py index 2cc058ef..22226f12 100644 --- a/auto_selfcal/prepare_selfcal.py +++ b/auto_selfcal/prepare_selfcal.py @@ -606,7 +606,7 @@ def plan_selfcal_per_solint(selfcal_library, selfcal_plan, optimize_spw_combine= if min_SNR_spw > 2.0: selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') #selfcal_plan[target][band][vis]['solint_settings'][solint]['preapply_this_gaintable']=True # leave default to off and have it decide after eval - elif n_spws == 1: # make sure we populate when only single spectral window, otherwise can errorr + elif nspws == 1: # make sure we populate when only single spectral window, otherwise can errorr selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt'].append('per_spw') if min_SNR_bb > 2.0 and maxspws_per_bb > 1.0 and selfcal_library[target][band]['spectral_scan']==False and n_basebands > 1: # only do the per baseband solutions if there are more than 1 spw and more than 1 baseband if 'per_bb' not in selfcal_plan[target][band][vis]['solint_settings'][solint]['modes_to_attempt']: diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 6b1a8be8..cda0e8ad 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -88,7 +88,7 @@ def test_benchmark(tmp_path, dataset): pytest.param("Band8-7m-2.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAnFUlx_SR0SLdj_vX8MVUWAbMu9BveHCsb9NB07-OD3bo?e=TNBTXq&download=1', id="Band8-7m-2"), pytest.param("M82-C-conf-C-band_small.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCT-CkbAW7YT5LIev5CrDSOAZt5uC_tUIReMwlA14Tu9y4?e=2zF126&download=1', id="M82-C-conf-C-band_small"), pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic'), - pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=KHkjlK', id='Band8-7m-cocal'), + pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=M7gnSx&download=1', id='Band8-7m-cocal'), pytest.param("2019.1.00691.S_SB.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAoeNLrwb1aSrWCfo8ekE6NAXVJ4Qlpps0gdAeY1U9Axn0?e=MMXCaT&download=1', id='2019.1.00691.S_SB') ] ) From e7d5d7c5aff7f58dd908242fa95cc8d2d2d50fe5 Mon Sep 17 00:00:00 2001 From: Patrick Sheehan Date: Tue, 19 May 2026 16:45:55 -0500 Subject: [PATCH 23/24] Add reference values/weblog for online cocal test --- auto_selfcal/tests/test_auto_selfcal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index cda0e8ad..e19b4719 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -88,7 +88,7 @@ def test_benchmark(tmp_path, dataset): pytest.param("Band8-7m-2.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAnFUlx_SR0SLdj_vX8MVUWAbMu9BveHCsb9NB07-OD3bo?e=TNBTXq&download=1', id="Band8-7m-2"), pytest.param("M82-C-conf-C-band_small.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQCT-CkbAW7YT5LIev5CrDSOAZt5uC_tUIReMwlA14Tu9y4?e=2zF126&download=1', id="M82-C-conf-C-band_small"), pytest.param("K-band-mini-mosaic.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQBCQc-REEGYQrBwBu2F9uUTAfpCRQq1gYAPO18e-CL_IUk?e=7adSCD&download=1', id='K-band-mini-mosaic'), - pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=M7gnSx&download=1', id='Band8-7m-cocal'), + pytest.param("Band8-7m-cocal.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAjUGmKxJpoSq25LNp6qnhUAfhcQ-o3RnP7q13r0OgPCfc?e=rX9LaN&download=1', id='Band8-7m-cocal'), pytest.param("2019.1.00691.S_SB.tar.gz", 'https://nrao-my.sharepoint.com/:u:/g/personal/psheehan_nrao_edu/IQAoeNLrwb1aSrWCfo8ekE6NAXVJ4Qlpps0gdAeY1U9Axn0?e=MMXCaT&download=1', id='2019.1.00691.S_SB') ] ) From 833ed1756d57ae47a33c3a8d8b52bec2a9281714 Mon Sep 17 00:00:00 2001 From: John Tobin Date: Fri, 22 May 2026 14:02:44 -0400 Subject: [PATCH 24/24] re-enable cocal on github tests --- auto_selfcal/tests/test_auto_selfcal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_selfcal/tests/test_auto_selfcal.py b/auto_selfcal/tests/test_auto_selfcal.py index 025083c8..89102473 100644 --- a/auto_selfcal/tests/test_auto_selfcal.py +++ b/auto_selfcal/tests/test_auto_selfcal.py @@ -122,7 +122,7 @@ def test_on_github(tmp_path, request, zip_file, link): auto_selfcal(iscalibrator=True, refant='FD,NL,PT', do_delay_cal=True, shorter_amp_solints=True, targets='J1154+6022', applytargets='J1203+6031', imsize=640, cell='0.0002arcsec') else: - auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True, usermask=usermask, usermodel=usermodel) + auto_selfcal(sort_targets_and_EBs=True, align_EBs=True, weblog=True, allow_cocal=True, usermask=usermask, usermodel=usermodel) os.system('rm -rf *.ms*') # Delete MS files as space is limited on GitHub.