Enable calculating solints on a per-EB basis

auto_selfcal/__main__.py

@@ -26,6 +26,7 @@
 parser.add_argument('--do_amp_selfcal', default=True)
 parser.add_argument('--usermask', default={}, type=ast.literal_eval)  # require that it is a CRTF region (CASA region format)
 parser.add_argument('--usermodel', default={}, type=ast.literal_eval) 
+parser.add_argument('--uniform_solints', default=False)
 parser.add_argument('--inf_EB_gaincal_combine', default='scan', type=str)  # should we get rid of this option?
 parser.add_argument('--inf_EB_gaintype', default='G', type=str)
 parser.add_argument('--inf_EB_override', action='store_true')

auto_selfcal/auto_selfcal.py

@@ -36,6 +36,7 @@ def auto_selfcal(
         inf_EB_gaintype='G',
         inf_EB_override=False,
         optimize_spw_combine=True,      # if False, will not attempt per spw or per baseband solutions for any solint except inf_EB
+        uniform_solints=False,
         gaincal_minsnr=2.0,
         gaincal_unflag_minsnr=5.0,
         minsnr_to_proceed=2.95,
@@ -449,7 +450,7 @@ def auto_selfcal(
                     vis_for_targets[target][band]['vislist'], 
                     spectral_average=spectral_average, sort_targets_and_EBs=sort_targets_and_EBs, scale_fov=scale_fov, inf_EB_gaincal_combine=inf_EB_gaincal_combine, 
                     inf_EB_gaintype=inf_EB_gaintype, apply_cal_mode_default=apply_cal_mode_default, do_amp_selfcal=do_amp_selfcal, 
-                    usermask=usermask, usermodel=usermodel,guess_scan_combine=guess_scan_combine,max_solint=max_solint,
+                    uniform_solints=uniform_solints, usermask=usermask, usermodel=usermodel,guess_scan_combine=guess_scan_combine,max_solint=max_solint,
                     iscalibrator=iscalibrator, do_delay_cal=do_delay_cal, shorter_amp_solints=shorter_amp_solints,
                     imsize=imsize, cell=cell, refant=refant, debug=debug)
 

auto_selfcal/image_analysis_helpers.py

@@ -12,7 +12,12 @@ def get_image_stats(image, mask, backup_mask, selfcal_library, use_nfmask, solin
     else:
        SNR_NF, RMS_NF = SNR, RMS
 
-    for vis in selfcal_library['vislist']:
+    if suffix in ['dirty','orig','initial','final']:
+        vislist = selfcal_library['vislist']
+    else:
+        vislist = selfcal_library['vislist-to-gaincal']
+
+    for vis in vislist:
        if suffix in ['dirty','orig','initial','final']:
            if spw == 'all':
                update_dict = selfcal_library

auto_selfcal/mosaic_helpers.py

@@ -1,14 +1,14 @@
 import numpy as np
 from .selfcal_helpers import *
 
-def evaluate_subfields_to_gaincal(selfcal_library, target, band, solint, iteration, solmode, solints, selfcal_plan, 
+def evaluate_subfields_to_gaincal(vis, selfcal_library, target, band, solint, iteration, solmode, solints, selfcal_plan, 
         minsnr_to_proceed, allow_gain_interpolation=False,):
 
      sani_target=sanitize_string(target)
 
      # Fields that don't have any mask in the primary beam should be removed from consideration, as their models are likely bad.
      new_fields_to_selfcal = []
-     for fid in selfcal_library['sub-fields-to-selfcal']:
+     for fid in selfcal_library[vis]['sub-fields-to-selfcal']:
          os.system('rm -rf test*.mask')
          tmp_SNR_NF,tmp_RMS_NF=estimate_near_field_SNR(sani_target+'_field_'+str(fid)+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0', \
                  las=selfcal_library['LAS'], mosaic_sub_field=True, save_near_field_mask=False)
@@ -41,8 +41,8 @@ def evaluate_subfields_to_gaincal(selfcal_library, target, band, solint, iterati
          if not checkmask(sani_target+'_field_'+str(fid)+'_'+band+'_'+solint+'_'+str(iteration)+'.image.tt0'):
              print("Removing field "+str(fid)+" from gaincal because there is no signal within the primary beam.")
              skip_reason = "No signal"
-         elif selfcal_plan[fid]['solint_snr_per_field'][solints[iteration]] < minsnr_to_proceed and \
-                 solint not in ['inf_EB','scan_inf']:
+         elif selfcal_plan[fid][vis]['solint_snr_per_field'][solints[iteration]] < minsnr_to_proceed and \
+                 selfcal_plan[vis]['solint_settings'][solint]['sub-name'] not in ['inf_EB','scan_inf']:
              print("Removing field "+str(fid)+" from gaincal because the estimated solint snr is too low.")
              skip_reason = "Estimated SNR"
          elif updated_intflux > selfcal_library['flux_threshold'] * original_intflux:
@@ -53,45 +53,44 @@ def evaluate_subfields_to_gaincal(selfcal_library, target, band, solint, iterati
              new_fields_to_selfcal.append(fid)
 
          if fid not in new_fields_to_selfcal and solint != "inf_EB" and not allow_gain_interpolation:
-             for vis in selfcal_library[fid]['vislist']:
-                 #selfcal_library[fid][vis][solint]['interpolated_gains'] = True
-                 #selfcal_library[fid]['Stop_Reason'] = "Gaincal solutions would be interpolated"
-                 selfcal_library[fid]['Stop_Reason'] = skip_reason
-                 selfcal_library[fid][vis][solint]['Pass'] = "None"
-                 selfcal_library[fid][vis][solint]['Fail_Reason'] = skip_reason
+             #selfcal_library[fid][vis][solint]['interpolated_gains'] = True
+             #selfcal_library[fid]['Stop_Reason'] = "Gaincal solutions would be interpolated"
+             selfcal_library[fid][vis]['Stop_Reason'] = skip_reason
+             selfcal_library[fid][vis][solint]['Pass'] = "None"
+             selfcal_library[fid][vis][solint]['Fail_Reason'] = skip_reason
 
      return new_fields_to_selfcal
 
 
 
 
-def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iteration, solmode, allow_gain_interpolation=False):
+def evaluate_subfields_after_gaincal(vis, selfcal_library, selfcal_plan, target, band, solint, iteration, solmode, allow_gain_interpolation=False):
 
-     new_fields_to_selfcal = selfcal_library['sub-fields-to-selfcal'].copy()
+     new_fields_to_selfcal = selfcal_library[vis]['sub-fields-to-selfcal'].copy()
 
      sani_target=sanitize_string(target)
 
      if selfcal_library['obstype'] == 'mosaic' and ((solint != "inf_EB" and not allow_gain_interpolation) or \
              (allow_gain_interpolation and "inf" not in solint)):
         # With gaincal done and bad fields removed from gain tables if necessary, check whether any fields should no longer be selfcal'd
         # because they have too much interpolation.
-        for vis in selfcal_library['vislist']:
+        #for vis in selfcal_library['vislist']:
+        if True:
             ## If an EB had no fields to gaincal on, remove all fields in that EB from being selfcal'd as there is no calibration available
             ## in this EB.
-            if np.intersect1d(selfcal_library['sub-fields-to-gaincal'],\
+            if np.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],\
                     list(selfcal_library['sub-fields-fid_map'][vis].keys())).size == 0:
                 for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())):
                     new_fields_to_selfcal.remove(fid)
 
-                    selfcal_library[fid]['Stop_Reason'] = 'No viable calibrator fields in at least 1 EB'
-                    for v in selfcal_library[fid]['vislist']:
-                        selfcal_library[fid][v][solint]['Pass'] = 'None'
-                        if 'Fail_Reason' in selfcal_library[fid][v][solint]:
-                            selfcal_library[fid][v][solint]['Fail_Reason'] += '; '
-                        else:
-                            selfcal_library[fid][v][solint]['Fail_Reason'] = ''
-                        selfcal_library[fid][v][solint]['Fail_Reason'] += 'No viable fields'
-                continue
+                    selfcal_library[fid][vis]['Stop_Reason'] = 'No viable calibrator fields in at least 1 EB'
+                    selfcal_library[fid][vis][solint]['Pass'] = 'None'
+                    if 'Fail_Reason' in selfcal_library[fid][vis][solint]:
+                        selfcal_library[fid][vis][solint]['Fail_Reason'] += '; '
+                    else:
+                        selfcal_library[fid][vis][solint]['Fail_Reason'] = ''
+                    selfcal_library[fid][vis][solint]['Fail_Reason'] += 'No viable fields'
+                return new_fields_to_selfcal
             ## NEXT TO DO: check % of flagged solutions - DONE, see above
             ## After that enable option for interpolation through inf - DONE
             tb.open(sani_target+'_'+vis+'_'+band+'_'+solint+'_'+str(iteration)+'_'+solmode[iteration]+'_'+
@@ -100,7 +99,7 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter
             scans = tb.getcol("SCAN_NUMBER")
 
             for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())):
-                if solint == "scan_inf":
+                if selfcal_plan[vis]['solint_settings'][solint]['sub-name'] == "scan_inf":
                     msmd.open(vis)
                     cals_for_scan = []
                     total_cals_for_scan = []
@@ -125,29 +124,29 @@ def evaluate_subfields_after_gaincal(selfcal_library, target, band, solint, iter
                         new_fields_to_selfcal.remove(fid)
 
                 if fid not in new_fields_to_selfcal:
-                    # We need to update all the EBs, not just the one that failed.
-                    for v in selfcal_library[fid]['vislist']:
-                        selfcal_library[fid][v][solint]['Pass'] = 'None'
-                        if allow_gain_interpolation:
-                            selfcal_library[fid][v][solint]['Fail_Reason'] = 'Interpolation beyond inf'
-                        else:
-                            selfcal_library[fid][v][solint]['Fail_Reason'] = 'Bad gaincal solutions'
+                    selfcal_library[fid][vis][solint]['Pass'] = 'None'
+                    if allow_gain_interpolation:
+                        selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Interpolation beyond inf'
+                    else:
+                        selfcal_library[fid][vis][solint]['Fail_Reason'] = 'Bad gaincal solutions'
 
 
             tb.close()
      elif selfcal_library['obstype'] == 'mosaic' and solint == "inf_EB":
         ## If an EB had no fields to gaincal on, remove all fields in that EB from being selfcal'd as there is no calibration available
         ## in this EB.
-        for vis in selfcal_library['vislist']:
-            if np.intersect1d(selfcal_library['sub-fields-to-gaincal'],\
+        #for vis in selfcal_library['vislist']:
+        if True:
+            if np.intersect1d(selfcal_library[vis]['sub-fields-to-gaincal'],\
                     list(selfcal_library['sub-fields-fid_map'][vis].keys())).size == 0:
                 for fid in np.intersect1d(new_fields_to_selfcal,list(selfcal_library['sub-fields-fid_map'][vis].keys())):
                     new_fields_to_selfcal.remove(fid)
 
-                    selfcal_library[fid]['Stop_Reason'] = 'No viable calibrator fields for inf_EB in at least 1 EB'
-                    for v in selfcal_library[fid]['vislist']:
-                        selfcal_library[fid][v][solint]['Pass'] = 'None'
-                        selfcal_library[fid][v][solint]['Fail_Reason'] = 'No viable inf_EB fields'
+                    selfcal_library[fid][vis]['Stop_Reason'] = 'No viable calibrator fields for inf_EB in at least 1 EB'
+                    selfcal_library[fid][vis][solint]['Pass'] = 'None'
+                    selfcal_library[fid][vis][solint]['Fail_Reason'] = 'No viable inf_EB fields'
+
+     print("new_fields_to_selfcal", new_fields_to_selfcal)
 
      return new_fields_to_selfcal
 

auto_selfcal/original_ms_helpers.py

@@ -75,7 +75,7 @@ def applycal_to_orig_MSes(selfcal_library='selfcal_library.pickle', write_only=T
        for band in selfcal_library[target].keys():
           if selfcal_library[target][band]['SC_success']:
              for vis in selfcal_library[target][band]['vislist']: 
-                solint=selfcal_library[target][band]['final_solint']
+                solint=selfcal_library[target][band][vis]['final_solint']
                 iteration=selfcal_library[target][band][vis][solint]['iteration']    
 
                 # Write the line to the command log