Fakered

pro/.gitignore

@@ -1,3 +1,3 @@
 *~
 *.bak
-*.bak*
+*.bak*

pro/fakered_addstar.pro

@@ -95,12 +95,12 @@ nmulti          = getparam(nmulti          , 'nmulti'          , setup, '1'
 scriptsdir      = getparam(scriptsdir      , 'scriptsdir'      , setup, ''            , logfile)
 sepfielddir     = getparam(sepfielddir     , 'sepfielddir'     , setup, '0'           , logfile, /bool)
 sepchipdir      = getparam(sepchipdir      , 'sepchipdir'      , setup, '0'           , logfile, /bool)
-starssplit      = getparam(starssplit      , 'starssplit'      , setup, '0'           , logfile, /bool)
+starssplit      = getparam(starssplit      , 'starssplit'      , setup, '0'           , logfile)
 starsfile       = getparam(starsfile       , 'starsfile'       , setup, ''            , logfile)
 starscols       = getparam(starscols       , 'starscols'       , setup, ''            , logfile)
 datadir         = getparam(datadir         , 'datadir'         , setup, '.'           , logfile)
 datatransfer    = getparam(datatransfer    , 'datatransfer'    , setup, 'skip'        , logfile)
-chipsfile       = getparam(chipsfile       , 'chipssfile'      , setup, '*chips.fits' , logfile)
+chipsfile       = getparam(chipsfile       , 'chipsfile'       , setup, '*chips.fits' , logfile)
 maxccdsize      = getparam(maxccdsize      , 'maxccdsize'      , setup, '2048,4096'   , logfile)
 magext          = getparam(magext          , 'magext'          , setup, '0'           , logfile)
 radcent         = getparam(radcent         , 'radcent'         , setup, '*'           , logfile)

pro/photred_commonsources_global.pro

@@ -206,7 +206,7 @@ printlog,logfile,'Creating cmn.coo and cmn.ap for ',strtrim(nfieldfiles,2),' fil
 
 ; Now run PBS_DAEMON.PRO
 if n_elements(cmd) gt 0 then $
-  PBS_DAEMON,cmd,nmulti=nmulti,prefix='dcmn',hyperthread=hyperthread,/idle,waittime=5,scriptsdir=scriptsdir
+  PBS_DAEMON,cmd,nmulti=nmulti,prefix='dcmn',hyperthread=hyperthread,/idle,waittime=1,scriptsdir=scriptsdir
 
 ; Now concatenate and merge all of the catalogs
 ;----------------------------------------------

scripts/fakered_addstar.py

@@ -342,6 +342,8 @@ def get_input_stars(fn_stars, cols_order, stars_shuffle):
                 stars[col] = stars_raw[pos]
                 if col == STAR_ID:
                 	  print ("Column " + str(pos+1) + ": stars ID. Total stars: " + str(len(stars[col])))
+                elif col == XPOS_COL or col == YPOS_COL:
+                	  print ("Column " + str(pos+1) + ": Position " + col + ". Total stars: " + str(len(stars[col])))
                 else:
                 	  print ("Column " + str(pos+1) + ": filter " + col + ". Total stars: " + str(len(stars[col])))
             else:
@@ -490,7 +492,7 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
 
      offset -- random values of offset [offx, offy]
 
-      shift -- shift in magnitude to move the stars in the CMD after every iteration
+      shift -- shift in magnitude to move the stars in the CMD after every iteration (NOT USED!!)
 
     """
 
@@ -511,7 +513,7 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
         offset = [random.uniform(-lim,lim), random.uniform(-lim,lim)]
     if shift is None:
         lim = 0.07
-        shift = random.uniform(-lim,lim)
+        shift = random.uniform(-lim,lim) # NOT USED!!!
 
     # Get dimensions
     radcent /= 2
@@ -582,9 +584,9 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
     if max_iters > 0 and max_iters < addmax:
         print_warning("Limiting number of Mocks to " + str(max_iters))
         addmax = max_iters
-    max_files = int(round(999/nimages)-1)
-    if addmax > max_files:
-       exit_error_msg('Too many open files\nMaximun number is: ' + str(max_files) + "\nDecrease the total number of artificial stars", False)
+    #max_files = int(round(999/nimages)-1)
+    #if addmax > max_files:
+    #   exit_error_msg('Too many open files\nMaximun number is: ' + str(max_files) + "\nDecrease the total number of artificial stars", False)
 
 
     #-----------------------------------------------------------------------------
@@ -704,14 +706,21 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
     xaux = 0 
     yaux = 0 
 
+    # See whether we have to create the GRID or the positions are given
+    GIVEN_POS = XPOS_COL in data_in and YPOS_COL in data_in
+
 
     rowpos = 0
     print ('Writing set of files %s...' % numcaj)
     for star_pos,star_id in enumerate(data_in[STAR_ID]):
 
-        # Get initial positions
-        xpos_init = xmin + (2 * (xaux + 1) - 1) * radcent + offx
-        ypos_init = ymin + (2 * (yaux + 1) - 1) * radcent + offy
+        if GIVEN_POS:
+            xpos_init = data_in[XPOS_COL][star_pos]
+            ypos_init = data_in[YPOS_COL][star_pos]
+        else:
+            # Get initial positions
+            xpos_init = xmin + (2 * (xaux + 1) - 1) * radcent + offx
+            ypos_init = ymin + (2 * (yaux + 1) - 1) * radcent + offy
 
         # Write line in file .mag for the current star. Format:
         # ID  XREF YREF   F1 F1ERR  F2 F2ERR ... FN FNERR  CHI  SHARP  FLAG  PROB
@@ -721,7 +730,11 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
         # Second and consecutive lines has 27 spaces before first data
         MAX_MAG_COLS = 12
         num_cols = 0
-        mag_line = "%9d %8.3f %8.3f" % (star_id, xpos_init, ypos_init)
+        #mag_line = "%9d %8.3f %8.3f" % (star_id, xpos_init, ypos_init)
+        mag_line = "%9d" % star_id
+        # if we have negative numbers of 4 digits, remove one decimal to leave place for sign
+        mag_line += " %8.3f" % xpos_init if xpos_init > -1000.0 else " %7.2f" % xpos_init
+        mag_line += " %8.3f" % ypos_init if ypos_init > -1000.0 else " %7.2f" % ypos_init
 
         # -------------------------------------------------------------------------
 
@@ -777,16 +790,16 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
 
             # Get EBV data from all images to later calculate the average (needed to write calmag files)
             chip_img = chip_info[nimg]
-            flt = chip_img['FILTER']
-            data_ebv[flt] += chip_img['EBV']
-            count_ebv[flt] += 1
+            flt_img = chip_img['FILTER']
+            data_ebv[flt_img] += chip_img['EBV']
+            count_ebv[flt_img] += 1
 
             # After transformations, SAVE TO FILE (xpos, ypos) ONLY if it is inside rectangle given by corners
             if in_rectangle (corners, [xpos, ypos]):
                 # This star is INSIDE the limits, transform the MAGNITUDE and write it to ADD
                 try:
                     # Get calibrated magnitude after applying absorption and distance modulus
-                    calmag = get_calmag(data_in[flt][star_pos], distance, chip_img['EBV'], magext[flt])
+                    calmag = get_calmag(data_in[flt_img][star_pos], distance, chip_img['EBV'], magext[flt_img])
                     # Get calibrated color: colsign * (band - colband)
                     color = chip_img['COLSIGN'] * (data_in[chip_img['BAND']][star_pos] - data_in[chip_img['COLBAND']][star_pos]) 
                     # PERFORM MAGNITUDE TRANSFORMATION
@@ -816,7 +829,7 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
             # END MAG file with fixed fields: CHI SHARP FLAG PROB
             num_cols += 2
             mag_line += "   1.0000   0.0000"
-            if num_cols % MAX_MAG_COLS == 0: mag_line += "\n%23s" % ''
+            if num_cols % MAX_MAG_COLS == 0: mag_line += "\n%25s" % ''
             mag_line += "    0   1.00\n"
             #mag_line += "   1.0000   0.0000    0   1.00\n"
             f_mag.write(mag_line)
@@ -829,7 +842,10 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
             for flt in filters:
                 inimag = data_in[flt][star_pos]  
                 # EBV depends on the image, get the average value
-                ebv_val = data_ebv[flt]/count_ebv[flt]
+                if count_ebv[flt] > 0:
+                  ebv_val = data_ebv[flt]/count_ebv[flt]
+                else:
+                  ebv_val = 0
                 calmag = get_calmag(inimag, distance, ebv_val, magext[flt])
                 out_inimag += " %10.3f" % inimag
                 out_calmag += " %10.3f" % calmag
@@ -839,59 +855,60 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m
 
 #-----------------------------------------------------------------------------
 
-        # UPDATE COUNTERS: Get ready for next iteration over the stars loop 
-        # (Go to next star and next col of CCD)
-        cn -= 1
-        yaux += 1
-    		# If we are in the last col, go to first position of next row 
-        if (yaux >= ysize):
-            xaux += XSEP
-            rowpos += 1
-            # If we are in even row, y position is half in order to distribute stars in triangles
-            if (rowpos % 2) == 0:
-                yaux = 0.0
-            else:
-                yaux = 0.5
+        if not GIVEN_POS:
+            # UPDATE COUNTERS: Get ready for next iteration over the stars loop 
+            # (Go to next star and next col of CCD)
+            cn -= 1
+            yaux += 1
+        		# If we are in the last col, go to first position of next row 
+            if (yaux >= ysize):
+                xaux += XSEP
+                rowpos += 1
+                # If we are in even row, y position is half in order to distribute stars in triangles
+                if (rowpos % 2) == 0:
+                    yaux = 0.0
+                else:
+                    yaux = 0.5
 
 
-        # Check if CCD is FULL (xaux >= xsize) and there are still some stars to process (cn > 0)
-        if (xaux >= xsize) and (cn > 0):
+            # Check if CCD is FULL (xaux >= xsize) and there are still some stars to process (cn > 0)
+            if (xaux >= xsize) and (cn > 0):
 
-            # Check if we have reached the limit of iters (Mocks). If so, quit loop!
-            if numcaj >= addmax: break        
+                # Check if we have reached the limit of iters (Mocks). If so, quit loop!
+                if numcaj >= addmax: break        
 
-            # CCD IS FULL, write in new files
-            xaux = 0
-            yaux = 0
-            numcaj += 1
-            nfopen += 1
+                # CCD IS FULL, write in new files
+                xaux = 0
+                yaux = 0
+                numcaj += 1
+                nfopen += 1
 
-            # Close all open files
-            for f in fidimag:
-                if not f.closed:
-                    f.close()
-            # Open new files
-            for k in xrange(nimages):
-                fidimag.append(open(filenames[nfopen * nimages + k], 'w'))
+                # Close all open files
+                for f in fidimag:
+                    if not f.closed:
+                        f.close()
+                # Open new files
+                for k in xrange(nimages):
+                    fidimag.append(open(filenames[nfopen * nimages + k], 'w'))
 
 
-            # Close previous MAG file and create new one for current iteration
-            f_mag.close()
-            fn_mag = get_iter_filename(mch_fnames[0], field, chip, numcaj, ".mag").replace("_", "-add_")
-            f_mag = open(fn_mag, "w+")
-            f_mag.write(mag_header)
+                # Close previous MAG file and create new one for current iteration
+                f_mag.close()
+                fn_mag = get_iter_filename(mch_fnames[0], field, chip, numcaj, ".mag").replace("_", "-add_")
+                f_mag = open(fn_mag, "w+")
+                f_mag.write(mag_header)
 
-            f_inimag.close()
-            fn_inimag = "%sM%d-inimag_%s.mag" % (field, numcaj, chip)
-            f_inimag = open(fn_inimag, "w+")
-            f_inimag.write(inimag_header)
+                f_inimag.close()
+                fn_inimag = "%sM%d-inimag_%s.mag" % (field, numcaj, chip)
+                f_inimag = open(fn_inimag, "w+")
+                f_inimag.write(inimag_header)
 
-            f_calmag.close()
-            fn_calmag = "%sM%d-calmag_%s.mag" % (field, numcaj, chip)
-            f_calmag = open(fn_calmag, "w+")
-            f_calmag.write(calmag_header)
+                f_calmag.close()
+                fn_calmag = "%sM%d-calmag_%s.mag" % (field, numcaj, chip)
+                f_calmag = open(fn_calmag, "w+")
+                f_calmag.write(calmag_header)
 
-            print ('Writing set of files %s...' % numcaj)
+                print ('Writing set of files %s...' % numcaj)
 
 
 #-----------------------------------------------------------------------------
@@ -1333,8 +1350,8 @@ def process_argv(args, mch_ext):
     ARG_STARSSHUF  = argc; argc+=1
     ARG_MAGEXT     = argc; argc+=1
     ARG_MAXCCDSIZE = argc; argc+=1
-    ARG_DIMFIELD   = argc; argc+=1
     ARG_RADCENT    = argc; argc+=1
+    ARG_DIMFIELD   = argc; argc+=1
     ARG_DISTANCE   = argc; argc+=1
     ARG_REFINEMCH  = argc; argc+=1
     NUM_ARGS       = argc
@@ -1408,7 +1425,7 @@ def process_argv(args, mch_ext):
         # Get Filters
         filters = [];
         for flt in cols_order:
-            if flt != STAR_ID and flt != IGNORE_COL:
+            if flt != STAR_ID and flt != IGNORE_COL and flt != XPOS_COL and flt != YPOS_COL:
                 filters.append(flt)
     except:
         error_msg += " * Column order\n"
@@ -1454,20 +1471,19 @@ def process_argv(args, mch_ext):
             magext_aux = [float(x) for x in magext_data.replace(" ","").split(',')]
 
             # Go for each filter:
-            for flt in cols_order:
+            for flt in filters:
 
                 # Check whether we have already processed all filters
                 if len(magext_aux) != 1 and num_filters >= len(magext_aux):
                     num_filters += 1
                     break
 
                 # Skip STAR_ID or columns to ignore 
-                if flt != STAR_ID and flt != IGNORE_COL:
-                    if len(magext_aux) == 1:
-                        magext[flt] = magext_aux[0]             # One common value
-                    else:
-                        magext[flt] = magext_aux[num_filters]   # List of values
-                    num_filters += 1
+                if len(magext_aux) == 1:
+                    magext[flt] = magext_aux[0]             # One common value
+                else:
+                    magext[flt] = magext_aux[num_filters]   # List of values
+                num_filters += 1
 
             if len(magext) != num_filters or (len(magext_aux) != 1 and len(magext_aux) != num_filters):
                 error_msg += " * MagExt should be one common value or have a value for each filter\n"
@@ -1596,7 +1612,7 @@ def get_instmag(data, calmag, colsub):
     instmag = calmag + data['ZPTERM'] + data['AMTERM'] * data['AIRMASS'] \
                      + data['COLTERM'] * colsub + data['APCOR'] - 2.5 * log10(data['EXPTIME'])
     return instmag
- 
+
 
 ########################################################################################
 
@@ -1902,6 +1918,8 @@ def create_trans_eq_calib(field, chip, numiters, chip_info):
 MAX_COEF      = 22                      # Max. number of coefficients in MCH files (with NO filename)
 STAR_ID       = "@"
 IGNORE_COL    = "*"
+XPOS_COL      = "XPOS"
+YPOS_COL      = "YPOS"
 FILE_ITER_SEP = 'M'			           # Separator used when creating the "add" files
 FILE_ITER_EXT = 'add'              # Add files extension
 MCH_FILE_EXT  = '.alf.mch'         # Extension of MCH files
@@ -1939,6 +1957,27 @@ def create_trans_eq_calib(field, chip, numiters, chip_info):
  = process_argv(sys.argv, MCH_FILE_EXT)
 
 
+
+# XXX XXX XXX XXX XXX XXX XXX
+# def check_add_files(field, chip):
+#
+#  
+#  add_fn = field+"M*-*_"+chip+".add"
+#  add_list = glob.glob(add_fn)
+#  numiters = 0
+#  for add in add_list:
+#    mock = int((add.replace("%sM" % field, "").split("-")[0]))
+#    if mock  > numiters:
+#      numiters = mock
+#  print(numiters)
+#  sys.exit()
+#
+#check_add_files(field, chip)
+# XXX XXX XXX XXX XXX XXX XXX
+
+
+
+
 # Get and process mch_file of MCH 
 mch_file = get_mch(orig_mch_file, refine_mch, field, chip)
 

scripts/fakered_transfer.sh

@@ -8,6 +8,7 @@
 ##
 ## AUTHOR: Antonio Dorta <adorta@iac.es>
 ## DATE: 2017-03-02
+##       2018-02-08: if starssplit == 2 -> Split stars using first column as chip ID
 ##
 ## Operations:
 ## 1) Read filenames to get the number of chips
@@ -27,7 +28,7 @@
 # New extension for MCH with .alf 
 ext=".alf.mch"
 # Files to be copied
-EXT_TO_COPY=(.fits .opt .alf .mch .psf .als .opt .als.opt .ap .raw .mag .log .weights .scale .zero _comb.psf _comb.opt _comb.als.opt _shift.mch .phot)
+EXT_TO_COPY=(.fits .fits.fz .opt .alf .mch .psf .als .opt .als.opt .ap .raw .mag .log .weights .scale .zero _comb.psf _comb.opt _comb.als.opt _shift.mch .phot)
 #EXT_TO_COPY=(alf)
 FILES_TO_COPY=(apcor.lst extinction fields)
 
@@ -172,8 +173,19 @@ then
   done
 
 else
-  cp $starsOrig $starsDest/${starsFn}.txt
-  starsFile=${starsFn}.txt
+  if [[ $starsSplit -eq 2 ]]
+  then
+    # SPLIT input stars file using the FIRST COLUMN as chip ID
+    for chip in "${CHIPS[@]}"
+    do
+      grep "^$chip " $starsOrig > $starsDest/${starsFn}_${chip}.txt
+    done
+
+  else
+    # NO splitting
+    cp $starsOrig $starsDest/${starsFn}.txt
+    starsFile=${starsFn}.txt
+  fi
 fi
 
 
@@ -277,7 +289,7 @@ do
     # Transfer stars file
 
     cd $destDir
-    if [[ $starsSplit -eq 1 ]]
+    if [[ $starsSplit -ne 0 ]]
     then
       starsFile="*${chp}.txt" 
     fi