From d7e6a4e9c0e969ed53c3a8f277237f328f58e931 Mon Sep 17 00:00:00 2001 From: Antonio Dorta Date: Tue, 30 Jan 2018 09:59:10 +0000 Subject: [PATCH 1/5] Sync --- pro/.gitignore | 2 +- pro/loadraw.pro | 2 +- pro/photred_commonsources_global.pro | 2 +- pro/writecol.pro | 0 4 files changed, 3 insertions(+), 3 deletions(-) mode change 100755 => 100644 pro/writecol.pro diff --git a/pro/.gitignore b/pro/.gitignore index 3de9b8c..31366f7 100644 --- a/pro/.gitignore +++ b/pro/.gitignore @@ -1,3 +1,3 @@ *~ *.bak -*.bak* \ No newline at end of file +*.bak* diff --git a/pro/loadraw.pro b/pro/loadraw.pro index 4b34aff..92fda04 100644 --- a/pro/loadraw.pro +++ b/pro/loadraw.pro @@ -136,7 +136,7 @@ if arr1[0] eq 'NL' and strtrim(line3,2) eq '' then begin ; Loop through the lines per star for k=0l,nstarline-1 do begin readf, unit, instr1 - if k gt 0 then instr1=strmid(instr1,25) ; 2nd and later lines have 25 leading spaces + if k gt 0 then instr1=strmid(instr1,24) ; 2nd and later lines have 25 leading spaces instr += instr1 endfor diff --git a/pro/photred_commonsources_global.pro b/pro/photred_commonsources_global.pro index 874a45e..89e89e1 100644 --- a/pro/photred_commonsources_global.pro +++ b/pro/photred_commonsources_global.pro @@ -206,7 +206,7 @@ printlog,logfile,'Creating cmn.coo and cmn.ap for ',strtrim(nfieldfiles,2),' fil ; Now run PBS_DAEMON.PRO if n_elements(cmd) gt 0 then $ - PBS_DAEMON,cmd,nmulti=nmulti,prefix='dcmn',hyperthread=hyperthread,/idle,waittime=5,scriptsdir=scriptsdir + PBS_DAEMON,cmd,nmulti=nmulti,prefix='dcmn',hyperthread=hyperthread,/idle,waittime=1,scriptsdir=scriptsdir ; Now concatenate and merge all of the catalogs ;---------------------------------------------- diff --git a/pro/writecol.pro b/pro/writecol.pro old mode 100755 new mode 100644 From 0b0dcd6a195c2d844b2d73018367760bfe8e2a39 Mon Sep 17 00:00:00 2001 From: Antonio Dorta Date: Tue, 30 Jan 2018 10:04:00 +0000 Subject: [PATCH 2/5] Getting new loadraw.pro --- pro/loadraw.pro | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pro/loadraw.pro b/pro/loadraw.pro index 4b34aff..e26d7f0 100644 --- a/pro/loadraw.pro +++ b/pro/loadraw.pro @@ -136,7 +136,15 @@ if arr1[0] eq 'NL' and strtrim(line3,2) eq '' then begin ; Loop through the lines per star for k=0l,nstarline-1 do begin readf, unit, instr1 - if k gt 0 then instr1=strmid(instr1,25) ; 2nd and later lines have 25 leading spaces + if k gt 0 then begin + ; There are leading spaces, 24 or 25 + ; Use the first character AFTER the first column to figure out + ; how many spaces we need to strip off + trial = strmid(instr1,34,1) + if trial eq ' ' then nspaces=25 else nspaces=24 + instr1 = strmid(instr1,nspaces) + endif + ;if k gt 0 then instr1=strmid(instr1,25) ; 2nd and later lines have 25 leading spaces instr += instr1 endfor From e72f4084a47717839b15241b96c320375d7dcb09 Mon Sep 17 00:00:00 2001 From: Antonio Dorta Date: Mon, 25 Jun 2018 16:18:05 +0100 Subject: [PATCH 3/5] After getting David's new files for complete. Also after adding Clusters --- pro/allframe_combine_orig.pro | 3 +- pro/fakered_addstar.pro | 2 +- scripts/fakered_addstar.py | 138 +++++++++++++++++++--------------- scripts/fakered_transfer.sh | 20 ++++- 4 files changed, 95 insertions(+), 68 deletions(-) diff --git a/pro/allframe_combine_orig.pro b/pro/allframe_combine_orig.pro index 70d11a3..e7e80ae 100644 --- a/pro/allframe_combine_orig.pro +++ b/pro/allframe_combine_orig.pro @@ -219,7 +219,8 @@ endfor ; FAKE, check that we have all the files that we need if keyword_set(fake) then begin ; weights, scale, zero, comb_psf, _shift.mch - chkfiles = mchbase+['.weights','.scale','.zero','_comb.psf','_shift.mch'] + ; adorta chkfiles = mchbase+['.weights','.scale','.zero','_comb.psf','_shift.mch'] + chkfiles = mchbase+['.weights','.scale','.zero'] bdfiles = where(file_test(chkfiles) eq 0,nbdfiles) if nbdfiles gt 0 then begin error = 'FAKE. Some necessary files not found. '+strjoin(chkfiles[bdfiles],' ') diff --git a/pro/fakered_addstar.pro b/pro/fakered_addstar.pro index 151a66e..1bf0822 100644 --- a/pro/fakered_addstar.pro +++ b/pro/fakered_addstar.pro @@ -95,7 +95,7 @@ nmulti = getparam(nmulti , 'nmulti' , setup, '1' scriptsdir = getparam(scriptsdir , 'scriptsdir' , setup, '' , logfile) sepfielddir = getparam(sepfielddir , 'sepfielddir' , setup, '0' , logfile, /bool) sepchipdir = getparam(sepchipdir , 'sepchipdir' , setup, '0' , logfile, /bool) -starssplit = getparam(starssplit , 'starssplit' , setup, '0' , logfile, /bool) +starssplit = getparam(starssplit , 'starssplit' , setup, '0' , logfile) starsfile = getparam(starsfile , 'starsfile' , setup, '' , logfile) starscols = getparam(starscols , 'starscols' , setup, '' , logfile) datadir = getparam(datadir , 'datadir' , setup, '.' , logfile) diff --git a/scripts/fakered_addstar.py b/scripts/fakered_addstar.py index 72d3ddc..a764dfe 100755 --- a/scripts/fakered_addstar.py +++ b/scripts/fakered_addstar.py @@ -342,6 +342,8 @@ def get_input_stars(fn_stars, cols_order, stars_shuffle): stars[col] = stars_raw[pos] if col == STAR_ID: print ("Column " + str(pos+1) + ": stars ID. Total stars: " + str(len(stars[col]))) + elif col == XPOS_COL or col == YPOS_COL: + print ("Column " + str(pos+1) + ": Position " + col + ". Total stars: " + str(len(stars[col]))) else: print ("Column " + str(pos+1) + ": filter " + col + ". Total stars: " + str(len(stars[col]))) else: @@ -490,7 +492,7 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m offset -- random values of offset [offx, offy] - shift -- shift in magnitude to move the stars in the CMD after every iteration + shift -- shift in magnitude to move the stars in the CMD after every iteration (NOT USED!!) """ @@ -511,7 +513,7 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m offset = [random.uniform(-lim,lim), random.uniform(-lim,lim)] if shift is None: lim = 0.07 - shift = random.uniform(-lim,lim) + shift = random.uniform(-lim,lim) # NOT USED!!! # Get dimensions radcent /= 2 @@ -704,14 +706,21 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m xaux = 0 yaux = 0 + # See whether we have to create the GRID or the positions are given + GIVEN_POS = XPOS_COL in data_in and YPOS_COL in data_in + rowpos = 0 print ('Writing set of files %s...' % numcaj) for star_pos,star_id in enumerate(data_in[STAR_ID]): - # Get initial positions - xpos_init = xmin + (2 * (xaux + 1) - 1) * radcent + offx - ypos_init = ymin + (2 * (yaux + 1) - 1) * radcent + offy + if GIVEN_POS: + xpos_init = data_in[XPOS_COL][star_pos] + ypos_init = data_in[YPOS_COL][star_pos] + else: + # Get initial positions + xpos_init = xmin + (2 * (xaux + 1) - 1) * radcent + offx + ypos_init = ymin + (2 * (yaux + 1) - 1) * radcent + offy # Write line in file .mag for the current star. Format: # ID XREF YREF F1 F1ERR F2 F2ERR ... FN FNERR CHI SHARP FLAG PROB @@ -777,16 +786,16 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m # Get EBV data from all images to later calculate the average (needed to write calmag files) chip_img = chip_info[nimg] - flt = chip_img['FILTER'] - data_ebv[flt] += chip_img['EBV'] - count_ebv[flt] += 1 + flt_img = chip_img['FILTER'] + data_ebv[flt_img] += chip_img['EBV'] + count_ebv[flt_img] += 1 # After transformations, SAVE TO FILE (xpos, ypos) ONLY if it is inside rectangle given by corners if in_rectangle (corners, [xpos, ypos]): # This star is INSIDE the limits, transform the MAGNITUDE and write it to ADD try: # Get calibrated magnitude after applying absorption and distance modulus - calmag = get_calmag(data_in[flt][star_pos], distance, chip_img['EBV'], magext[flt]) + calmag = get_calmag(data_in[flt_img][star_pos], distance, chip_img['EBV'], magext[flt_img]) # Get calibrated color: colsign * (band - colband) color = chip_img['COLSIGN'] * (data_in[chip_img['BAND']][star_pos] - data_in[chip_img['COLBAND']][star_pos]) # PERFORM MAGNITUDE TRANSFORMATION @@ -829,7 +838,10 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m for flt in filters: inimag = data_in[flt][star_pos] # EBV depends on the image, get the average value - ebv_val = data_ebv[flt]/count_ebv[flt] + if count_ebv[flt] > 0: + ebv_val = data_ebv[flt]/count_ebv[flt] + else: + ebv_val = 0 calmag = get_calmag(inimag, distance, ebv_val, magext[flt]) out_inimag += " %10.3f" % inimag out_calmag += " %10.3f" % calmag @@ -839,59 +851,60 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m #----------------------------------------------------------------------------- - # UPDATE COUNTERS: Get ready for next iteration over the stars loop - # (Go to next star and next col of CCD) - cn -= 1 - yaux += 1 - # If we are in the last col, go to first position of next row - if (yaux >= ysize): - xaux += XSEP - rowpos += 1 - # If we are in even row, y position is half in order to distribute stars in triangles - if (rowpos % 2) == 0: - yaux = 0.0 - else: - yaux = 0.5 + if not GIVEN_POS: + # UPDATE COUNTERS: Get ready for next iteration over the stars loop + # (Go to next star and next col of CCD) + cn -= 1 + yaux += 1 + # If we are in the last col, go to first position of next row + if (yaux >= ysize): + xaux += XSEP + rowpos += 1 + # If we are in even row, y position is half in order to distribute stars in triangles + if (rowpos % 2) == 0: + yaux = 0.0 + else: + yaux = 0.5 - # Check if CCD is FULL (xaux >= xsize) and there are still some stars to process (cn > 0) - if (xaux >= xsize) and (cn > 0): + # Check if CCD is FULL (xaux >= xsize) and there are still some stars to process (cn > 0) + if (xaux >= xsize) and (cn > 0): - # Check if we have reached the limit of iters (Mocks). If so, quit loop! - if numcaj >= addmax: break + # Check if we have reached the limit of iters (Mocks). If so, quit loop! + if numcaj >= addmax: break - # CCD IS FULL, write in new files - xaux = 0 - yaux = 0 - numcaj += 1 - nfopen += 1 + # CCD IS FULL, write in new files + xaux = 0 + yaux = 0 + numcaj += 1 + nfopen += 1 - # Close all open files - for f in fidimag: - if not f.closed: - f.close() - # Open new files - for k in xrange(nimages): - fidimag.append(open(filenames[nfopen * nimages + k], 'w')) + # Close all open files + for f in fidimag: + if not f.closed: + f.close() + # Open new files + for k in xrange(nimages): + fidimag.append(open(filenames[nfopen * nimages + k], 'w')) - # Close previous MAG file and create new one for current iteration - f_mag.close() - fn_mag = get_iter_filename(mch_fnames[0], field, chip, numcaj, ".mag").replace("_", "-add_") - f_mag = open(fn_mag, "w+") - f_mag.write(mag_header) + # Close previous MAG file and create new one for current iteration + f_mag.close() + fn_mag = get_iter_filename(mch_fnames[0], field, chip, numcaj, ".mag").replace("_", "-add_") + f_mag = open(fn_mag, "w+") + f_mag.write(mag_header) - f_inimag.close() - fn_inimag = "%sM%d-inimag_%s.mag" % (field, numcaj, chip) - f_inimag = open(fn_inimag, "w+") - f_inimag.write(inimag_header) + f_inimag.close() + fn_inimag = "%sM%d-inimag_%s.mag" % (field, numcaj, chip) + f_inimag = open(fn_inimag, "w+") + f_inimag.write(inimag_header) - f_calmag.close() - fn_calmag = "%sM%d-calmag_%s.mag" % (field, numcaj, chip) - f_calmag = open(fn_calmag, "w+") - f_calmag.write(calmag_header) + f_calmag.close() + fn_calmag = "%sM%d-calmag_%s.mag" % (field, numcaj, chip) + f_calmag = open(fn_calmag, "w+") + f_calmag.write(calmag_header) - print ('Writing set of files %s...' % numcaj) + print ('Writing set of files %s...' % numcaj) #----------------------------------------------------------------------------- @@ -1333,8 +1346,8 @@ def process_argv(args, mch_ext): ARG_STARSSHUF = argc; argc+=1 ARG_MAGEXT = argc; argc+=1 ARG_MAXCCDSIZE = argc; argc+=1 - ARG_DIMFIELD = argc; argc+=1 ARG_RADCENT = argc; argc+=1 + ARG_DIMFIELD = argc; argc+=1 ARG_DISTANCE = argc; argc+=1 ARG_REFINEMCH = argc; argc+=1 NUM_ARGS = argc @@ -1408,7 +1421,7 @@ def process_argv(args, mch_ext): # Get Filters filters = []; for flt in cols_order: - if flt != STAR_ID and flt != IGNORE_COL: + if flt != STAR_ID and flt != IGNORE_COL and flt != XPOS_COL and flt != YPOS_COL: filters.append(flt) except: error_msg += " * Column order\n" @@ -1454,7 +1467,7 @@ def process_argv(args, mch_ext): magext_aux = [float(x) for x in magext_data.replace(" ","").split(',')] # Go for each filter: - for flt in cols_order: + for flt in filters: # Check whether we have already processed all filters if len(magext_aux) != 1 and num_filters >= len(magext_aux): @@ -1462,12 +1475,11 @@ def process_argv(args, mch_ext): break # Skip STAR_ID or columns to ignore - if flt != STAR_ID and flt != IGNORE_COL: - if len(magext_aux) == 1: - magext[flt] = magext_aux[0] # One common value - else: - magext[flt] = magext_aux[num_filters] # List of values - num_filters += 1 + if len(magext_aux) == 1: + magext[flt] = magext_aux[0] # One common value + else: + magext[flt] = magext_aux[num_filters] # List of values + num_filters += 1 if len(magext) != num_filters or (len(magext_aux) != 1 and len(magext_aux) != num_filters): error_msg += " * MagExt should be one common value or have a value for each filter\n" @@ -1902,6 +1914,8 @@ def create_trans_eq_calib(field, chip, numiters, chip_info): MAX_COEF = 22 # Max. number of coefficients in MCH files (with NO filename) STAR_ID = "@" IGNORE_COL = "*" +XPOS_COL = "XPOS" +YPOS_COL = "YPOS" FILE_ITER_SEP = 'M' # Separator used when creating the "add" files FILE_ITER_EXT = 'add' # Add files extension MCH_FILE_EXT = '.alf.mch' # Extension of MCH files diff --git a/scripts/fakered_transfer.sh b/scripts/fakered_transfer.sh index 31dcc6c..d2c5c93 100755 --- a/scripts/fakered_transfer.sh +++ b/scripts/fakered_transfer.sh @@ -8,6 +8,7 @@ ## ## AUTHOR: Antonio Dorta ## DATE: 2017-03-02 +## 2018-02-08: if starssplit == 2 -> Split stars using first column as chip ID ## ## Operations: ## 1) Read filenames to get the number of chips @@ -27,7 +28,7 @@ # New extension for MCH with .alf ext=".alf.mch" # Files to be copied -EXT_TO_COPY=(.fits .opt .alf .mch .psf .als .opt .als.opt .ap .raw .mag .log .weights .scale .zero _comb.psf _comb.opt _comb.als.opt _shift.mch .phot) +EXT_TO_COPY=(.fits .fits.fz .opt .alf .mch .psf .als .opt .als.opt .ap .raw .mag .log .weights .scale .zero _comb.psf _comb.opt _comb.als.opt _shift.mch .phot) #EXT_TO_COPY=(alf) FILES_TO_COPY=(apcor.lst extinction fields) @@ -172,8 +173,19 @@ then done else - cp $starsOrig $starsDest/${starsFn}.txt - starsFile=${starsFn}.txt + if [[ $starsSplit -eq 2 ]] + then + # SPLIT input stars file using the FIRST COLUMN as chip ID + for chip in "${CHIPS[@]}" + do + grep "^$chip " $starsOrig > $starsDest/${starsFn}_${chip}.txt + done + + else + # NO splitting + cp $starsOrig $starsDest/${starsFn}.txt + starsFile=${starsFn}.txt + fi fi @@ -277,7 +289,7 @@ do # Transfer stars file cd $destDir - if [[ $starsSplit -eq 1 ]] + if [[ $starsSplit -ne 0 ]] then starsFile="*${chp}.txt" fi From ca9e2d5a0a19ffebc0d5d6186429340aeb1a8159 Mon Sep 17 00:00:00 2001 From: Antonio Dorta Date: Fri, 3 Aug 2018 10:10:52 +0100 Subject: [PATCH 4/5] A few minor bugs were fixed --- pro/fakered_addstar.pro | 2 +- scripts/fakered_addstar.py | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/pro/fakered_addstar.pro b/pro/fakered_addstar.pro index 1bf0822..a7d7a0d 100644 --- a/pro/fakered_addstar.pro +++ b/pro/fakered_addstar.pro @@ -100,7 +100,7 @@ starsfile = getparam(starsfile , 'starsfile' , setup, '' starscols = getparam(starscols , 'starscols' , setup, '' , logfile) datadir = getparam(datadir , 'datadir' , setup, '.' , logfile) datatransfer = getparam(datatransfer , 'datatransfer' , setup, 'skip' , logfile) -chipsfile = getparam(chipsfile , 'chipssfile' , setup, '*chips.fits' , logfile) +chipsfile = getparam(chipsfile , 'chipsfile' , setup, '*chips.fits' , logfile) maxccdsize = getparam(maxccdsize , 'maxccdsize' , setup, '2048,4096' , logfile) magext = getparam(magext , 'magext' , setup, '0' , logfile) radcent = getparam(radcent , 'radcent' , setup, '*' , logfile) diff --git a/scripts/fakered_addstar.py b/scripts/fakered_addstar.py index a764dfe..d0950a0 100755 --- a/scripts/fakered_addstar.py +++ b/scripts/fakered_addstar.py @@ -584,9 +584,9 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m if max_iters > 0 and max_iters < addmax: print_warning("Limiting number of Mocks to " + str(max_iters)) addmax = max_iters - max_files = int(round(999/nimages)-1) - if addmax > max_files: - exit_error_msg('Too many open files\nMaximun number is: ' + str(max_files) + "\nDecrease the total number of artificial stars", False) + #max_files = int(round(999/nimages)-1) + #if addmax > max_files: + # exit_error_msg('Too many open files\nMaximun number is: ' + str(max_files) + "\nDecrease the total number of artificial stars", False) #----------------------------------------------------------------------------- @@ -730,7 +730,11 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m # Second and consecutive lines has 27 spaces before first data MAX_MAG_COLS = 12 num_cols = 0 - mag_line = "%9d %8.3f %8.3f" % (star_id, xpos_init, ypos_init) + #mag_line = "%9d %8.3f %8.3f" % (star_id, xpos_init, ypos_init) + mag_line = "%9d" % star_id + # if we have negative numbers of 4 digits, remove one decimal to leave place for sign + mag_line += " %8.3f" % xpos_init if xpos_init > -1000.0 else " %7.2f" % xpos_init + mag_line += " %8.3f" % ypos_init if ypos_init > -1000.0 else " %7.2f" % ypos_init # ------------------------------------------------------------------------- @@ -825,7 +829,7 @@ def crowdingmultipro(max_iters, field, chip, mode, mch_fnames, mch_data, caja, m # END MAG file with fixed fields: CHI SHARP FLAG PROB num_cols += 2 mag_line += " 1.0000 0.0000" - if num_cols % MAX_MAG_COLS == 0: mag_line += "\n%23s" % '' + if num_cols % MAX_MAG_COLS == 0: mag_line += "\n%25s" % '' mag_line += " 0 1.00\n" #mag_line += " 1.0000 0.0000 0 1.00\n" f_mag.write(mag_line) @@ -1608,7 +1612,7 @@ def get_instmag(data, calmag, colsub): instmag = calmag + data['ZPTERM'] + data['AMTERM'] * data['AIRMASS'] \ + data['COLTERM'] * colsub + data['APCOR'] - 2.5 * log10(data['EXPTIME']) return instmag - + ######################################################################################## @@ -1953,6 +1957,27 @@ def create_trans_eq_calib(field, chip, numiters, chip_info): = process_argv(sys.argv, MCH_FILE_EXT) + +# XXX XXX XXX XXX XXX XXX XXX +# def check_add_files(field, chip): +# +# +# add_fn = field+"M*-*_"+chip+".add" +# add_list = glob.glob(add_fn) +# numiters = 0 +# for add in add_list: +# mock = int((add.replace("%sM" % field, "").split("-")[0])) +# if mock > numiters: +# numiters = mock +# print(numiters) +# sys.exit() +# +#check_add_files(field, chip) +# XXX XXX XXX XXX XXX XXX XXX + + + + # Get and process mch_file of MCH mch_file = get_mch(orig_mch_file, refine_mch, field, chip) From 69828e55b0f1869789c3ec3cd03b353db7671adb Mon Sep 17 00:00:00 2001 From: Antonio Dorta Date: Fri, 3 Aug 2018 10:24:05 +0100 Subject: [PATCH 5/5] Restoring pro/allframe_combine_orig.pro --- pro/allframe_combine_orig.pro | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pro/allframe_combine_orig.pro b/pro/allframe_combine_orig.pro index e7e80ae..70d11a3 100644 --- a/pro/allframe_combine_orig.pro +++ b/pro/allframe_combine_orig.pro @@ -219,8 +219,7 @@ endfor ; FAKE, check that we have all the files that we need if keyword_set(fake) then begin ; weights, scale, zero, comb_psf, _shift.mch - ; adorta chkfiles = mchbase+['.weights','.scale','.zero','_comb.psf','_shift.mch'] - chkfiles = mchbase+['.weights','.scale','.zero'] + chkfiles = mchbase+['.weights','.scale','.zero','_comb.psf','_shift.mch'] bdfiles = where(file_test(chkfiles) eq 0,nbdfiles) if nbdfiles gt 0 then begin error = 'FAKE. Some necessary files not found. '+strjoin(chkfiles[bdfiles],' ')