Merge pull request #494 from desihub/sv

support SV, especially SV1_DESI_TARGET

.travis.yml

@@ -55,7 +55,7 @@ env:
         - DESISIM_TESTDATA_VERSION=0.6.1
         - SPECSIM_VERSION=v0.12
         - DESISPEC_VERSION=0.26.0
-        - DESITARGET_VERSION=0.25.0
+        - DESITARGET_VERSION=0.29.1
         - SIMQSO_VERSION=v1.2.3
         - DESI_LOGLEVEL=DEBUG
         - MAIN_CMD='python setup.py'

bin/wrap-fastframe

@@ -80,7 +80,8 @@ if rank == 0:
                     framefile = desispec.io.findfile('cframe', night, expid, camera)
                 else :
                     framefile = desispec.io.findfile('frame', night, expid, camera)
-                if not os.path.exists(framefile):
+
+                if args.clobber or not os.path.exists(framefile):
                     simspecfiles.append( (flavor, filename) )
                     done = False
                     break
@@ -105,9 +106,11 @@ if rank < len(simspecfiles):
     for flavor, filename in simspecfiles[rank::size]:
         cmd = "fastframe --simspec {}".format(filename)
         if args.outdir:
-            cmd = cmd + " --outdir {}".format(args.outdir)
+           cmd = cmd + " --outdir {}".format(args.outdir)
         if args.clobber:
-            cmd = cmd + " --clobber"
+            pass 
+            ##  clobber not supported by fastframe (it clobbers by default)
+            ##  cmd = cmd + " --clobber"
         if args.cframe:
             cmd = cmd + " --cframe"
 
@@ -118,9 +121,17 @@ if rank < len(simspecfiles):
 
         try:
             t0 = time.time()
+
             logfile = filename.replace('simspec', 'fastframe').replace('.fits', '.log')
+
             assert logfile != filename
 
+            if args.outdir:
+                ##  If outdir provided, log files inherit /night/exposure structure of simspec files but in outdir. 
+                ##  Else placed in original dir. with simspecs.
+                logfile = logfile.replace(desisim.io.simdir(), args.outdir)
+                os.makedirs(os.path.dirname(logfile), exist_ok=args.clobber)
+
             #- Use subprocess.call instead of fastframe.main() to avoid
             #- potential memory leaks and separate logging; this does incur
             #- extra python interpreter startup time.

bin/wrap-newexp

@@ -26,8 +26,8 @@ parser.add_argument('--force', action='store_true', help="force re-processing to
 parser.add_argument('--nonuniform', action='store_true', help="simulate non-uniform calibration screen illumination")
 parser.add_argument('--dryrun', action='store_true', help="print commands but don't run them")
 parser.add_argument('--mpi', action='store_true', help="use MPI parallelism")
-parser.add_argument('--start', metavar='YYYYMMDD', default='19000101', help="Start date (YEARMMDD).")
-parser.add_argument('--stop', metavar='YYYYMMDD', default='21000101', help="Stop date (YEARMMDD).")
+parser.add_argument('--start', metavar='YYYYMMDD', default=19000101, help="Start date (YEARMMDD).")
+parser.add_argument('--stop', metavar='YYYYMMDD', default=21000101, help="Stop date (YEARMMDD).")
 
 args = parser.parse_args()
 
@@ -52,8 +52,11 @@ from astropy.table import Table
 
 import desisim.io
 
-args.start = args.start.replace('-', '')
-args.stop = args.stop.replace('-', '')
+if type(args.start) == str:
+    args.start = args.start.replace('-', '')
+
+if type(args.start) == str:
+    args.stop = args.stop.replace('-', '')
 
 def read_tiles(filename):
     if filename.endswith('.fits'):
@@ -100,7 +103,8 @@ if comm is not None:
 if rank == 0:
     todo = obs[:0].copy()
     for thisobs in obs:
-        if (thisobs['NIGHT'] < args.start) or (args.stop <= thisobs['NIGHT']):
+        tonight = int(thisobs['NIGHT'])
+        if (tonight < args.start) or (args.stop <= tonight):
             continue
 
         simspec = desisim.io.findfile('simspec', thisobs['NIGHT'], thisobs['EXPID'])

py/desisim/scripts/newexp_mock.py

@@ -58,10 +58,10 @@ def main(args=None):
 
     if os.path.isdir(args.fiberassign):
         #- TODO: move file location logic to desispec / desitarget / fiberassign
-        args.fiberassign = os.path.join(args.fiberassign, 'tile-{:05d}.fits'.format(tileid))
+        args.fiberassign = os.path.join(args.fiberassign, 'tile-{:06d}.fits'.format(tileid))
         if not os.path.exists(args.fiberassign):
             #- try previous name
-            args.fiberassign = os.path.join(args.fiberassign, 'tile_{:05d}.fits'.format(tileid))
+            args.fiberassign = os.path.join(os.path.dirname(args.fiberassign), 'tile_{:06d}.fits'.format(tileid))
 
     fiberassign = astropy.table.Table.read(args.fiberassign, 'FIBERASSIGN')
 

py/desisim/simexp.py

@@ -14,6 +14,7 @@
 
 import desitarget
 import desitarget.targetmask
+from desitarget.targets import main_cmx_or_sv
 import desispec.io
 import desispec.io.util
 import desimodel.io
@@ -302,12 +303,17 @@ def fibermeta2fibermap(fiberassign, meta):
     A future refactor will standardize the column names of fiber assignment,
     target catalogs, and fibermaps, but in the meantime this is needed.
     '''
-    from desitarget.targetmask import desi_mask
+    #- Handle DESI_TARGET vs. SV1_DESI_TARGET etc.
+    target_colnames, target_masks, survey = main_cmx_or_sv(fiberassign)
+    targetcol = target_colnames[0]  #- DESI_TARGET or SV1_DESI_TARGET
+    desi_mask = target_masks[0]     #- desi_mask or sv1_desi_mask
 
     #- Copy column names in common
     fibermap = desispec.io.empty_fibermap(len(fiberassign))
-    for c in ['FIBER', 'TARGETID', 'DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET',
-              'BRICKNAME']:
+    for c in ['FIBER', 'TARGETID', 'BRICKNAME']:
+        fibermap[c] = fiberassign[c]
+
+    for c in target_colnames:
         fibermap[c] = fiberassign[c]
 
     for band in ['G', 'R', 'Z', 'W1', 'W2']:
@@ -325,9 +331,9 @@ def fibermeta2fibermap(fiberassign, meta):
         if name in desi_mask.names():
             stdmask |= desi_mask[name]
 
-    isSTD = (fiberassign['DESI_TARGET'] & stdmask) != 0
+    isSTD = (fiberassign[targetcol] & stdmask) != 0
 
-    isSKY = (fiberassign['DESI_TARGET'] & desi_mask.SKY) != 0
+    isSKY = (fiberassign[targetcol] & desi_mask.SKY) != 0
     isSCI = (~isSTD & ~isSKY)
     fibermap['OBJTYPE'][isSKY] = 'SKY'
     fibermap['OBJTYPE'][isSCI | isSTD] = 'TGT'
@@ -361,7 +367,8 @@ def simulate_spectra(wave, flux, fibermap=None, obsconditions=None, redshift=Non
         wave (array): 1D wavelengths in Angstroms
         flux (array): 2D[nspec,nwave] flux in 1e-17 erg/s/cm2/Angstrom
             or astropy Quantity with flux units
-        fibermap (Table, optional): table from fiberassign or fibermap; uses X/YFOCAL_DESIGN, TARGETID, DESI_TARGET
+        fibermap (Table, optional): table from fiberassign or fibermap; uses
+            X/YFOCAL_DESIGN, TARGETID, DESI_TARGET
         obsconditions(dict-like, optional): observation metadata including
             SEEING (arcsec), EXPTIME (sec), AIRMASS,
             MOONFRAC (0-1), MOONALT (deg), MOONSEP (deg)
@@ -631,26 +638,31 @@ def get_source_types(fibermap):
     so BGS targets get source_type = 'lrg' (!)
     '''
     from desiutil.log import get_logger
-    log = get_logger('DEBUG')
-    if 'DESI_TARGET' not in fibermap.dtype.names:
-        log.warning("DESI_TARGET not in fibermap table; using source_type='star' for everything")
+    log = get_logger()
+
+    if ('DESI_TARGET' not in fibermap.dtype.names) and \
+       ('SV1_DESI_TARGET' not in fibermap.dtype.names):
+        log.warning("(SV1_)DESI_TARGET not in fibermap table; using source_type='star' for everything")
         return np.array(['star',] * len(fibermap))
 
+    target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
+    targetcol = target_colnames[0]  #- DESI_TARGET or SV1_DESI_TARGET
+    tm = target_masks[0]     #- desi_mask or sv1_desi_mask
+
     source_type = np.zeros(len(fibermap), dtype='U4')
     assert np.all(source_type == '')
 
-    tm = desitarget.targetmask.desi_mask
     if 'TARGETID' in fibermap.dtype.names:
         unassigned = fibermap['TARGETID'] == -1
         source_type[unassigned] = 'sky'
 
     source_type[(fibermap['OBJTYPE'] == 'FLT')] = 'FLAT'
     source_type[(fibermap['OBJTYPE'] == 'ARC')] = 'ARC'
-    source_type[(fibermap['DESI_TARGET'] & tm.SKY) != 0] = 'sky'
-    source_type[(fibermap['DESI_TARGET'] & tm.ELG) != 0] = 'elg'
-    source_type[(fibermap['DESI_TARGET'] & tm.LRG) != 0] = 'lrg'
-    source_type[(fibermap['DESI_TARGET'] & tm.QSO) != 0] = 'qso'
-    source_type[(fibermap['DESI_TARGET'] & tm.BGS_ANY) != 0] = 'bgs'
+    source_type[(fibermap[targetcol] & tm.SKY) != 0] = 'sky'
+    source_type[(fibermap[targetcol] & tm.ELG) != 0] = 'elg'
+    source_type[(fibermap[targetcol] & tm.LRG) != 0] = 'lrg'
+    source_type[(fibermap[targetcol] & tm.QSO) != 0] = 'qso'
+    source_type[(fibermap[targetcol] & tm.BGS_ANY) != 0] = 'bgs'
 
     starmask = 0
     for name in ['STD', 'STD_FSTAR', 'STD_WD', 'MWS_ANY',
@@ -659,7 +671,7 @@ def get_source_types(fibermap):
         if name in desitarget.targetmask.desi_mask.names():
             starmask |= desitarget.targetmask.desi_mask[name]
 
-    source_type[(fibermap['DESI_TARGET'] & starmask) != 0] = 'star'
+    source_type[(fibermap[targetcol] & starmask) != 0] = 'star'
 
     #- Simulate unassigned fibers as sky
     ## TODO: when fiberassign and desitarget are updated, use
@@ -746,7 +758,11 @@ def get_mock_spectra(fiberassign, mockdir=None, nside=64):
     wave = None
     objmeta = None
 
-    issky = (fiberassign['DESI_TARGET'] & desitarget.targetmask.desi_mask.SKY) != 0
+    target_colnames, target_masks, survey = main_cmx_or_sv(fiberassign)
+    targetcol = target_colnames[0]  #- DESI_TARGET or SV1_DESI_TARGET
+    desi_mask = target_masks[0]     #- desi_mask or sv1_desi_mask
+
+    issky = (fiberassign[targetcol] & desi_mask.SKY) != 0
     skyids = fiberassign['TARGETID'][issky]
 
     #- check several ways in which an unassigned fiber might appear