Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

Code Block
languagebash
themeConfluence
makeCoaddTempExp.py $DEMO_DIR/calexp_dir \
  --config bgSubtracted=False \
  warpAndPsfMatch.warp.warpingKernelName='lanczos4' \
  warpAndPsfMatch.warp.cacheSize=0 \
  --output $DEMO_DIR/coadd_r_dir \
  @$DEMO_DIR/patches_r.txt \
  -j $NCORES >& mkCoaddExp_log.txt

...

Code Block
languagebash
themeConfluence
titleSummary of DRP processing commands
collapsetrue
# Processing Command Summary
# Tutorial: Co-Add and forced-photometry of a small portion of Stripe 82
#
# Load the environment, set up packages for processing:
source /path/to/your/lsstInstallDir/loadLSST.sh      # modify for your installation
setup obs_sdss -t Winter2014
setup pipe_tasks -t Winter2014
setup datarel -t Winter2014
 
# Be sure to define the path to the processing directory on your system:
export DEMO_DIR=/path/to/your/working/directory
 
# Fetch the tutorials package
cd /path/to/install/directory
git clone https://github.com/lsst-dm/tutorials.git
export TUT_DIR=$PWD/tutorials/sdssDrpTutorial

# Create a repository:
cd $DEMO_DIR
mkdir Stripe82
cd Stripe82 && mkdir runs
export DATA_DIR=$DEMO_DIR/Stripe82/runs
echo "lsst.obs.sdss.sdssMapper.SdssMapper" > runs/_mapper
 
# Create a SkyMap
cd $DEMO_DIR
makeSkyMap.py $DATA_DIR/ --config skyMap.active.decRange=-1.2,1.2 skyMap.active.patchInnerDimensions=2000,1907 skyMap.active.patchBorder=30 skyMap.active.tractOverlap=0.0154 --output tempSkyMap_dir

# Identify patches that cover the SkyMap for the RA/Dec range:
reportPatches.py tempSkyMap_dir --config raDecRange="5.0, 0.05, 5.2, 0.25" --id tract=0 patch=0,0 > patches.txt

# Identify SDSS fields to process (finds 715 exposures, of which 702 are retained):
reportImagesToCoadd.py tempSkyMap_dir --config raDecRange="5.0, 0.05, 5.2, 0.25" select.strip=Both select.quality=None select.rejectWholeRuns=False showImageIds=True --id filter=r > rawInputs.txt

# Reformat the output of the rawInputs file:
$TUT_DIR/python/refCoaddList.py rawInputs.txt > rawInputs_r.txt
cp rawInputs_r.txt ./Stripe82

## Acquire the data:
# Install the Astrometry Catalog
curl -O http://lsst-web.ncsa.illinois.edu/sdss-s12/sdss-2012-05-01-0.tgz 
tar xzf sdss-2012-05-01-0.tgz 
eups declare -r sdss-2012-05-01-0 astrometry_net_data sdss-2012-05-01-0
setup astrometry_net_data sdss-2012-05-01-0 --keep

# Retrieve SDSS images:
cd $DEMO_DIR/Stripe82
$TUT_DIR/python/getRetrieveList rawInputs_r.txt retrieve.txt
wget -r -b -R "index.html*" -np -nH --cut-dirs=1 -P ./runs -i retrieve.txt

# Create a registry:
genInputRegistry.py runs
mv registry.sqlite3 ./runs

# Set environment variable for number of cores to use in parallel processing:
cd $DEMO_DIR
export NCORES=$((sysctl -n hw.ncpu || (test -r /proc/cpuinfo && grep processor /proc/cpuinfo | wc -l) || echo 2) 2>/dev/null)
 
# Process the input SDSS images:
setup meas_extensions_multiShapelet --keep
processCcdSdss.py $DATA_DIR/ --output $DEMO_DIR/calexp_dir --configfile $TUT_DIR/config/processConfig.py @rawInputs_r.txt -j $NCORES --logdest procCcdSdss_log.txt

# Prepare DB for ingest; be sure to select your own DB name, prepended by your MySQL username:
export DR_PATH=$DATAREL_DIR/bin/ingest
export DB_NAME=<username>_Stripe82_demo2
$DR_PATH/prepareDb.py $DB_NAME --camera=sdss

# Ingest metadata from the calexp's
mkdir $DEMO_DIR/ingestProcessed_csv_dir
$DR_PATH/ingestProcessed.py --database=$DB_NAME $DEMO_DIR/ingestProcessed_csv_dir $DEMO_DIR/calexp_dir  --camera=sdss >& ingestProcessed_csv_log.txt

# Create the mapping from SDSS fields to the output patches:
sed -e 's/^/--id filter=r /' patches.txt > patches_r.txt
makeSkyMap.py $DEMO_DIR/calexp_dir --config skyMap.active.decRange=0.05,0.25 skyMap.active.patchInnerDimensions=2000,1907 skyMap.active.patchBorder=30 skyMap.active.tractOverlap=0.0154 --output coadd_r_dir

# Warp the calexp's to the output geometry, and store the temporary exposures:
makeCoaddTempExp.py $DEMO_DIR/calexp_dir --config bgSubtracted=False warpAndPsfMatch.warp.warpingKernelName="lanczos4" warpAndPsfMatch.warp.cacheSize=0 --output $DEMO_DIR/coadd_r_dir @$DEMO_DIR/patches_r.txt -j $NCORES >& mkCoaddExp_log.txt

# Assemble the Co-Adds
sed -e 's/\([13579]$\)/\1 run=5823/' patches_r.txt | sed -e 's/\([02468]$\)/\1 run=6955/' > patches_r_runs.txt
assembleCoadd.py $DEMO_DIR/coadd_r_dir --config maxMatchResidualRatio=1.2 maxMatchResidualRMS=0.5 scaleZeroPoint.selectFluxMag0.database=$DB_NAME @$DEMO_DIR/patches_r_runs.txt -j $NCORES >& assembleCoadd_log.txt

# Process the Co-Adds to measure sources
processCoadd.py $DEMO_DIR/coadd_r_dir --configfile $TUT_DIR/config/processConfig.py @$DEMO_DIR/patches_r.txt -j $NCORES >& processCoadd_log.txt

# Ingest all of the Co-Add sources into the DB. 
mkdir $DEMO_DIR/ingestCoadd_r_csv_dir
$DR_PATH/ingestCoadd.py --camera=sdss --database=$DB_NAME $DEMO_DIR/ingestCoadd_r_csv_dir $DEMO_DIR/coadd_r_dir --coadd-names=deep --create-views 

# Perform forced photometry
cut -d " " -f1,2,3,4,6 rawInputs_r.txt > forcedPhotInputs_r.txt
forcedPhot.py $DEMO_DIR/calexp_dir --output $DEMO_DIR/forcedPhot_dir --configfile $TUT_DIR/config/forcedPhotConfig.py --config references.dbName=$DB_NAME references.filterName=r @$DEMO_DIR/forcedPhotInputs_r.txt -j $NCORES >& forcedPhot_log.txt

# Create Science database
mkdir $DEMO_DIR/forcedPhot_csv_dir
mkdir $DEMO_DIR/sourceAssoc-csv
$DR_PATH/ingestForcedSource.py --camera=sdss --database=$DB_NAME --coadd-name=deep --create-views $DEMO_DIR/forcedPhot_csv_dir $DEMO_DIR/forcedPhot_dir >& ingestForcedSource_log.txt

# Associate the Co-Add sources with the single-frame sources
sourceAssoc.py $DEMO_DIR/coadd_r_dir --output $DEMO_DIR/sourceAssoc_dir >& sourceAssoc_log.txt
 
# Ingest the associated sources into the DB
$DR_PATH/ingestSourceAssoc.py -d $DB_NAME --camera sdss --create-views -j $NCORES $DEMO_DIR/sourceAssoc-csv $DEMO_DIR/sourceAssoc_dir >& ingestSourceAssoc_log.txt
 
# Enable the table keys:
mysql $DB_NAME -h lsst-db.ncsa.illinois.edu -u <user> -p < $TUT_DIR/enable_keys.sql