Versions Compared


  • This line was added.
  • This line was removed.
  • Formatting was changed.


Code Block
themeConfluence $DEMO_DIR/calexp_dir \
  --config bgSubtracted=False \
  warpAndPsfMatch.warp.warpingKernelName='lanczos4' \
  warpAndPsfMatch.warp.cacheSize=0 \
  --output $DEMO_DIR/coadd_r_dir \
  @$DEMO_DIR/patches_r.txt \
  -j $NCORES >& mkCoaddExp_log.txt


Code Block
titleSummary of DRP processing commands
# Processing Command Summary
# Tutorial: Co-Add and forced-photometry of a small portion of Stripe 82
# Load the environment, set up packages for processing:
source /path/to/your/lsstInstallDir/      # modify for your installation
setup obs_sdss -t Winter2014
setup pipe_tasks -t Winter2014
setup datarel -t Winter2014
# Be sure to define the path to the processing directory on your system:
export DEMO_DIR=/path/to/your/working/directory
# Fetch the tutorials package
cd /path/to/install/directory
git clone
export TUT_DIR=$PWD/tutorials/sdssDrpTutorial

# Create a repository:
mkdir Stripe82
cd Stripe82 && mkdir runs
export DATA_DIR=$DEMO_DIR/Stripe82/runs
echo "lsst.obs.sdss.sdssMapper.SdssMapper" > runs/_mapper
# Create a SkyMap
cd $DEMO_DIR $DATA_DIR/ --config,1.2,1907 --output tempSkyMap_dir

# Identify patches that cover the SkyMap for the RA/Dec range: tempSkyMap_dir --config raDecRange="5.0, 0.05, 5.2, 0.25" --id tract=0 patch=0,0 > patches.txt

# Identify SDSS fields to process (finds 715 exposures, of which 702 are retained): tempSkyMap_dir --config raDecRange="5.0, 0.05, 5.2, 0.25" select.strip=Both select.quality=None select.rejectWholeRuns=False showImageIds=True --id filter=r > rawInputs.txt

# Reformat the output of the rawInputs file:
$TUT_DIR/python/ rawInputs.txt > rawInputs_r.txt
cp rawInputs_r.txt ./Stripe82

## Acquire the data:
# Install the Astrometry Catalog
curl -O 
tar xzf sdss-2012-05-01-0.tgz 
eups declare -r sdss-2012-05-01-0 astrometry_net_data sdss-2012-05-01-0
setup astrometry_net_data sdss-2012-05-01-0 --keep

# Retrieve SDSS images:
cd $DEMO_DIR/Stripe82
$TUT_DIR/python/getRetrieveList rawInputs_r.txt retrieve.txt
wget -r -b -R "index.html*" -np -nH --cut-dirs=1 -P ./runs -i retrieve.txt

# Create a registry: runs
mv registry.sqlite3 ./runs

# Set environment variable for number of cores to use in parallel processing:
export NCORES=$((sysctl -n hw.ncpu || (test -r /proc/cpuinfo && grep processor /proc/cpuinfo | wc -l) || echo 2) 2>/dev/null)
# Process the input SDSS images:
setup meas_extensions_multiShapelet --keep $DATA_DIR/ --output $DEMO_DIR/calexp_dir --configfile $TUT_DIR/config/ @rawInputs_r.txt -j $NCORES --logdest procCcdSdss_log.txt

# Prepare DB for ingest; be sure to select your own DB name, prepended by your MySQL username:
export DR_PATH=$DATAREL_DIR/bin/ingest
export DB_NAME=<username>_Stripe82_demo2
$DR_PATH/ $DB_NAME --camera=sdss

# Ingest metadata from the calexp's
mkdir $DEMO_DIR/ingestProcessed_csv_dir
$DR_PATH/ --database=$DB_NAME $DEMO_DIR/ingestProcessed_csv_dir $DEMO_DIR/calexp_dir  --camera=sdss >& ingestProcessed_csv_log.txt

# Create the mapping from SDSS fields to the output patches:
sed -e 's/^/--id filter=r /' patches.txt > patches_r.txt $DEMO_DIR/calexp_dir --config,0.25,1907 --output coadd_r_dir

# Warp the calexp's to the output geometry, and store the temporary exposures: $DEMO_DIR/calexp_dir --config bgSubtracted=False warpAndPsfMatch.warp.warpingKernelName="lanczos4" warpAndPsfMatch.warp.cacheSize=0 --output $DEMO_DIR/coadd_r_dir @$DEMO_DIR/patches_r.txt -j $NCORES >& mkCoaddExp_log.txt

# Assemble the Co-Adds
sed -e 's/\([13579]$\)/\1 run=5823/' patches_r.txt | sed -e 's/\([02468]$\)/\1 run=6955/' > patches_r_runs.txt $DEMO_DIR/coadd_r_dir --config maxMatchResidualRatio=1.2 maxMatchResidualRMS=0.5 scaleZeroPoint.selectFluxMag0.database=$DB_NAME @$DEMO_DIR/patches_r_runs.txt -j $NCORES >& assembleCoadd_log.txt

# Process the Co-Adds to measure sources $DEMO_DIR/coadd_r_dir --configfile $TUT_DIR/config/ @$DEMO_DIR/patches_r.txt -j $NCORES >& processCoadd_log.txt

# Ingest all of the Co-Add sources into the DB. 
mkdir $DEMO_DIR/ingestCoadd_r_csv_dir
$DR_PATH/ --camera=sdss --database=$DB_NAME $DEMO_DIR/ingestCoadd_r_csv_dir $DEMO_DIR/coadd_r_dir --coadd-names=deep --create-views 

# Perform forced photometry
cut -d " " -f1,2,3,4,6 rawInputs_r.txt > forcedPhotInputs_r.txt $DEMO_DIR/calexp_dir --output $DEMO_DIR/forcedPhot_dir --configfile $TUT_DIR/config/ --config references.dbName=$DB_NAME references.filterName=r @$DEMO_DIR/forcedPhotInputs_r.txt -j $NCORES >& forcedPhot_log.txt

# Create Science database
mkdir $DEMO_DIR/forcedPhot_csv_dir
mkdir $DEMO_DIR/sourceAssoc-csv
$DR_PATH/ --camera=sdss --database=$DB_NAME --coadd-name=deep --create-views $DEMO_DIR/forcedPhot_csv_dir $DEMO_DIR/forcedPhot_dir >& ingestForcedSource_log.txt

# Associate the Co-Add sources with the single-frame sources $DEMO_DIR/coadd_r_dir --output $DEMO_DIR/sourceAssoc_dir >& sourceAssoc_log.txt
# Ingest the associated sources into the DB
$DR_PATH/ -d $DB_NAME --camera sdss --create-views -j $NCORES $DEMO_DIR/sourceAssoc-csv $DEMO_DIR/sourceAssoc_dir >& ingestSourceAssoc_log.txt
# Enable the table keys:
mysql $DB_NAME -h -u <user> -p < $TUT_DIR/enable_keys.sql