// // C++ Implementation: STApplyCal // // Description: // // // Author: Takeshi Nakazato (C) 2012 // // Copyright: See COPYING file that comes with this distribution // // #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Scantable.h" #include "STApplyCal.h" #include "STApplyTable.h" #include "STCalTsysTable.h" #include "STCalSkyTable.h" #include "STCalEnum.h" #include "STIdxIter.h" #include "Calibrator.h" #include "PSAlmaCalibrator.h" #include "Interpolator1D.h" #include "NearestInterpolator1D.h" #include "BufferedLinearInterpolator1D.h" #include "PolynomialInterpolator1D.h" #include "CubicSplineInterpolator1D.h" #include using namespace casa; using namespace std; namespace { template class AccessorInterface { public: typedef Type TableType; static void GetSortedData(const vector &tablelist, const Vector &tableIndex, const uInt nrow, const uInt nchan, Vector &time, Matrix &data, Matrix &flag) { Vector timeUnsorted; Matrix dataUnsorted; Matrix flagUnsorted; GetFromTable(tablelist, tableIndex, nrow, nchan, timeUnsorted, dataUnsorted, flagUnsorted); SortData(timeUnsorted, dataUnsorted, flagUnsorted, time, data, flag); } private: static void GetFromTable(const vector &tableList, const Vector &tableIndex, const uInt nrow, const uInt nchan, Vector &time, Matrix &data, Matrix &flag) { time.resize(nrow, False); const IPosition shape(2, nrow, nchan); data.resize(shape, False); flag.resize(shape, False); uInt rowIndex = 0; for (uInt i = 0 ; i < tableIndex.nelements(); i++) { TableType *p = tableList[tableIndex[i]]; Vector t = Accessor::GetTime(p); Matrix dt = Accessor::GetData(p); Matrix fl = Accessor::GetFlag(p); for (uInt j = 0; j < t.nelements(); j++) { time[rowIndex] = t[j]; data.row(rowIndex) = dt.column(j); flag.row(rowIndex) = fl.column(j); rowIndex++; } } } static Vector IndexSort(const Vector &t) { Sort sort; sort.sortKey(&t[0], TpDouble, 0, Sort::Ascending); Vector idx; sort.sort(idx, t.nelements(), Sort::QuickSort|Sort::NoDuplicates); return idx; } static void SortData(const Vector &key, const Matrix &data, const Matrix &flag, Vector &sortedKey, Matrix &sortedData, Matrix &sortedFlag) { Vector sortIndex = IndexSort(key); uInt len = sortIndex.nelements(); IPosition shape = data.shape(); shape[0] = len; Int64 nelements = shape.product(); sortedKey.takeStorage(IPosition(1, len), new Double[len], TAKE_OVER); sortedData.takeStorage(shape, new Float[nelements], TAKE_OVER); sortedFlag.takeStorage(shape, new uChar[nelements], TAKE_OVER); for (uInt i = 0 ; i < len; i++) { sortedKey[i] = key[sortIndex[i]]; } for (uInt i = 0; i < len; ++i) { sortedData.row(i) = data.row(sortIndex[i]); sortedFlag.row(i) = flag.row(sortIndex[i]); } } }; class SkyTableAccessor : public AccessorInterface { public: static Vector GetTime(const TableType *t) {return t->getTime();} static Matrix GetData(const TableType *t) {return t->getSpectra();} static Matrix GetFlag(const TableType *t) {return t->getFlagtra();} }; class TsysTableAccessor : public AccessorInterface { public: static Vector GetTime(const TableType *t) {return t->getTime();} static Matrix GetData(const TableType *t) {return t->getTsys();} static Matrix GetFlag(const TableType *t) {return t->getFlagtra();} }; } namespace asap { STApplyCal::STApplyCal() { init(); } STApplyCal::STApplyCal(CountedPtr target) : target_(target) { init(); } STApplyCal::~STApplyCal() { } void STApplyCal::init() { caltype_ = STCalEnum::NoType; doTsys_ = False; iTime_ = STCalEnum::DefaultInterpolation; iFreq_ = STCalEnum::DefaultInterpolation; } void STApplyCal::reset() { // call init init(); // clear apply tables // do not delete object here skytable_.resize(0); tsystable_.resize(0); // clear mapping for Tsys transfer spwmap_.clear(); // reset selector sel_.reset(); // delete interpolators interpolatorT_ = 0; interpolatorS_ = 0; interpolatorF_ = 0; // clear working scantable work_ = 0; // clear calibrator calibrator_ = 0; } void STApplyCal::completeReset() { reset(); target_ = 0; } void STApplyCal::setTarget(CountedPtr target) { target_ = target; } void STApplyCal::setTarget(const String &name) { // always create PlainTable target_ = new Scantable(name, Table::Plain); } void STApplyCal::push(STCalSkyTable *table) { os_.origin(LogOrigin("STApplyCal","push",WHERE)); skytable_.push_back(table); STCalEnum::CalType caltype = STApplyTable::getCalType(table); os_ << "caltype=" << caltype << LogIO::POST; if (caltype_ == STCalEnum::NoType || caltype_ == STCalEnum::DefaultType || caltype_ == STCalEnum::CalTsys) { caltype_ = caltype; } os_ << "caltype_=" << caltype_ << LogIO::POST; } void STApplyCal::push(STCalTsysTable *table) { tsystable_.push_back(table); doTsys_ = True; } void STApplyCal::setTimeInterpolation(STCalEnum::InterpolationType itype, Int order) { iTime_ = itype; order_ = order; } void STApplyCal::setFrequencyInterpolation(STCalEnum::InterpolationType itype, Int order) { iFreq_ = itype; order_ = order; } void STApplyCal::setTsysTransfer(uInt from, Vector to) { os_.origin(LogOrigin("STApplyCal","setTsysTransfer",WHERE)); os_ << "from=" << from << ", to=" << to << LogIO::POST; map >::iterator i = spwmap_.find(from); if (i == spwmap_.end()) { spwmap_.insert(pair >(from, to)); } else { Vector toNew = i->second; spwmap_.erase(i); uInt k = toNew.nelements(); toNew.resize(k+to.nelements(), True); for (uInt i = 0; i < to.nelements(); i++) toNew[i+k] = to[i]; spwmap_.insert(pair >(from, toNew)); } } void STApplyCal::apply(Bool insitu, Bool filltsys) { os_.origin(LogOrigin("STApplyCal","apply",WHERE)); //assert(!target_.null()); assert_(!target_.null(),"You have to set target scantable first."); // calibrator if (caltype_ == STCalEnum::CalPSAlma) calibrator_ = new PSAlmaCalibrator(); // interpolator initInterpolator(); // select data sel_.reset(); sel_ = target_->getSelection(); if (caltype_ == STCalEnum::CalPSAlma || caltype_ == STCalEnum::CalPS) { sel_.setTypes(vector(1,(int)SrcType::PSON)); } target_->setSelection(sel_); //os_ << "sel_.print()=" << sel_.print() << LogIO::POST; // working data if (insitu) { os_.origin(LogOrigin("STApplyCal","apply",WHERE)); os_ << "Overwrite input scantable" << LogIO::POST; work_ = target_; } else { os_.origin(LogOrigin("STApplyCal","apply",WHERE)); os_ << "Create output scantable from input" << LogIO::POST; work_ = new Scantable(*target_, false); } //os_ << "work_->nrow()=" << work_->nrow() << LogIO::POST; // list of apply tables for sky calibration Vector skycalList(skytable_.size()); uInt numSkyCal = 0; // list of apply tables for Tsys calibration for (uInt i = 0 ; i < skytable_.size(); i++) { STCalEnum::CalType caltype = STApplyTable::getCalType(skytable_[i]); if (caltype == caltype_) { skycalList[numSkyCal] = i; numSkyCal++; } } skycalList.resize(numSkyCal, True); vector cols( 3 ) ; cols[0] = "BEAMNO" ; cols[1] = "POLNO" ; cols[2] = "IFNO" ; CountedPtr iter = new STIdxIter2(work_, cols) ; double start = mathutil::gettimeofday_sec(); os_ << LogIO::DEBUGGING << "start iterative doapply: " << start << LogIO::POST; while (!iter->pastEnd()) { Record ids = iter->currentValue(); Vector rows = iter->getRows(SHARE); if (rows.nelements() > 0) doapply(ids.asuInt("BEAMNO"), ids.asuInt("IFNO"), ids.asuInt("POLNO"), rows, skycalList, filltsys); iter->next(); } double end = mathutil::gettimeofday_sec(); os_ << LogIO::DEBUGGING << "end iterative doapply: " << end << LogIO::POST; os_ << LogIO::DEBUGGING << "elapsed time for doapply: " << end - start << " sec" << LogIO::POST; target_->unsetSelection(); } void STApplyCal::doapply(uInt beamno, uInt ifno, uInt polno, Vector &rows, Vector &skylist, Bool filltsys) { os_.origin(LogOrigin("STApplyCal","doapply",WHERE)); Bool doTsys = doTsys_; STSelector sel; vector id(1); id[0] = beamno; sel.setBeams(id); id[0] = ifno; sel.setIFs(id); id[0] = polno; sel.setPolarizations(id); // apply selection to apply tables uInt nrowSkyTotal = 0; uInt nrowTsysTotal = 0; for (uInt i = 0; i < skylist.nelements(); i++) { skytable_[skylist[i]]->setSelection(sel); nrowSkyTotal += skytable_[skylist[i]]->nrow(); os_ << "nrowSkyTotal=" << nrowSkyTotal << LogIO::POST; } // Skip IFNO without sky data if (nrowSkyTotal == 0) return; uInt nchanTsys = 0; Vector ftsys; uInt tsysifno = getIFForTsys(ifno); os_ << "tsysifno=" << (Int)tsysifno << LogIO::POST; if (tsystable_.size() == 0) { os_.origin(LogOrigin("STApplyTable", "doapply", WHERE)); os_ << "No Tsys tables are given. Skip Tsys calibratoin." << LogIO::POST; doTsys = False; } else if (tsysifno == (uInt)-1) { os_.origin(LogOrigin("STApplyTable", "doapply", WHERE)); os_ << "No corresponding Tsys for IFNO " << ifno << ". Skip Tsys calibration" << LogIO::POST; doTsys = False; } else { id[0] = (int)tsysifno; sel.setIFs(id); for (uInt i = 0; i < tsystable_.size() ; i++) { tsystable_[i]->setSelection(sel); uInt nrowThisTsys = tsystable_[i]->nrow(); nrowTsysTotal += nrowThisTsys; if (nrowThisTsys > 0 && nchanTsys == 0) { nchanTsys = tsystable_[i]->nchan(tsysifno); ftsys = tsystable_[i]->getBaseFrequency(0); } } } uInt nchanSp = skytable_[skylist[0]]->nchan(ifno); uInt nrowSky = nrowSkyTotal; Vector timeSky; Matrix spoff; Matrix flagoff; SkyTableAccessor::GetSortedData(skytable_, skylist, nrowSkyTotal, nchanSp, timeSky, spoff, flagoff); nrowSky = timeSky.nelements(); uInt nrowTsys = nrowTsysTotal; Vector timeTsys; Matrix tsys; Matrix flagtsys; if (doTsys) { //os_ << "doTsys" << LogIO::POST; Vector tsyslist(tsystable_.size()); indgen(tsyslist); TsysTableAccessor::GetSortedData(tsystable_, tsyslist, nrowTsysTotal, nchanTsys, timeTsys, tsys, flagtsys); nrowTsys = timeTsys.nelements(); } Table tab = work_->table(); ArrayColumn spCol(tab, "SPECTRA"); ArrayColumn flCol(tab, "FLAGTRA"); ArrayColumn tsysCol(tab, "TSYS"); ScalarColumn timeCol(tab, "TIME"); // Array for scaling factor (aka Tsys) Vector iTsys(IPosition(1, nchanSp), new Float[nchanSp], TAKE_OVER); // Array for Tsys interpolation // This is empty array and is never referenced if doTsys == false // (i.e. nchanTsys == 0) Vector iTsysT(IPosition(1, nchanTsys), new Float[nchanTsys], TAKE_OVER); // Array for interpolated off spectrum Vector iOff(IPosition(1, nchanSp), new Float[nchanSp], TAKE_OVER); // working array for interpolation with flags uInt arraySize = max(max(nrowTsys, nchanTsys), nrowSky); Vector xwork(IPosition(1, arraySize), new Double[arraySize], TAKE_OVER); Vector ywork(IPosition(1, arraySize), new Float[arraySize], TAKE_OVER); Vector fwork(IPosition(1, nchanTsys), new uChar[nchanTsys], TAKE_OVER); for (uInt i = 0; i < rows.nelements(); i++) { //os_ << "start i = " << i << " (row = " << rows[i] << ")" << LogIO::POST; uInt irow = rows[i]; // target spectral data Vector on = spCol(irow); Vector flag = flCol(irow); //os_ << "on=" << on[0] << LogIO::POST; calibrator_->setSource(on); // interpolation Double t0 = timeCol(irow); Double *xwork_p = xwork.data(); Float *ywork_p = ywork.data(); for (uInt ichan = 0; ichan < nchanSp; ichan++) { Float *tmpY = &(spoff.data()[ichan * nrowSky]); uChar *tmpF = &(flagoff.data()[ichan * nrowSky]); uInt wnrow = 0; for (uInt ir = 0; ir < nrowSky; ++ir) { if (tmpF[ir] == 0) { xwork_p[wnrow] = timeSky.data()[ir]; ywork_p[wnrow] = tmpY[ir]; wnrow++; } } if (wnrow > 0) { // any valid reference data interpolatorS_->setData(xwork_p, ywork_p, wnrow); } else { // no valid reference data // interpolate data regardless of flag interpolatorS_->setData(timeSky.data(), tmpY, nrowSky); // flag this channel for calibrated data flag[ichan] = 1 << 7; // user flag } iOff[ichan] = interpolatorS_->interpolate(t0); } //os_ << "iOff=" << iOff[0] << LogIO::POST; calibrator_->setReference(iOff); if (doTsys) { // Tsys correction // interpolation on time axis Float *yt = iTsysT.data(); uChar *fwork_p = fwork.data(); for (uInt ichan = 0; ichan < nchanTsys; ichan++) { Float *tmpY = &(tsys.data()[ichan * nrowTsys]); uChar *tmpF = &(flagtsys.data()[ichan * nrowTsys]); uInt wnrow = 0; for (uInt ir = 0; ir < nrowTsys; ++ir) { if (tmpF[ir] == 0) { xwork_p[wnrow] = timeTsys.data()[ir]; ywork_p[wnrow] = tmpY[ir]; wnrow++; } } if (wnrow > 0) { // any valid value exists interpolatorT_->setData(xwork_p, ywork_p, wnrow); iTsysT[ichan] = interpolatorT_->interpolate(t0); fwork_p[ichan] = 0; } else { // no valid data fwork_p[ichan] = 1 << 7; // user flag } } if (nchanSp == 1) { // take average iTsys[0] = mean(iTsysT); } else { // interpolation on frequency axis Vector fsp = getBaseFrequency(rows[i]); uInt wnchan = 0; for (uInt ichan = 0; ichan < nchanTsys; ++ichan) { if (fwork_p[ichan] == 0) { xwork_p[wnchan] = ftsys.data()[ichan]; ywork_p[wnchan] = yt[ichan]; ++wnchan; } } //interpolatorF_->setY(yt, nchanTsys); interpolatorF_->setData(xwork_p, ywork_p, wnchan); for (uInt ichan = 0; ichan < nchanSp; ichan++) { iTsys[ichan] = interpolatorF_->interpolate(fsp[ichan]); } } } else { Vector tsysInRow = tsysCol(irow); if (tsysInRow.nelements() == 1) { iTsys = tsysInRow[0]; } else { for (uInt ichan = 0; ichan < tsysInRow.nelements(); ++ichan) iTsys[ichan] = tsysInRow[ichan]; } } //os_ << "iTsys=" << iTsys[0] << LogIO::POST; calibrator_->setScaler(iTsys); // do calibration calibrator_->calibrate(); // update table //os_ << "calibrated=" << calibrator_->getCalibrated()[0] << LogIO::POST; spCol.put(irow, calibrator_->getCalibrated()); flCol.put(irow, flag); if (filltsys) tsysCol.put(irow, iTsys); } // reset selection on apply tables for (uInt i = 0; i < skylist.nelements(); i++) skytable_[i]->unsetSelection(); for (uInt i = 0; i < tsystable_.size(); i++) tsystable_[i]->unsetSelection(); // reset interpolator interpolatorS_->reset(); interpolatorF_->reset(); interpolatorT_->reset(); } uInt STApplyCal::getIFForTsys(uInt to) { for (map >::iterator i = spwmap_.begin(); i != spwmap_.end(); i++) { Vector tolist = i->second; os_ << "from=" << i->first << ": tolist=" << tolist << LogIO::POST; for (uInt j = 0; j < tolist.nelements(); j++) { if (tolist[j] == to) return i->first; } } return (uInt)-1; } void STApplyCal::save(const String &name) { //assert(!work_.null()); assert_(!work_.null(),"You have to execute apply method first."); work_->setSelection(sel_); work_->makePersistent(name); work_->unsetSelection(); } Vector STApplyCal::getBaseFrequency(uInt whichrow) { //assert(whichrow <= (uInt)work_->nrow()); assert_(whichrow <= (uInt)work_->nrow(),"row index out of range."); ROTableColumn col(work_->table(), "IFNO"); uInt ifno = col.asuInt(whichrow); col.attach(work_->table(), "FREQ_ID"); uInt freqid = col.asuInt(whichrow); uInt nc = work_->nchan(ifno); STFrequencies ftab = work_->frequencies(); Double rp, rf, inc; ftab.getEntry(rp, rf, inc, freqid); Vector r(nc); indgen(r, rf-rp*inc, inc); return r; } void STApplyCal::initInterpolator() { os_.origin(LogOrigin("STApplyCal","initInterpolator",WHERE)); int order = (order_ > 0) ? order_ : 1; switch (iTime_) { case STCalEnum::NearestInterpolation: { os_ << "use NearestInterpolator in time axis" << LogIO::POST; interpolatorS_ = new NearestInterpolator1D(); interpolatorT_ = new NearestInterpolator1D(); break; } case STCalEnum::LinearInterpolation: { os_ << "use BufferedLinearInterpolator in time axis" << LogIO::POST; interpolatorS_ = new BufferedLinearInterpolator1D(); interpolatorT_ = new BufferedLinearInterpolator1D(); break; } case STCalEnum::CubicSplineInterpolation: { os_ << "use CubicSplineInterpolator in time axis" << LogIO::POST; interpolatorS_ = new CubicSplineInterpolator1D(); interpolatorT_ = new CubicSplineInterpolator1D(); break; } case STCalEnum::PolynomialInterpolation: { os_ << "use PolynomialInterpolator in time axis" << LogIO::POST; if (order == 0) { interpolatorS_ = new NearestInterpolator1D(); interpolatorT_ = new NearestInterpolator1D(); } else { interpolatorS_ = new PolynomialInterpolator1D(); interpolatorT_ = new PolynomialInterpolator1D(); interpolatorS_->setOrder(order); interpolatorT_->setOrder(order); } break; } default: { os_ << "use BufferedLinearInterpolator in time axis" << LogIO::POST; interpolatorS_ = new BufferedLinearInterpolator1D(); interpolatorT_ = new BufferedLinearInterpolator1D(); break; } } switch (iFreq_) { case STCalEnum::NearestInterpolation: { os_ << "use NearestInterpolator in frequency axis" << LogIO::POST; interpolatorF_ = new NearestInterpolator1D(); break; } case STCalEnum::LinearInterpolation: { os_ << "use BufferedLinearInterpolator in frequency axis" << LogIO::POST; interpolatorF_ = new BufferedLinearInterpolator1D(); break; } case STCalEnum::CubicSplineInterpolation: { os_ << "use CubicSplineInterpolator in frequency axis" << LogIO::POST; interpolatorF_ = new CubicSplineInterpolator1D(); break; } case STCalEnum::PolynomialInterpolation: { os_ << "use PolynomialInterpolator in frequency axis" << LogIO::POST; if (order == 0) { interpolatorF_ = new NearestInterpolator1D(); } else { interpolatorF_ = new PolynomialInterpolator1D(); interpolatorF_->setOrder(order); } break; } default: { os_ << "use LinearInterpolator in frequency axis" << LogIO::POST; interpolatorF_ = new BufferedLinearInterpolator1D(); break; } } } }