source: trunk/src/ScantableWrapper.h@ 2287

Last change on this file since 2287 was 2286, checked in by Kana Sugimoto, 13 years ago

New Development: No (performance tuning)

JIRA Issue: No

Ready for Test: Yes

Interface Changes: Yes

What Interface Changed: a parameter "filename" is added to Scantable::summary. scantable.summary doesn't return a string anymore

Test Programs: sdlist unittest/ scantable.summary("summary.txt")

Put in Release Notes: Yes

Module(s): sdlist, asap.summary

Description:

scantable.summary is very slow for large data sets (in row number) often outputted
by modern telescopes. It takes > 1.5 hours to list OTF raster scan with 350,000 rows.

This was because, the methods accumulates the whole text string (~700,000 lines) and
returns it as a string. Once the summary string exceed several tens thousands lines,
elapse time increases non-linearly, may be because very massive output string starts
to overweigh the memory.

I updated scantable.summary so that it flushes the summary string more often to file/logger.
After the modification, scantable.summary could list the data mentioned above in ~ 7 minutes.
The side effect of it is that scantable.summary doesn't return summary string anymore.
(But people may not happy with sub-million lines of string anyway.)


  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 12.0 KB
RevLine 
[847]1//
2// C++ Interface: Scantable
3//
4// Description:
5//
6//
7// Author: Malte Marquarding <asap@atnf.csiro.au>, (C) 2006
8//
9// Copyright: See COPYING file that comes with this distribution
10//
11//
12#ifndef ASAPSCANTABLEWRAPPER_H
13#define ASAPSCANTABLEWRAPPER_H
[2]14
15#include <vector>
16#include <string>
[380]17#include <casa/Arrays/Vector.h>
[2]18
[465]19#include "MathUtils.h"
[901]20#include "STFit.h"
[1931]21#include "STFitEntry.h"
[847]22#include "Scantable.h"
[1598]23#include "STCoordinate.h"
[2]24
[83]25namespace asap {
[847]26/**
27 * This class contains and wraps a CountedPtr<Scantable>, as the CountedPtr
28 * class does not provide the dor operator which is need for references
29 * in boost::python
30 * see Scantable for interfce description
31 *
32 * @brief The main ASAP data container wrapper
33 * @author Malte Marquarding
34 * @date 2006-02-23
35 * @version 2.0a
36*/
37class ScantableWrapper {
[2]38
39public:
[1385]40 explicit ScantableWrapper( const std::string& name,
[1077]41 int type=0)
[864]42 {
43 casa::Table::TableType tp = casa::Table::Memory;
[1077]44 if ( type == 1 ) tp = casa::Table::Plain;
[864]45 table_ = new Scantable(name, tp);
46 }
[90]47
[1350]48 explicit ScantableWrapper(int type=0)
[864]49 {
50 casa::Table::TableType tp = casa::Table::Memory;
[1077]51 if ( type == 1) tp = casa::Table::Plain;
[864]52 table_= new Scantable(tp);
53 }
[90]54
[1350]55 explicit ScantableWrapper(casa::CountedPtr<Scantable> cp) : table_(cp) {;}
[90]56
[847]57 ScantableWrapper(const ScantableWrapper& mt) :
58 table_(mt.getCP()) {;}
[90]59
[868]60 void assign(const ScantableWrapper& mt)
61 { table_= mt.getCP(); }
62
[847]63 ScantableWrapper copy() {
64 return ScantableWrapper(new Scantable(*(this->getCP()), false));
[80]65 }
[868]66
[896]67 std::vector<float> getSpectrum( int whichrow=0,
[905]68 const std::string& poltype="" ) const {
[896]69 return table_->getSpectrum(whichrow, poltype);
[2]70 }
[527]71 // std::string getPolarizationLabel(bool linear, bool stokes, bool linPol, int polIdx) const {
72 // Boost fails with 4 arguments.
[902]73 std::string getPolarizationLabel(int index, const std::string& ptype) const {
74 return table_->getPolarizationLabel(index, ptype);
[494]75 }
[527]76
[896]77 std::vector<double> getAbcissa(int whichrow=0) const
78 { return table_->getAbcissa(whichrow); }
[41]79
[896]80 std::string getAbcissaLabel(int whichrow=0) const
81 { return table_->getAbcissaLabel(whichrow); }
[157]82
[896]83 float getTsys(int whichrow=0) const
84 { return table_->getTsys(whichrow); }
[2]85
[2161]86 std::vector<float> getTsysSpectrum(int whichrow=0) const
87 { return table_->getTsysSpectrum(whichrow); }
88
[1947]89 //std::string getTime(int whichrow=0) const
90 // { return table_->getTime(whichrow); }
91 std::string getTime(int whichrow=0, int prec = 0) const
92 { return table_->getTime(whichrow, true, casa::uInt(prec)); }
[207]93
[1350]94 double getIntTime(int whichrow=0) const
95 { return table_->getIntTime(whichrow); }
96
[1068]97 std::string getDirectionString(int whichrow=0) const
98 { return table_->getDirectionString(whichrow); }
99
[864]100 std::string getFluxUnit() const { return table_->getFluxUnit(); }
101
102 void setFluxUnit(const std::string& unit) { table_->setFluxUnit(unit); }
103
[238]104 void setInstrument(const std::string& name) {table_->setInstrument(name);}
[1189]105 void setFeedType(const std::string& ftype) {table_->setFeedType(ftype);}
[238]106
[896]107 std::vector<bool> getMask(int whichrow=0) const
108 { return table_->getMask(whichrow); }
[2]109
[1430]110 /**
[1000]111 void flag(const std::vector<bool>& msk=std::vector<bool>())
112 { table_->flag(msk); }
[1430]113 **/
[1994]114 /**
[1430]115 void flag(const std::vector<bool>& msk=std::vector<bool>(), bool unflag=false)
116 { table_->flag(msk, unflag); }
[1994]117 **/
118 void flag(int whichrow=-1, const std::vector<bool>& msk=std::vector<bool>(), bool unflag=false)
119 { table_->flag(whichrow, msk, unflag); }
[2]120
[1819]121 void flagRow(const std::vector<casa::uInt>& rows=std::vector<casa::uInt>(), bool unflag=false)
122 { table_->flagRow(rows, unflag); }
123
124 bool getFlagRow(int whichrow=0) const
125 { return table_->getFlagRow(whichrow); }
126
127 void clip(const casa::Float uthres, const casa::Float dthres, bool clipoutside=true, bool unflag=false)
128 { table_->clip(uthres, dthres, clipoutside, unflag); }
129
130 std::vector<bool> getClipMask(int whichrow, const casa::Float uthres, const casa::Float dthres, bool clipoutside, bool unflag) const
131 { return table_->getClipMask(whichrow, uthres, dthres, clipoutside, unflag); }
132
[896]133 std::string getSourceName(int whichrow=0) const
134 { return table_->getSourceName(whichrow); }
[794]135
[896]136 float getElevation(int whichrow=0) const
137 { return table_->getElevation(whichrow); }
[90]138
[896]139 float getAzimuth(int whichrow=0) const
140 { return table_->getAzimuth(whichrow); }
[2]141
[896]142 float getParAngle(int whichrow=0) const
143 { return table_->getParAngle(whichrow); }
[2]144
[864]145
[884]146 void setSpectrum(std::vector<float> spectrum, int whichrow=0)
147 { table_->setSpectrum(spectrum, whichrow); }
[868]148
[1111]149 std::vector<uint> getIFNos() { return table_->getIFNos(); }
[864]150 int getIF(int whichrow) const {return table_->getIF(whichrow);}
[1111]151 std::vector<uint> getBeamNos() { return table_->getBeamNos(); }
[864]152 int getBeam(int whichrow) const {return table_->getBeam(whichrow);}
[1111]153 std::vector<uint> getPolNos() { return table_->getPolNos(); }
[864]154 int getPol(int whichrow) const {return table_->getPol(whichrow);}
[868]155 int getCycle(int whichrow) const {return table_->getCycle(whichrow);}
[1111]156 std::vector<uint> getScanNos() { return table_->getScanNos(); }
[868]157 int getScan(int whichrow) const {return table_->getScan(whichrow);}
[1819]158 std::vector<uint> getMolNos() { return table_->getMolNos();}
[864]159
160 STSelector getSelection() const { return table_->getSelection(); }
[868]161 void setSelection(const STSelector& sts)
162 { return table_->setSelection(sts);}
[864]163
[896]164 std::string getPolType() const { return table_->getPolType(); }
165
[864]166 int nif(int scanno=-1) const {return table_->nif(scanno);}
167 int nbeam(int scanno=-1) const {return table_->nbeam(scanno);}
168 int npol(int scanno=-1) const {return table_->npol(scanno);}
169 int nchan(int ifno=-1) const {return table_->nchan(ifno);}
170 int nscan() const {return table_->nscan();}
171 int nrow() const {return table_->nrow();}
[902]172 int ncycle(int scanno) const {return table_->ncycle(scanno);}
[2]173
[864]174 void makePersistent(const std::string& fname)
175 { table_->makePersistent(fname); }
[90]176
[1068]177 void setSourceType(int stype)
178 { table_->setSourceType(stype); }
179
[1360]180 void shift(int npix)
181 { table_->shift(npix); }
182
[1819]183/**
184 commented out by TT
[1170]185 void setRestFrequencies(double rf, const std::string& name,
186 const std::string& unit)
187 { table_->setRestFrequencies(rf, name, unit); }
[1819]188**/
189 void setRestFrequencies(vector<double> rf, const vector<std::string>& name,
190 const std::string& unit)
191 { table_->setRestFrequencies(rf, name, unit); }
192
[925]193/*
[868]194 void setRestFrequencies(const std::string& name) {
195 table_->setRestFrequencies(name);
[847]196 }
[925]197*/
[745]198
[1819]199/*
[864]200 std::vector<double> getRestFrequencies() const
201 { return table_->getRestFrequencies(); }
[1819]202*/
203 std::vector<double> getRestFrequency(int id) const
204 { return table_->getRestFrequency(id); }
[251]205
[105]206 void setCoordInfo(std::vector<string> theinfo) {
207 table_->setCoordInfo(theinfo);
208 }
209 std::vector<string> getCoordInfo() const {
210 return table_->getCoordInfo();
211 }
[90]212
[987]213 void setDirection(const std::string& refstr="")
214 { table_->setDirectionRefString(refstr); }
215
[847]216 casa::CountedPtr<Scantable> getCP() const {return table_;}
217 Scantable* getPtr() {return &(*table_);}
[380]218
[2286]219 // std::string summary() const {
220 // return table_->summary();
221 // }
222 void summary(const std::string& filename="") {
223 return table_->summary(filename);
[380]224 }
[745]225
[2178]226 std::string listHeader() const {
[2163]227 return table_->headerSummary();
[2111]228 }
229
[2163]230 std::vector<std::string> getHistory() const
[864]231 { return table_->getHistory(); }
[207]232
[864]233 void addHistory(const std::string& hist)
234 { table_->addHistory(hist); }
[745]235
[972]236 void addFit(const STFitEntry& fit, int row)
237 { table_->addFit(fit, row); }
238
239 STFitEntry getFit(int whichrow) const
240 { return table_->getFit(whichrow); }
241
[2012]242 void calculateAZEL() { table_->calculateAZEL(); }
[465]243
[1003]244 std::vector<std::string> columnNames() const
245 { return table_->columnNames(); }
246
[1391]247 std::string getAntennaName() const
248 { return table_->getAntennaName(); }
249
250 int checkScanInfo(const vector<int>& scanlist) const
251 { return table_->checkScanInfo(scanlist); }
[1819]252
[1730]253 std::vector<double> getDirectionVector(int whichrow) const
[1391]254 { return table_->getDirectionVector(whichrow); }
255
[1586]256 void parallactify(bool flag)
257 { table_->parallactify(flag); }
258
[1598]259 STCoordinate getCoordinate(int row) {
260 return STCoordinate(table_->getSpectralCoordinate(row));
261 }
262
[1730]263 std::vector<float> getWeather(int whichrow) const
264 { return table_->getWeather(whichrow); }
265
[1819]266 void reshapeSpectrum( int nmin, int nmax )
267 { table_->reshapeSpectrum( nmin, nmax ); }
268
[2189]269 void polyBaseline(const std::vector<bool>& mask, int order, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
270 { table_->polyBaseline(mask, order, getresidual, showprogress, outlog, blfile); }
[1907]271
[2189]272 void autoPolyBaseline(const std::vector<bool>& mask, int order, const std::vector<int>& edge, float threshold=5.0, int chan_avg_limit=1, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
273 { table_->autoPolyBaseline(mask, order, edge, threshold, chan_avg_limit, getresidual, showprogress, outlog, blfile); }
[1907]274
[2189]275 void cubicSplineBaseline(const std::vector<bool>& mask, int npiece, float clipthresh, int clipniter, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
276 { table_->cubicSplineBaseline(mask, npiece, clipthresh, clipniter, getresidual, showprogress, outlog, blfile); }
[2012]277
[2189]278 void autoCubicSplineBaseline(const std::vector<bool>& mask, int npiece, float clipthresh, int clipniter, const std::vector<int>& edge, float threshold=5.0, int chan_avg_limit=1, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
279 { table_->autoCubicSplineBaseline(mask, npiece, clipthresh, clipniter, edge, threshold, chan_avg_limit, getresidual, showprogress, outlog, blfile); }
[2012]280
[2189]281 void sinusoidBaseline(const std::vector<bool>& mask, const bool applyfft, const std::string& fftmethod, const std::string& fftthresh, const std::vector<int>& addwn, const std::vector<int>& rejwn, float clipthresh, int clipniter, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
282 { table_->sinusoidBaseline(mask, applyfft, fftmethod, fftthresh, addwn, rejwn, clipthresh, clipniter, getresidual, showprogress, outlog, blfile); }
[2047]283
[2189]284 void autoSinusoidBaseline(const std::vector<bool>& mask, const bool applyfft, const std::string& fftmethod, const std::string& fftthresh, const std::vector<int>& addwn, const std::vector<int>& rejwn, float clipthresh, int clipniter, const std::vector<int>& edge, float threshold=5.0, int chan_avg_limit=1, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
285 { table_->autoSinusoidBaseline(mask, applyfft, fftmethod, fftthresh, addwn, rejwn, clipthresh, clipniter, edge, threshold, chan_avg_limit, getresidual, showprogress, outlog, blfile); }
[2047]286
[2012]287 float getRms(const std::vector<bool>& mask, int whichrow)
288 { return table_->getRms(mask, whichrow); }
289
290 std::string formatBaselineParams(const std::vector<float>& params, const std::vector<bool>& fixed, float rms, const std::string& masklist, int whichrow, bool verbose=false)
[2193]291 { return table_->formatBaselineParams(params, fixed, rms, -1, masklist, whichrow, verbose); }
[2012]292
293 std::string formatPiecewiseBaselineParams(const std::vector<int>& ranges, const std::vector<float>& params, const std::vector<bool>& fixed, float rms, const std::string& masklist, int whichrow, bool verbose=false)
[2193]294 { return table_->formatPiecewiseBaselineParams(ranges, params, fixed, rms, -1, masklist, whichrow, verbose); }
[2012]295
[1907]296 bool getFlagtraFast(int whichrow=0) const
[2047]297 { return table_->getFlagtraFast(casa::uInt(whichrow)); }
[1907]298
[2186]299 std::vector<float> execFFT(int whichrow, const std::vector<bool>& mask, bool getRealImag=false, bool getAmplitudeOnly=false)
300 { return table_->execFFT(whichrow, mask, getRealImag, getAmplitudeOnly); }
[1907]301
[2012]302
[2186]303
[2]304private:
[847]305 casa::CountedPtr<Scantable> table_;
[2]306};
307
308} // namespace
309#endif
[1819]310
Note: See TracBrowser for help on using the repository browser.