source: trunk/src/ScantableWrapper.h @ 2286

Last change on this file since 2286 was 2286, checked in by Kana Sugimoto, 13 years ago

New Development: No (performance tuning)

JIRA Issue: No

Ready for Test: Yes

Interface Changes: Yes

What Interface Changed: a parameter "filename" is added to Scantable::summary. scantable.summary doesn't return a string anymore

Test Programs: sdlist unittest/ scantable.summary("summary.txt")

Put in Release Notes: Yes

Module(s): sdlist, asap.summary

Description:

scantable.summary is very slow for large data sets (in row number) often outputted
by modern telescopes. It takes > 1.5 hours to list OTF raster scan with 350,000 rows.

This was because, the methods accumulates the whole text string (~700,000 lines) and
returns it as a string. Once the summary string exceed several tens thousands lines,
elapse time increases non-linearly, may be because very massive output string starts
to overweigh the memory.

I updated scantable.summary so that it flushes the summary string more often to file/logger.
After the modification, scantable.summary could list the data mentioned above in ~ 7 minutes.
The side effect of it is that scantable.summary doesn't return summary string anymore.
(But people may not happy with sub-million lines of string anyway.)


  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 12.0 KB
RevLine 
[847]1//
2// C++ Interface: Scantable
3//
4// Description:
5//
6//
7// Author: Malte Marquarding <asap@atnf.csiro.au>, (C) 2006
8//
9// Copyright: See COPYING file that comes with this distribution
10//
11//
12#ifndef ASAPSCANTABLEWRAPPER_H
13#define ASAPSCANTABLEWRAPPER_H
[2]14
15#include <vector>
16#include <string>
[380]17#include <casa/Arrays/Vector.h>
[2]18
[465]19#include "MathUtils.h"
[901]20#include "STFit.h"
[1931]21#include "STFitEntry.h"
[847]22#include "Scantable.h"
[1598]23#include "STCoordinate.h"
[2]24
[83]25namespace asap {
[847]26/**
27  * This class contains and wraps a CountedPtr<Scantable>, as the CountedPtr
28  * class does not provide the dor operator which is need for references
29  * in boost::python
30  * see Scantable for interfce description
31  *
32  * @brief The main ASAP data container wrapper
33  * @author Malte Marquarding
34  * @date 2006-02-23
35  * @version 2.0a
36*/
37class ScantableWrapper {
[2]38
39public:
[1385]40  explicit ScantableWrapper( const std::string& name,
[1077]41                    int type=0)
[864]42  {
43    casa::Table::TableType tp = casa::Table::Memory;
[1077]44    if ( type == 1 ) tp = casa::Table::Plain;
[864]45    table_ = new Scantable(name, tp);
46  }
[90]47
[1350]48  explicit ScantableWrapper(int type=0)
[864]49  {
50    casa::Table::TableType tp = casa::Table::Memory;
[1077]51    if ( type == 1) tp = casa::Table::Plain;
[864]52    table_= new Scantable(tp);
53  }
[90]54
[1350]55  explicit ScantableWrapper(casa::CountedPtr<Scantable> cp) : table_(cp) {;}
[90]56
[847]57  ScantableWrapper(const ScantableWrapper& mt) :
58    table_(mt.getCP()) {;}
[90]59
[868]60  void assign(const ScantableWrapper& mt)
61    { table_= mt.getCP(); }
62
[847]63  ScantableWrapper copy() {
64    return ScantableWrapper(new Scantable(*(this->getCP()), false));
[80]65  }
[868]66
[896]67  std::vector<float> getSpectrum( int whichrow=0,
[905]68                                  const std::string& poltype="" ) const {
[896]69    return table_->getSpectrum(whichrow, poltype);
[2]70  }
[527]71  //  std::string getPolarizationLabel(bool linear, bool stokes, bool linPol, int polIdx) const {
72  // Boost fails with 4 arguments.
[902]73  std::string getPolarizationLabel(int index, const std::string& ptype) const {
74    return table_->getPolarizationLabel(index, ptype);
[494]75  }
[527]76
[896]77  std::vector<double> getAbcissa(int whichrow=0) const
78    { return table_->getAbcissa(whichrow); }
[41]79
[896]80  std::string getAbcissaLabel(int whichrow=0) const
81    { return table_->getAbcissaLabel(whichrow); }
[157]82
[896]83  float getTsys(int whichrow=0) const
84    { return table_->getTsys(whichrow); }
[2]85
[2161]86  std::vector<float> getTsysSpectrum(int whichrow=0) const
87    { return table_->getTsysSpectrum(whichrow); }
88
[1947]89  //std::string getTime(int whichrow=0) const
90  //  { return table_->getTime(whichrow); }
91  std::string getTime(int whichrow=0, int prec = 0) const
92    { return table_->getTime(whichrow, true, casa::uInt(prec)); }
[207]93
[1350]94  double getIntTime(int whichrow=0) const
95    { return table_->getIntTime(whichrow); }
96
[1068]97  std::string getDirectionString(int whichrow=0) const
98    { return table_->getDirectionString(whichrow); }
99
[864]100  std::string getFluxUnit() const { return table_->getFluxUnit(); }
101
102  void setFluxUnit(const std::string& unit) { table_->setFluxUnit(unit); }
103
[238]104  void setInstrument(const std::string& name) {table_->setInstrument(name);}
[1189]105  void setFeedType(const std::string& ftype) {table_->setFeedType(ftype);}
[238]106
[896]107  std::vector<bool> getMask(int whichrow=0) const
108    { return table_->getMask(whichrow); }
[2]109
[1430]110  /**
[1000]111  void flag(const std::vector<bool>& msk=std::vector<bool>())
112    { table_->flag(msk); }
[1430]113  **/
[1994]114  /**
[1430]115  void flag(const std::vector<bool>& msk=std::vector<bool>(), bool unflag=false)
116    { table_->flag(msk, unflag); }
[1994]117  **/
118  void flag(int whichrow=-1, const std::vector<bool>& msk=std::vector<bool>(), bool unflag=false)
119    { table_->flag(whichrow, msk, unflag); }
[2]120
[1819]121  void flagRow(const std::vector<casa::uInt>& rows=std::vector<casa::uInt>(), bool unflag=false)
122    { table_->flagRow(rows, unflag); }
123
124  bool getFlagRow(int whichrow=0) const
125    { return table_->getFlagRow(whichrow); }
126
127  void clip(const casa::Float uthres, const casa::Float dthres, bool clipoutside=true, bool unflag=false)
128    { table_->clip(uthres, dthres, clipoutside, unflag); }
129
130  std::vector<bool> getClipMask(int whichrow, const casa::Float uthres, const casa::Float dthres, bool clipoutside, bool unflag) const
131    { return table_->getClipMask(whichrow, uthres, dthres, clipoutside, unflag); }
132
[896]133  std::string getSourceName(int whichrow=0) const
134    { return table_->getSourceName(whichrow); }
[794]135
[896]136  float getElevation(int whichrow=0) const
137    { return table_->getElevation(whichrow); }
[90]138
[896]139  float getAzimuth(int whichrow=0) const
140    { return table_->getAzimuth(whichrow); }
[2]141
[896]142  float getParAngle(int whichrow=0) const
143    { return table_->getParAngle(whichrow); }
[2]144
[864]145
[884]146  void setSpectrum(std::vector<float> spectrum, int whichrow=0)
147    { table_->setSpectrum(spectrum, whichrow); }
[868]148
[1111]149  std::vector<uint> getIFNos() { return table_->getIFNos(); }
[864]150  int getIF(int whichrow) const {return table_->getIF(whichrow);}
[1111]151  std::vector<uint> getBeamNos() { return table_->getBeamNos(); }
[864]152  int getBeam(int whichrow) const {return table_->getBeam(whichrow);}
[1111]153  std::vector<uint> getPolNos() { return table_->getPolNos(); }
[864]154  int getPol(int whichrow) const {return table_->getPol(whichrow);}
[868]155  int getCycle(int whichrow) const {return table_->getCycle(whichrow);}
[1111]156  std::vector<uint> getScanNos() { return table_->getScanNos(); }
[868]157  int getScan(int whichrow) const {return table_->getScan(whichrow);}
[1819]158  std::vector<uint> getMolNos() { return table_->getMolNos();}
[864]159
160  STSelector getSelection() const { return table_->getSelection(); }
[868]161  void setSelection(const STSelector& sts)
162    { return table_->setSelection(sts);}
[864]163
[896]164  std::string getPolType() const { return table_->getPolType(); }
165
[864]166  int nif(int scanno=-1) const {return table_->nif(scanno);}
167  int nbeam(int scanno=-1) const {return table_->nbeam(scanno);}
168  int npol(int scanno=-1) const {return table_->npol(scanno);}
169  int nchan(int ifno=-1) const {return table_->nchan(ifno);}
170  int nscan() const {return table_->nscan();}
171  int nrow() const {return table_->nrow();}
[902]172  int ncycle(int scanno) const {return table_->ncycle(scanno);}
[2]173
[864]174  void makePersistent(const std::string& fname)
175    { table_->makePersistent(fname); }
[90]176
[1068]177  void setSourceType(int stype)
178    { table_->setSourceType(stype); }
179
[1360]180  void shift(int npix)
181  { table_->shift(npix); }
182
[1819]183/**
184  commented out by TT
[1170]185  void setRestFrequencies(double rf, const std::string& name,
186                          const std::string& unit)
187    { table_->setRestFrequencies(rf, name, unit); }
[1819]188**/
189  void setRestFrequencies(vector<double> rf, const vector<std::string>& name,
190                          const std::string& unit)
191    { table_->setRestFrequencies(rf, name, unit); }
192
[925]193/*
[868]194  void setRestFrequencies(const std::string& name) {
195    table_->setRestFrequencies(name);
[847]196  }
[925]197*/
[745]198
[1819]199/*
[864]200  std::vector<double> getRestFrequencies() const
201    { return table_->getRestFrequencies(); }
[1819]202*/
203  std::vector<double> getRestFrequency(int id) const
204    { return table_->getRestFrequency(id); }
[251]205
[105]206  void setCoordInfo(std::vector<string> theinfo) {
207    table_->setCoordInfo(theinfo);
208  }
209  std::vector<string> getCoordInfo() const {
210    return table_->getCoordInfo();
211  }
[90]212
[987]213  void setDirection(const std::string& refstr="")
214    { table_->setDirectionRefString(refstr); }
215
[847]216  casa::CountedPtr<Scantable> getCP() const {return table_;}
217  Scantable* getPtr() {return &(*table_);}
[380]218
[2286]219  //  std::string summary() const {
220  //  return table_->summary();
221  //  }
222  void summary(const std::string& filename="") {
223    return table_->summary(filename);
[380]224  }
[745]225
[2178]226  std::string listHeader() const {
[2163]227    return table_->headerSummary();
[2111]228  }
229
[2163]230  std::vector<std::string> getHistory() const
[864]231    { return table_->getHistory(); }
[207]232
[864]233  void addHistory(const std::string& hist)
234    { table_->addHistory(hist); }
[745]235
[972]236  void addFit(const STFitEntry& fit, int row)
237    { table_->addFit(fit, row); }
238
239  STFitEntry getFit(int whichrow) const
240  { return table_->getFit(whichrow); }
241
[2012]242  void calculateAZEL() { table_->calculateAZEL(); }
[465]243
[1003]244  std::vector<std::string> columnNames() const
245    { return table_->columnNames(); }
246
[1391]247  std::string getAntennaName() const
248    { return table_->getAntennaName(); }
249
250  int checkScanInfo(const vector<int>& scanlist) const
251    { return table_->checkScanInfo(scanlist); }
[1819]252 
[1730]253  std::vector<double> getDirectionVector(int whichrow) const
[1391]254    { return table_->getDirectionVector(whichrow); }
255
[1586]256  void parallactify(bool flag)
257    { table_->parallactify(flag); }
258
[1598]259  STCoordinate getCoordinate(int row) {
260    return STCoordinate(table_->getSpectralCoordinate(row));
261  }
262
[1730]263  std::vector<float> getWeather(int whichrow) const
264    { return table_->getWeather(whichrow); }
265
[1819]266  void reshapeSpectrum( int nmin, int nmax )
267  { table_->reshapeSpectrum( nmin, nmax ); }
268
[2189]269  void polyBaseline(const std::vector<bool>& mask, int order, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
270  { table_->polyBaseline(mask, order, getresidual, showprogress, outlog, blfile); }
[1907]271
[2189]272  void autoPolyBaseline(const std::vector<bool>& mask, int order, const std::vector<int>& edge, float threshold=5.0, int chan_avg_limit=1, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
273  { table_->autoPolyBaseline(mask, order, edge, threshold, chan_avg_limit, getresidual, showprogress, outlog, blfile); }
[1907]274
[2189]275  void cubicSplineBaseline(const std::vector<bool>& mask, int npiece, float clipthresh, int clipniter, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
276  { table_->cubicSplineBaseline(mask, npiece, clipthresh, clipniter, getresidual, showprogress, outlog, blfile); }
[2012]277
[2189]278  void autoCubicSplineBaseline(const std::vector<bool>& mask, int npiece, float clipthresh, int clipniter, const std::vector<int>& edge, float threshold=5.0, int chan_avg_limit=1, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
279  { table_->autoCubicSplineBaseline(mask, npiece, clipthresh, clipniter, edge, threshold, chan_avg_limit, getresidual, showprogress, outlog, blfile); }
[2012]280
[2189]281  void sinusoidBaseline(const std::vector<bool>& mask, const bool applyfft, const std::string& fftmethod, const std::string& fftthresh, const std::vector<int>& addwn, const std::vector<int>& rejwn, float clipthresh, int clipniter, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
282  { table_->sinusoidBaseline(mask, applyfft, fftmethod, fftthresh, addwn, rejwn, clipthresh, clipniter, getresidual, showprogress, outlog, blfile); }
[2047]283
[2189]284  void autoSinusoidBaseline(const std::vector<bool>& mask, const bool applyfft, const std::string& fftmethod, const std::string& fftthresh, const std::vector<int>& addwn, const std::vector<int>& rejwn, float clipthresh, int clipniter, const std::vector<int>& edge, float threshold=5.0, int chan_avg_limit=1, bool getresidual=true, const std::string& showprogress="true,1000", const bool outlog=false, const std::string& blfile="")
285  { table_->autoSinusoidBaseline(mask, applyfft, fftmethod, fftthresh, addwn, rejwn, clipthresh, clipniter, edge, threshold, chan_avg_limit, getresidual, showprogress, outlog, blfile); }
[2047]286
[2012]287  float getRms(const std::vector<bool>& mask, int whichrow)
288  { return table_->getRms(mask, whichrow); }
289
290  std::string formatBaselineParams(const std::vector<float>& params, const std::vector<bool>& fixed, float rms, const std::string& masklist, int whichrow, bool verbose=false)
[2193]291  { return table_->formatBaselineParams(params, fixed, rms, -1, masklist, whichrow, verbose); }
[2012]292
293  std::string formatPiecewiseBaselineParams(const std::vector<int>& ranges, const std::vector<float>& params, const std::vector<bool>& fixed, float rms, const std::string& masklist, int whichrow, bool verbose=false)
[2193]294  { return table_->formatPiecewiseBaselineParams(ranges, params, fixed, rms, -1, masklist, whichrow, verbose); }
[2012]295
[1907]296  bool getFlagtraFast(int whichrow=0) const
[2047]297  { return table_->getFlagtraFast(casa::uInt(whichrow)); }
[1907]298
[2186]299  std::vector<float> execFFT(int whichrow, const std::vector<bool>& mask, bool getRealImag=false, bool getAmplitudeOnly=false)
300  { return table_->execFFT(whichrow, mask, getRealImag, getAmplitudeOnly); }
[1907]301
[2012]302
[2186]303
[2]304private:
[847]305  casa::CountedPtr<Scantable> table_;
[2]306};
307
308} // namespace
309#endif
[1819]310
Note: See TracBrowser for help on using the repository browser.