source: trunk/src/python_Scantable.cpp@ 2418

Last change on this file since 2418 was 2286, checked in by Kana Sugimoto, 13 years ago

New Development: No (performance tuning)

JIRA Issue: No

Ready for Test: Yes

Interface Changes: Yes

What Interface Changed: a parameter "filename" is added to Scantable::summary. scantable.summary doesn't return a string anymore

Test Programs: sdlist unittest/ scantable.summary("summary.txt")

Put in Release Notes: Yes

Module(s): sdlist, asap.summary

Description:

scantable.summary is very slow for large data sets (in row number) often outputted
by modern telescopes. It takes > 1.5 hours to list OTF raster scan with 350,000 rows.

This was because, the methods accumulates the whole text string (~700,000 lines) and
returns it as a string. Once the summary string exceed several tens thousands lines,
elapse time increases non-linearly, may be because very massive output string starts
to overweigh the memory.

I updated scantable.summary so that it flushes the summary string more often to file/logger.
After the modification, scantable.summary could list the data mentioned above in ~ 7 minutes.
The side effect of it is that scantable.summary doesn't return summary string anymore.
(But people may not happy with sub-million lines of string anyway.)


  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 7.6 KB
RevLine 
[2]1//#---------------------------------------------------------------------------
[872]2//# python_Scantable.cc: python exposure of c++ Scantable class
[2]3//#---------------------------------------------------------------------------
4//# Copyright (C) 2004
[125]5//# ATNF
[2]6//#
7//# This program is free software; you can redistribute it and/or modify it
8//# under the terms of the GNU General Public License as published by the Free
9//# Software Foundation; either version 2 of the License, or (at your option)
10//# any later version.
11//#
12//# This program is distributed in the hope that it will be useful, but
13//# WITHOUT ANY WARRANTY; without even the implied warranty of
14//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
15//# Public License for more details.
16//#
17//# You should have received a copy of the GNU General Public License along
18//# with this program; if not, write to the Free Software Foundation, Inc.,
19//# 675 Massachusetts Ave, Cambridge, MA 02139, USA.
20//#
21//# Correspondence concerning this software should be addressed as follows:
22//# Internet email: Malte.Marquarding@csiro.au
23//# Postal address: Malte Marquarding,
24//# Australia Telescope National Facility,
25//# P.O. Box 76,
26//# Epping, NSW, 2121,
27//# AUSTRALIA
28//#
[1068]29//# $Id: python_Scantable.cpp 2286 2011-09-02 09:05:11Z KanaSugimoto $
[2]30//#---------------------------------------------------------------------------
31#include <vector>
32
33#include <boost/python.hpp>
[16]34#include <boost/python/args.hpp>
[125]35
[872]36#include "ScantableWrapper.h"
[2]37
38using namespace boost::python;
39
[83]40namespace asap {
[2]41 namespace python {
42
[872]43void python_Scantable() {
44 class_<ScantableWrapper>("Scantable")
[96]45 //.def( init <> () )
[1077]46 .def( init < int > () )
47 .def( init < const std::string&, int > () )
[872]48 .def( init < const ScantableWrapper& > () )
49 .def("_copy", &ScantableWrapper::copy)
50 .def("_assign", &ScantableWrapper::assign)
51 .def("getif", &ScantableWrapper::getIF)
[1111]52 .def("getifnos", &ScantableWrapper::getIFNos)
[872]53 .def("getbeam", &ScantableWrapper::getBeam)
[1111]54 .def("getbeamnos", &ScantableWrapper::getBeamNos)
[872]55 .def("getpol", &ScantableWrapper::getPol)
[1111]56 .def("getpolnos", &ScantableWrapper::getPolNos)
[872]57 .def("getscan", &ScantableWrapper::getScan)
[1111]58 .def("getscannos", &ScantableWrapper::getScanNos)
[872]59 .def("getcycle", &ScantableWrapper::getCycle)
[1819]60 .def("getmolnos", &ScantableWrapper::getMolNos)
[902]61 .def("nif", &ScantableWrapper::nif,
62 (boost::python::arg("scanno")=-1) )
63 .def("nbeam", &ScantableWrapper::nbeam,
64 (boost::python::arg("scanno")=-1) )
65 .def("npol", &ScantableWrapper::npol,
66 (boost::python::arg("scanno")=-1) )
67 .def("nchan", &ScantableWrapper::nchan,
68 (boost::python::arg("ifno")=-1) )
69 .def("ncycle", &ScantableWrapper::ncycle,
70 (boost::python::arg("scanno")=-1) )
[872]71 .def("nscan", &ScantableWrapper::nscan)
72 .def("nrow", &ScantableWrapper::nrow)
73 .def("get_fluxunit", &ScantableWrapper::getFluxUnit)
74 .def("set_fluxunit", &ScantableWrapper::setFluxUnit)
75 .def("_setInstrument", &ScantableWrapper::setInstrument)
[1189]76 .def("_setfeedtype", &ScantableWrapper::setFeedType)
[872]77 .def("_getspectrum", &ScantableWrapper::getSpectrum,
[905]78 (arg("whichrow")=0, arg("poltype")=std::string("")) )
[896]79 .def("poltype", &ScantableWrapper::getPolType )
[1259]80 .def("get_column_names", &ScantableWrapper::columnNames)
[902]81 .def("_getpollabel", &ScantableWrapper::getPolarizationLabel)
[896]82 .def("_setspectrum",&ScantableWrapper::setSpectrum,
[884]83 (boost::python::arg("whichrow")=0) )
[872]84 .def("_getabcissa", &ScantableWrapper::getAbcissa,
[884]85 (boost::python::arg("whichrow")=0) )
[872]86 .def("_getabcissalabel", &ScantableWrapper::getAbcissaLabel,
[884]87 (boost::python::arg("whichrow")=0) )
[872]88 .def("_getmask", &ScantableWrapper::getMask,
[884]89 (boost::python::arg("whichrow")=0) )
[1819]90 .def("_getclipmask", &ScantableWrapper::getClipMask,
91 (boost::python::arg("whichrow")=0) )
[872]92 .def("_gettsys", &ScantableWrapper::getTsys)
[2161]93 .def("_gettsysspectrum", &ScantableWrapper::getTsysSpectrum )
[872]94 .def("_getsourcename", &ScantableWrapper::getSourceName,
[884]95 (boost::python::arg("whichrow")=0) )
[872]96 .def("_getelevation", &ScantableWrapper::getElevation,
[884]97 (boost::python::arg("whichrow")=0) )
[872]98 .def("_getazimuth", &ScantableWrapper::getAzimuth,
[884]99 (boost::python::arg("whichrow")=0) )
[872]100 .def("_getparangle", &ScantableWrapper::getParAngle,
[884]101 (boost::python::arg("whichrow")=0) )
[1947]102 //.def("_gettime", &ScantableWrapper::getTime,
103 // (boost::python::arg("whichrow")=0) )
[872]104 .def("_gettime", &ScantableWrapper::getTime,
[1947]105 (boost::python::arg("whichrow")=0,
106 boost::python::arg("prec")=0) )
[1350]107 .def("_getinttime", &ScantableWrapper::getIntTime,
108 (boost::python::arg("whichrow")=0) )
[1068]109 .def("_getdirection", &ScantableWrapper::getDirectionString,
110 (boost::python::arg("whichrow")=0) )
[1391]111 .def("get_antennaname", &ScantableWrapper::getAntennaName)
[1819]112 .def("_flag", &ScantableWrapper::flag)
113 .def("_flag_row", &ScantableWrapper::flagRow)
114 .def("_getflagrow", &ScantableWrapper::getFlagRow,
115 (boost::python::arg("whichrow")=0) )
116 .def("_clip", &ScantableWrapper::clip,
117 (boost::python::arg("clipoutside")=true,
118 boost::python::arg("unflag")=false) )
[872]119 .def("_save", &ScantableWrapper::makePersistent)
[2286]120 //.def("_summary", &ScantableWrapper::summary)
121 .def("_summary", &ScantableWrapper::summary,
122 (boost::python::arg("filename")=""))
[2178]123 .def("_list_header", &ScantableWrapper::listHeader)
[1819]124 //.def("_getrestfreqs", &ScantableWrapper::getRestFrequencies)
125 .def("_getrestfreqs", &ScantableWrapper::getRestFrequency)
[925]126 .def("_setrestfreqs", &ScantableWrapper::setRestFrequencies)
[1360]127 .def("shift_refpix", &ScantableWrapper::shift)
[872]128 .def("_setcoordinfo", &ScantableWrapper::setCoordInfo)
129 .def("_getcoordinfo", &ScantableWrapper::getCoordInfo)
[987]130 .def("set_dirframe", &ScantableWrapper::setDirection,
131 (boost::python::arg("refstr")="") )
[872]132 .def("_gethistory", &ScantableWrapper::getHistory)
133 .def("_addhistory", &ScantableWrapper::addHistory)
134 .def("_getselection", &ScantableWrapper::getSelection)
135 .def("_setselection", &ScantableWrapper::setSelection)
136 .def("_addfit", &ScantableWrapper::addFit)
[972]137 .def("_getfit", &ScantableWrapper::getFit)
[872]138 .def("_recalcazel", &ScantableWrapper::calculateAZEL)
[1068]139 .def("_setsourcetype", &ScantableWrapper::setSourceType)
[1391]140 .def("_getdirectionvec", &ScantableWrapper::getDirectionVector)
[1586]141 .def("_parallactify", &ScantableWrapper::parallactify)
[1598]142 .def("get_coordinate", &ScantableWrapper::getCoordinate)
[1730]143 .def("_get_weather", &ScantableWrapper::getWeather)
[1819]144 .def("_reshape", &ScantableWrapper::reshapeSpectrum,
145 (boost::python::arg("nmin")=-1,
146 boost::python::arg("nmax")=-1) )
[1907]147 .def("_poly_baseline", &ScantableWrapper::polyBaseline)
[2012]148 .def("_auto_poly_baseline", &ScantableWrapper::autoPolyBaseline)
149 .def("_cspline_baseline", &ScantableWrapper::cubicSplineBaseline)
150 .def("_auto_cspline_baseline", &ScantableWrapper::autoCubicSplineBaseline)
[2047]151 .def("_sinusoid_baseline", &ScantableWrapper::sinusoidBaseline)
152 .def("_auto_sinusoid_baseline", &ScantableWrapper::autoSinusoidBaseline)
[2012]153 .def("get_rms", &ScantableWrapper::getRms)
154 .def("format_blparams_row", &ScantableWrapper::formatBaselineParams)
155 .def("format_piecewise_blparams_row", &ScantableWrapper::formatPiecewiseBaselineParams)
[1907]156 .def("_getflagtrafast", &ScantableWrapper::getFlagtraFast,
157 (boost::python::arg("whichrow")=0) )
[2186]158 .def("_fft", &ScantableWrapper::execFFT)
[2125]159 //.def("_sspline_baseline", &ScantableWrapper::smoothingSplineBaseline)
[2012]160 //.def("_test_cin", &ScantableWrapper::testCin)
[2]161 ;
162};
163
164 } // python
[83]165} // asap
Note: See TracBrowser for help on using the repository browser.