source: branches/hpc33/src/python_Scantable.cpp@ 2567

Last change on this file since 2567 was 2286, checked in by Kana Sugimoto, 13 years ago

New Development: No (performance tuning)

JIRA Issue: No

Ready for Test: Yes

Interface Changes: Yes

What Interface Changed: a parameter "filename" is added to Scantable::summary. scantable.summary doesn't return a string anymore

Test Programs: sdlist unittest/ scantable.summary("summary.txt")

Put in Release Notes: Yes

Module(s): sdlist, asap.summary

Description:

scantable.summary is very slow for large data sets (in row number) often outputted
by modern telescopes. It takes > 1.5 hours to list OTF raster scan with 350,000 rows.

This was because, the methods accumulates the whole text string (~700,000 lines) and
returns it as a string. Once the summary string exceed several tens thousands lines,
elapse time increases non-linearly, may be because very massive output string starts
to overweigh the memory.

I updated scantable.summary so that it flushes the summary string more often to file/logger.
After the modification, scantable.summary could list the data mentioned above in ~ 7 minutes.
The side effect of it is that scantable.summary doesn't return summary string anymore.
(But people may not happy with sub-million lines of string anyway.)


  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 7.6 KB
Line 
1//#---------------------------------------------------------------------------
2//# python_Scantable.cc: python exposure of c++ Scantable class
3//#---------------------------------------------------------------------------
4//# Copyright (C) 2004
5//# ATNF
6//#
7//# This program is free software; you can redistribute it and/or modify it
8//# under the terms of the GNU General Public License as published by the Free
9//# Software Foundation; either version 2 of the License, or (at your option)
10//# any later version.
11//#
12//# This program is distributed in the hope that it will be useful, but
13//# WITHOUT ANY WARRANTY; without even the implied warranty of
14//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
15//# Public License for more details.
16//#
17//# You should have received a copy of the GNU General Public License along
18//# with this program; if not, write to the Free Software Foundation, Inc.,
19//# 675 Massachusetts Ave, Cambridge, MA 02139, USA.
20//#
21//# Correspondence concerning this software should be addressed as follows:
22//# Internet email: Malte.Marquarding@csiro.au
23//# Postal address: Malte Marquarding,
24//# Australia Telescope National Facility,
25//# P.O. Box 76,
26//# Epping, NSW, 2121,
27//# AUSTRALIA
28//#
29//# $Id: python_Scantable.cpp 2286 2011-09-02 09:05:11Z KanaSugimoto $
30//#---------------------------------------------------------------------------
31#include <vector>
32
33#include <boost/python.hpp>
34#include <boost/python/args.hpp>
35
36#include "ScantableWrapper.h"
37
38using namespace boost::python;
39
40namespace asap {
41 namespace python {
42
43void python_Scantable() {
44 class_<ScantableWrapper>("Scantable")
45 //.def( init <> () )
46 .def( init < int > () )
47 .def( init < const std::string&, int > () )
48 .def( init < const ScantableWrapper& > () )
49 .def("_copy", &ScantableWrapper::copy)
50 .def("_assign", &ScantableWrapper::assign)
51 .def("getif", &ScantableWrapper::getIF)
52 .def("getifnos", &ScantableWrapper::getIFNos)
53 .def("getbeam", &ScantableWrapper::getBeam)
54 .def("getbeamnos", &ScantableWrapper::getBeamNos)
55 .def("getpol", &ScantableWrapper::getPol)
56 .def("getpolnos", &ScantableWrapper::getPolNos)
57 .def("getscan", &ScantableWrapper::getScan)
58 .def("getscannos", &ScantableWrapper::getScanNos)
59 .def("getcycle", &ScantableWrapper::getCycle)
60 .def("getmolnos", &ScantableWrapper::getMolNos)
61 .def("nif", &ScantableWrapper::nif,
62 (boost::python::arg("scanno")=-1) )
63 .def("nbeam", &ScantableWrapper::nbeam,
64 (boost::python::arg("scanno")=-1) )
65 .def("npol", &ScantableWrapper::npol,
66 (boost::python::arg("scanno")=-1) )
67 .def("nchan", &ScantableWrapper::nchan,
68 (boost::python::arg("ifno")=-1) )
69 .def("ncycle", &ScantableWrapper::ncycle,
70 (boost::python::arg("scanno")=-1) )
71 .def("nscan", &ScantableWrapper::nscan)
72 .def("nrow", &ScantableWrapper::nrow)
73 .def("get_fluxunit", &ScantableWrapper::getFluxUnit)
74 .def("set_fluxunit", &ScantableWrapper::setFluxUnit)
75 .def("_setInstrument", &ScantableWrapper::setInstrument)
76 .def("_setfeedtype", &ScantableWrapper::setFeedType)
77 .def("_getspectrum", &ScantableWrapper::getSpectrum,
78 (arg("whichrow")=0, arg("poltype")=std::string("")) )
79 .def("poltype", &ScantableWrapper::getPolType )
80 .def("get_column_names", &ScantableWrapper::columnNames)
81 .def("_getpollabel", &ScantableWrapper::getPolarizationLabel)
82 .def("_setspectrum",&ScantableWrapper::setSpectrum,
83 (boost::python::arg("whichrow")=0) )
84 .def("_getabcissa", &ScantableWrapper::getAbcissa,
85 (boost::python::arg("whichrow")=0) )
86 .def("_getabcissalabel", &ScantableWrapper::getAbcissaLabel,
87 (boost::python::arg("whichrow")=0) )
88 .def("_getmask", &ScantableWrapper::getMask,
89 (boost::python::arg("whichrow")=0) )
90 .def("_getclipmask", &ScantableWrapper::getClipMask,
91 (boost::python::arg("whichrow")=0) )
92 .def("_gettsys", &ScantableWrapper::getTsys)
93 .def("_gettsysspectrum", &ScantableWrapper::getTsysSpectrum )
94 .def("_getsourcename", &ScantableWrapper::getSourceName,
95 (boost::python::arg("whichrow")=0) )
96 .def("_getelevation", &ScantableWrapper::getElevation,
97 (boost::python::arg("whichrow")=0) )
98 .def("_getazimuth", &ScantableWrapper::getAzimuth,
99 (boost::python::arg("whichrow")=0) )
100 .def("_getparangle", &ScantableWrapper::getParAngle,
101 (boost::python::arg("whichrow")=0) )
102 //.def("_gettime", &ScantableWrapper::getTime,
103 // (boost::python::arg("whichrow")=0) )
104 .def("_gettime", &ScantableWrapper::getTime,
105 (boost::python::arg("whichrow")=0,
106 boost::python::arg("prec")=0) )
107 .def("_getinttime", &ScantableWrapper::getIntTime,
108 (boost::python::arg("whichrow")=0) )
109 .def("_getdirection", &ScantableWrapper::getDirectionString,
110 (boost::python::arg("whichrow")=0) )
111 .def("get_antennaname", &ScantableWrapper::getAntennaName)
112 .def("_flag", &ScantableWrapper::flag)
113 .def("_flag_row", &ScantableWrapper::flagRow)
114 .def("_getflagrow", &ScantableWrapper::getFlagRow,
115 (boost::python::arg("whichrow")=0) )
116 .def("_clip", &ScantableWrapper::clip,
117 (boost::python::arg("clipoutside")=true,
118 boost::python::arg("unflag")=false) )
119 .def("_save", &ScantableWrapper::makePersistent)
120 //.def("_summary", &ScantableWrapper::summary)
121 .def("_summary", &ScantableWrapper::summary,
122 (boost::python::arg("filename")=""))
123 .def("_list_header", &ScantableWrapper::listHeader)
124 //.def("_getrestfreqs", &ScantableWrapper::getRestFrequencies)
125 .def("_getrestfreqs", &ScantableWrapper::getRestFrequency)
126 .def("_setrestfreqs", &ScantableWrapper::setRestFrequencies)
127 .def("shift_refpix", &ScantableWrapper::shift)
128 .def("_setcoordinfo", &ScantableWrapper::setCoordInfo)
129 .def("_getcoordinfo", &ScantableWrapper::getCoordInfo)
130 .def("set_dirframe", &ScantableWrapper::setDirection,
131 (boost::python::arg("refstr")="") )
132 .def("_gethistory", &ScantableWrapper::getHistory)
133 .def("_addhistory", &ScantableWrapper::addHistory)
134 .def("_getselection", &ScantableWrapper::getSelection)
135 .def("_setselection", &ScantableWrapper::setSelection)
136 .def("_addfit", &ScantableWrapper::addFit)
137 .def("_getfit", &ScantableWrapper::getFit)
138 .def("_recalcazel", &ScantableWrapper::calculateAZEL)
139 .def("_setsourcetype", &ScantableWrapper::setSourceType)
140 .def("_getdirectionvec", &ScantableWrapper::getDirectionVector)
141 .def("_parallactify", &ScantableWrapper::parallactify)
142 .def("get_coordinate", &ScantableWrapper::getCoordinate)
143 .def("_get_weather", &ScantableWrapper::getWeather)
144 .def("_reshape", &ScantableWrapper::reshapeSpectrum,
145 (boost::python::arg("nmin")=-1,
146 boost::python::arg("nmax")=-1) )
147 .def("_poly_baseline", &ScantableWrapper::polyBaseline)
148 .def("_auto_poly_baseline", &ScantableWrapper::autoPolyBaseline)
149 .def("_cspline_baseline", &ScantableWrapper::cubicSplineBaseline)
150 .def("_auto_cspline_baseline", &ScantableWrapper::autoCubicSplineBaseline)
151 .def("_sinusoid_baseline", &ScantableWrapper::sinusoidBaseline)
152 .def("_auto_sinusoid_baseline", &ScantableWrapper::autoSinusoidBaseline)
153 .def("get_rms", &ScantableWrapper::getRms)
154 .def("format_blparams_row", &ScantableWrapper::formatBaselineParams)
155 .def("format_piecewise_blparams_row", &ScantableWrapper::formatPiecewiseBaselineParams)
156 .def("_getflagtrafast", &ScantableWrapper::getFlagtraFast,
157 (boost::python::arg("whichrow")=0) )
158 .def("_fft", &ScantableWrapper::execFFT)
159 //.def("_sspline_baseline", &ScantableWrapper::smoothingSplineBaseline)
160 //.def("_test_cin", &ScantableWrapper::testCin)
161 ;
162};
163
164 } // python
165} // asap
Note: See TracBrowser for help on using the repository browser.