Changeset 2286

Timestamp:

09/02/11 19:05:11 (13 years ago)

Author:

Kana Sugimoto

Message:

New Development: No (performance tuning)

JIRA Issue: No

Ready for Test: Yes

Interface Changes: Yes

What Interface Changed: a parameter "filename" is added to Scantable::summary. scantable.summary doesn't return a string anymore

Test Programs: sdlist unittest/ scantable.summary("summary.txt")

Put in Release Notes: Yes

Module(s): sdlist, asap.summary

Description:

scantable.summary is very slow for large data sets (in row number) often outputted
by modern telescopes. It takes > 1.5 hours to list OTF raster scan with 350,000 rows.

This was because, the methods accumulates the whole text string (~700,000 lines) and
returns it as a string. Once the summary string exceed several tens thousands lines,
elapse time increases non-linearly, may be because very massive output string starts
to overweigh the memory.

I updated scantable.summary so that it flushes the summary string more often to file/logger.
After the modification, scantable.summary could list the data mentioned above in ~ 7 minutes.
The side effect of it is that scantable.summary doesn't return summary string anymore.
(But people may not happy with sub-million lines of string anyway.)

Location:

trunk

Files:

• python/scantable.py (modified) (2 diffs)
• src/Scantable.cpp (modified) (3 diffs)
• src/Scantable.h (modified) (1 diff)
• src/ScantableWrapper.h (modified) (1 diff)
• src/python_Scantable.cpp (modified) (1 diff)

trunk/python/scantable.py

@@ -445 +445 @@
 
         """
-        info = Scantable._summary(self)
+#         info = Scantable._summary(self)
         if filename is not None:
             if filename is "":
@@ -451 +451 @@
             from os.path import expandvars, isdir
             filename = expandvars(filename)
-            if not isdir(filename):
-                data = open(filename, 'w')
-                data.write(info)
-                data.close()
-            else:
+#             if not isdir(filename):
+#                 data = open(filename, 'w')
+#                 data.write(info)
+#                 data.close()
+#             else:
+            if isdir(filename):
                 msg = "Illegal file name '%s'." % (filename)
                 raise IOError(msg)
-        return page(info)
+        else:
+            filename = ""
+        Scantable._summary(self, filename)
+#         return page(info)
 
     def get_spectrum(self, rowno):

trunk/src/Scantable.cpp

@@ -1040 +1040 @@
 }
 
-std::string Scantable::summary()
+  //std::string Scantable::summary( const std::string& filename )
+void Scantable::summary( const std::string& filename )
 {
   ostringstream oss;
+  ofstream ofs;
+  LogIO ols(LogOrigin("Scantable", "summary", WHERE));
+
+  if (filename != "")
+    ofs.open( filename.c_str(),  ios::out );
+
   oss << endl;
   oss << asap::SEPERATOR << endl;
@@ -1067 +1074 @@
       << endl;
   oss << asap::SEPERATOR << endl;
+
+  // Flush summary and clear up the string
+  ols << String(oss) << LogIO::POST;
+  if (ofs) ofs << String(oss) << flush;
+  oss.str("");
+  oss.clear();
+
+//   TableIterator iter(table_, "SCANNO");
+//   while (!iter.pastEnd()) {
+//     Table subt = iter.table();
+//     ROTableRow row(subt);
+//     MEpoch::ROScalarColumn timeCol(subt,"TIME");
+//     const TableRecord& rec = row.get(0);
+//     oss << setw(4) << std::right << rec.asuInt("SCANNO")
+//         << std::left << setw(1) << ""
+//         << setw(15) << rec.asString("SRCNAME")
+//         << setw(10) << formatTime(timeCol(0), false);
+//     // count the cycles in the scan
+//     TableIterator cyciter(subt, "CYCLENO");
+//     int nint = 0;
+//     while (!cyciter.pastEnd()) {
+//       ++nint;
+//       ++cyciter;
+//     }
+//     oss << setw(3) << std::right << nint  << setw(3) << " x " << std::left
+//         << setw(11) <<  formatSec(rec.asFloat("INTERVAL")) << setw(1) << ""
+//      << setw(15) << SrcType::getName(rec.asInt("SRCTYPE")) << endl;
+
+//     TableIterator biter(subt, "BEAMNO");
+//     while (!biter.pastEnd()) {
+//       Table bsubt = biter.table();
+//       ROTableRow brow(bsubt);
+//       const TableRecord& brec = brow.get(0);
+//       uInt row0 = bsubt.rowNumbers(table_)[0];
+//       oss << setw(5) << "" <<  setw(4) << std::right << brec.asuInt("BEAMNO")<< std::left;
+//       oss  << setw(4) << ""  << formatDirection(getDirection(row0)) << endl;
+//       TableIterator iiter(bsubt, "IFNO");
+//       while (!iiter.pastEnd()) {
+//         Table isubt = iiter.table();
+//         ROTableRow irow(isubt);
+//         const TableRecord& irec = irow.get(0);
+//         oss << setw(9) << "";
+//         oss << setw(3) << std::right << irec.asuInt("IFNO") << std::left
+//             << setw(1) << "" << frequencies().print(irec.asuInt("FREQ_ID"))
+//             << setw(3) << "" << nchan(irec.asuInt("IFNO"))
+//             << endl;
+
+//         ++iiter;
+//       }
+//       ++biter;
+//     }
+//     ++iter;
+//   }
   TableIterator iter(table_, "SCANNO");
   while (!iter.pastEnd()) {
@@ -1111 +1171 @@
       ++biter;
     }
+    // Flush summary every scan and clear up the string
+    ols << String(oss) << LogIO::POST;
+    if (ofs) ofs << String(oss) << flush;
+    oss.str("");
+    oss.clear();
+
     ++iter;
   }
-  return String(oss);
+  oss << asap::SEPERATOR << endl;
+  ols << String(oss) << LogIO::POST;
+  if (ofs) {
+    //ofs << String(oss) << flush;
+    ofs.close();
+  }
+  //  return String(oss);
 }
 

trunk/src/Scantable.h

@@ -375 +375 @@
 
   std::string headerSummary();
-  std::string summary();
+  //  std::string summary();
+  void summary(const std::string& filename="");
   //std::string getTime(int whichrow=-1, bool showdate=true) const;
   std::string getTime(int whichrow=-1, bool showdate=true, casa::uInt prec=0) const;

trunk/src/ScantableWrapper.h

@@ -217 +217 @@
   Scantable* getPtr() {return &(*table_);}
 
-  std::string summary() const {
-    return table_->summary();
+  //  std::string summary() const {
+  //  return table_->summary();
+  //  }
+  void summary(const std::string& filename="") {
+    return table_->summary(filename);
   }
 

trunk/src/python_Scantable.cpp

@@ -118 +118 @@
           boost::python::arg("unflag")=false) )
     .def("_save",  &ScantableWrapper::makePersistent)
-    .def("_summary",  &ScantableWrapper::summary)
+    //.def("_summary",  &ScantableWrapper::summary)
+    .def("_summary",  &ScantableWrapper::summary,
+         (boost::python::arg("filename")=""))
     .def("_list_header",  &ScantableWrapper::listHeader)
     //.def("_getrestfreqs",  &ScantableWrapper::getRestFrequencies)