Index: /trunk/DBUpdater/DBUpdater.c
===================================================================
--- /trunk/DBUpdater/DBUpdater.c	(revision 422)
+++ /trunk/DBUpdater/DBUpdater.c	(revision 423)
@@ -1,4 +1,5 @@
 /**
  * Written by Dvaid Smith
+ * (c) 2008 ATNF CSIRO
  * 
  * This program scans the directories given on the command line for any pulsar observations
@@ -45,48 +46,9 @@
 
 
-/**
- * Take the files given to be a calibration and add an entry to the database
- * if none exists yet, otherwise mark the calibartion entry as present.
- **/
-
-void AddCalEntry( const vector<string> &files )
-{
-  /**
-   * if we have a single file
-   *   search the cals table for that file
-   *   if we have a match
-   *     mark it as present
-   *   otherwise
-   *     create a new entry in the database for that file
-   * otherwise
-   *   determine the band prefix from the first filename
-   *   search the cals table for a matching directory/band prefix
-   *   if we have a match
-   *     mark it as present
-   *   otherwise
-   *     create a new entry ni the database for that file
-   **/
-  
-  cerr << "Adding a cal entry for " << files.size() << " files under " << pathname(files.front()) << endl;
-}
-
-void AddCalEntry( const string &single_file )
-{
-  vector<string> c;
-  c.push_back( single_file );
-  AddCalEntry( c );
-}
-
-
-
-/**
- * Take the files given to be an observation and add an entry in the database
- * if none exists yet, otherwise mark the observation entry as present
- **/
-
-void AddObservationEntry( const vector<string> &files )
-{
-  cerr << "Adding a observation entry for " << files.size() << " files under " << files.front() << endl;
-  
+/// This function creates an ObservationEntry object that represents
+/// the files given
+
+ObservationEntry MakeObservationEntry( const vector<string> &files )
+{
   ObservationEntry entry;
   
@@ -104,8 +66,10 @@
   entry.files = files;
   
-  db.CheckObservation( entry );
-}
-
-void AddObservationEntry( const string &single_file )
+  return entry;
+}
+
+/// This function creates an ObservationEntry object for the file given.
+
+ObservationEntry MakeObservationEntry( const string &single_file )
 {
   ObservationEntry entry;
@@ -113,15 +77,18 @@
   entry.filename = basename( single_file );
   entry.data_loc = pathname( single_file );
-  entry.prefix = '\0';
-
-  db.CheckObservation( entry );
-}
-
-
-
-/**
- * This function iteratively examines a directory and calls itself to examine
- * any subdirectories
- **/
+  entry.prefix = "\\0";
+
+  return entry;
+}
+
+
+
+
+/// This function iteratively examines a directory and calls itself to examine
+/// any subdirectories. In each directory, it makes a list of all the filenames
+/// that match any pattern we expect for an archive file. We may have several lists
+/// for one directory if this directory contains cpsr2 data, so we add the filenames
+/// to separate lists. Once we have listed all the files, we create an observation
+/// for each filename (or list of filenames for cpsr2 data).
 
 void ExamineDirectory( const string path )
@@ -172,37 +139,12 @@
       else if( boost::regex_match( filename, r_cfb_matcher ) )
         r_cfb_files.push_back( (*it) );
-//       else
-//         cerr << "don't know how to handle file " << filename << endl;
     }
   }
-//   cerr << "Found " << rf_files.size() << " rf files " << endl;
-//   cerr << "Found " << cf_files.size() << " cf files " << endl;
-//   cerr << "Found " << m_fb_files.size() << " m*.fb files " << endl;
-//   cerr << "Found " << n_fb_files.size() << " n*.fb files " << endl;
-//   cerr << "Found " << m_cfb_files.size() << " m*.cfb files " << endl;
-//   cerr << "Found " << n_cfb_files.size() << " n*.cfb files " << endl;
-//   cerr << "Found " << r_cfb_files.size() << " r*.cfb files " << endl;
-  
-  /**
-   *   for each cf file
-   *     add this file as a calibration
-   *   for each rf file
-   *     if this directory ends in _R
-   *       add this file as a calibration
-   *     otherwise
-   *       add this file as an observation
-   * for each of the vectors n*.cfb m*.cfb r*.cfb m*.fb n*.fb
-   *   if we have any m*.cfb files
-   *     if this directory ends in _R
-   *       add these files as a calibration
-   *     otherwise
-   *       add these files as an observation
-   **/
   
   for( it = cf_files.begin(); it != cf_files.end(); it ++ )
-    AddCalEntry( (*it) );
+    db.CheckCalibration( MakeObservationEntry( (*it) ) );
   
   for( it = rf_files.begin(); it != rf_files.end(); it ++ )
-    AddObservationEntry( (*it) );
+    db.CheckObservation( MakeObservationEntry( (*it) ) );
   
   vector< vector<string> > cpsr2_observations;
@@ -213,6 +155,4 @@
   cpsr2_observations.push_back( n_fb_files );
   
-  boost::regex const r_matcher( "/.*_R\\//" );
-  
   vector< vector<string> >::iterator oit;
   for( oit = cpsr2_observations.begin(); oit != cpsr2_observations.end(); oit ++ )
@@ -220,11 +160,17 @@
     if( (*oit).size() != 0 )
     {
-      if( boost::regex_match( (*oit).front(), r_matcher ) )
+      string first_file = (*oit).front();
+      vector<string> dir_names = stringdecimate( first_file, "/" );
+      
+      string containing_dir = dir_names[dir_names.size()-3];
+      if( containing_dir.substr( containing_dir.size()-2 ) == "_R" )
       {
-        AddCalEntry( (*oit) );
+	cerr << "we have a calibration " << endl;
+        db.CheckCalibration( MakeObservationEntry( (*oit) ) );
       }
       else
       {
-        AddObservationEntry( (*oit) );
+	cerr << dir_names[dir_names.size()-3] <<  " does not end with _R" << endl;
+        db.CheckObservation( MakeObservationEntry( (*oit) ) );
       }
     }
@@ -234,4 +180,11 @@
 
 
+/// Connect to the database
+/// Mark all the observations and cals as not present
+/// iterate through all the directories given and their subdirectories
+/// for every observation we find, check that its in the database
+/// if its not in the database, add it and mark it as present
+/// if its in the database, mark it as present
+/// delete all the entries in the database that were not present.
 
 int main( int argc, char *argv[] )
@@ -255,5 +208,5 @@
     cerr << e.what() << endl;
   }
-        
+
   return 0;
 }
Index: /trunk/DBUpdater/psrdb.c
===================================================================
--- /trunk/DBUpdater/psrdb.c	(revision 422)
+++ /trunk/DBUpdater/psrdb.c	(revision 423)
@@ -3,6 +3,19 @@
 
 #include "psrdb.h"
+#include <tostring.h>
 #include <mysql++.h>
 #include <Pulsar/Archive.h>
+#include <Pulsar/ObsExtension.h>
+#include <Pulsar/Integration.h>
+#include <Pulsar/FITSHdrExtension.h>
+#include <Pulsar/FITSSUBHdrExtension.h>
+#include <Pulsar/Backend.h>
+#include <Pulsar/WidebandCorrelator.h>
+#include <Pulsar/Receiver.h>
+#include <Pulsar/Telescope.h>
+#include <dirutil.h>
+#include <sstream>
+#include <tempo++.h>
+
 
 
@@ -13,4 +26,12 @@
 using std::endl;
 using Pulsar::Archive;
+using Pulsar::ObsExtension;
+using Pulsar::FITSHdrExtension;
+using Pulsar::FITSSUBHdrExtension;
+using Pulsar::Backend;
+using Pulsar::WidebandCorrelator;
+using Pulsar::Receiver;
+using Pulsar::Telescope;
+using std::ostringstream;
 
 
@@ -25,8 +46,8 @@
 
 
-/**
- * Connect to the pulsararchive database using the parameters given,
- * If we failed to connect for any reason, simply throw an exception.
- **/
+
+/// Connect to the pulsararchive database using the parameters given,
+/// If we failed to connect for any reason, simply throw an exception.
+
 
 void psrdb::Connect( const char *db_name,
@@ -41,10 +62,9 @@
 
 
-/**
- * Mark all the entries in the observations and cals tables as being
- * asbent. As we update entries we will mark the as present, and we
- * will know any that don't get marked as present can be deleted from
- * the database.
- **/
+
+/// Mark all the entries in the observations and cals tables as being
+/// asbent. As we update entries we will mark the as present, and we
+/// will know any that don't get marked as present can be deleted from
+/// the database.
 
 void psrdb::MarkObservations( void )
@@ -62,7 +82,6 @@
 
 
-/**
- * Purge any records in the observations table that haven't been marked as present.
- **/
+
+/// Purge any records in the observations table that haven't been marked as present.
 
 void psrdb::PurgeMissingObservations()
@@ -76,7 +95,6 @@
 
 
-/**
- * Purge any records in the cals table that haven't been marked as present
- **/
+
+/// Purge any records in the cals table that haven't been marked as present
 
 void psrdb::PurgeMissingCals()
@@ -90,21 +108,154 @@
 
 
+/// Calculate the galactic latitude and longitude
+
+void psrdb::CalculateGalactic( double rajd, double decjd, double &gl, double &gb )
+{
+    double pi=asin(1) * 2.;
+    double pion180 = pi/180.;
+    double NGP_RA = 192.859508 * pion180;
+    double NGP_DEC = 27.128336 * pion180;
+    double ASC_NODE = 32.932;
+    
+    double draR = rajd*pion180 - NGP_RA;
+    double decR = decjd*pion180;
+    double sinb = cos(decR) * cos(NGP_DEC) * cos(draR) + sin(decR) * sin(NGP_DEC);
+    gb = asin(sinb);/* # this is the latitude, but in radians.*/
+    
+    double sinl = (sin(decR) * cos(NGP_DEC) - cos(decR) * cos(draR) * sin(NGP_DEC)) / cos(gb);
+    double cosl = cos(decR) * sin(draR) / cos(gb);
+    
+//     # Need to get the correct quadrant, as this isn't preserved by
+//     # atan, which returns angle between -90 and 90.
+    double gl_raw = atan(sinl/cosl);
+   
+    if(sinl > 0){
+	if(cosl > 0 ){ gl = gl_raw; }
+	else { gl = gl_raw + pi; }
+    }
+    else {
+	if(cosl > 0){ gl = gl_raw + 2.*pi; }
+	else{ gl = gl_raw + pi; }
+    }
+/*    # Now put them into degrees.*/
+    gb = gb / pion180;
+    gl = (gl / pion180) + ASC_NODE;
+}
+
+/// Read the archive parameters from the archive and put them in the ObservationDetails object.
+
 void psrdb::GetArchiveDetails( ObservationDetails &details, Reference::To<Archive> arch )
 {
+  Reference::To<ObsExtension> obs_ext = arch->get<ObsExtension>();
+  Reference::To<FITSHdrExtension> fits_ext = arch->get<FITSHdrExtension>();
+  Reference::To<FITSSUBHdrExtension> fits_sub_ext = arch->get<FITSSUBHdrExtension>();
+  Reference::To<Backend> backend = arch->get<Backend>();
+  Reference::To<WidebandCorrelator> wbcorr = arch->get<WidebandCorrelator>();
+  Reference::To<Receiver> receiver = arch->get<Receiver>();
+  Reference::To<Telescope> telescope = arch->get<Telescope>();
+  
+  details.src_name = arch->get_source();
+  if( obs_ext )
+  {
+    details.project_id = obs_ext->get_project_ID();
+  }
+  else
+  {
+    details.project_id = "";
+  }
+  details.raj = arch->get_coordinates().ra().getHMS();
+  details.decj = arch->get_coordinates().dec().getDMS();
+  if( backend )
+  {
+    details.data_type = backend->get_name();
+  }
+  else if( wbcorr )
+  {
+    details.data_type = "WBC";
+  }
+  details.obsfreq = arch->get_centre_frequency();
+  details.bw = arch->get_bandwidth();
+  details.scanlen = arch->integration_length();
+
+  MJD stt = arch->start_time();
+  details.MJD = fromstring<double>( tostring( stt, 15 ) );
+  char date_string[30];
+  stt.datestr( date_string, 30, "%Y-%m-%d" );
+  details.date = date_string;
+
+  details.rajd = arch->get_coordinates().ra().getDegrees();
+  details.decjd = arch->get_coordinates().dec().getDegrees();
+  CalculateGalactic( details.rajd, details.decjd, details.gl, details.gb );
+
+  if( !fits_ext )
+  {
+    details.bmaj = 0.0;
+    details.bmin = 0.0;
+    details.bpa = 0.0;
+    details.ut = "UNDEF";
+  }
+  else
+  {
+    details.bmaj = fits_ext->get_bmaj();
+    details.bmin = fits_ext->get_bmin();
+    details.bpa = fits_ext->get_bpa();
+    details.ut = fits_ext->get_stt_time();
+    details.hdrver = fits_ext->get_hdrver();
+  }
+
+  details.dm = arch->get_dispersion_measure();
+  details.period = arch->get_Integration(0)->get_folding_period();
   details.nbin = arch->get_nbin();
   details.npol = arch->get_npol();
   details.nchan = arch->get_nchan();
   details.nsub = arch->get_nsubint();
-  details.obsfreq = arch->get_centre_frequency();
-  details.bw = arch->get_bandwidth();
-}
-
-/**
- * This function retreives the details of an observation given the data_loc/filename/prefix
- **/
+  if( fits_sub_ext )
+  {
+    details.tsamp = fits_sub_ext->get_tsamp();
+    details.nbits = fits_sub_ext->get_nbits();
+  }
+  else
+  {
+    details.tsamp = 0.0;
+    details.nbits = 0;
+  }
+  if( wbcorr )
+  {
+    details.cnfg = wbcorr->get_config();
+    details.nbeam = wbcorr->get_nrcvr();
+  }
+//   details.inst = ;
+  if( receiver )
+    details.rcvr = receiver->get_name();
+
+//   details.survery = ;
+  
+  if( obs_ext )
+    details.telescope = obs_ext->get_telescope();
+  else if( telescope )
+    details.telescope = telescope->get_name();
+  else
+    details.telescope = "unknown";
+  
+  // if we didn't get a telescope, or get_name returne "unknown" then assume PARKES
+  if( details.telescope == "unknown" )
+    details.telescope = "PARKES";
+
+  ostringstream s;
+  s << Tempo::code(arch->get_telescope());
+  details.site = s.str();
+
+  if( obs_ext )
+    details.observer = obs_ext->get_observer();
+}
+
+
+/// This function retreives the details of an observation given the data_loc/filename/prefix
+/// It opens the file (or first file if multiple files) and calls GetArchiveDetails to read
+/// the actual details.
 
 void psrdb::GetObsDetails( const ObservationEntry &entry, ObservationDetails &details )
 {
-  if( entry.prefix == '\0' )
+  if( entry.prefix == "\\0" )
   {
     string archive_path = entry.data_loc + entry.filename;
@@ -114,4 +265,5 @@
     
     GetArchiveDetails( details, arch );
+    details.file_size_bytes = filesize( archive_path.c_str() );
   }
   else
@@ -126,77 +278,96 @@
     // the number of files. ie always 1 subint per file.
     details.nsub = entry.files.size();
-  }
-}
-
-
-
-/**
- * This function checks that an observation already exists in the database, if
- * not it will create an entry for it.
- **/
-
-void psrdb::CheckObservation( const ObservationEntry &entry )
-{
-	try
-	{
-  Query q = db.query();
-  
-  cerr << "checking that " << entry.filename << " exists" << endl;
-  
-  q << " SELECT file_id ";
-  q << " FROM observations ";
-  q << " WHERE data_loc = \"" << entry.data_loc << "\"";
-  q << " AND filename = \"" << entry.filename << "\"";
-  q << " AND prefix = '\\0'";
-  q << " AND deleted = 0 ";
-
-  cerr << "executing " << q.preview() << endl;
-
-  
-  Result res = q.store();
-  
-  cerr << "query executed" << endl;
-
-  if( res.num_rows() != 0 )
-  {
-    Row row = res.fetch_row();
-    string file_id = row["file_id"].get_string();
-
-    Query present_query = db.query();
-    present_query << " UPDATE observations SET present = 1 WHERE file_id = " << file_id;
-    cerr << "executing query " << present_query.preview() << endl;
-    present_query.execute();
-  }
-  else
-  {
-    ObservationDetails details;
-    GetObsDetails( entry, details );
-    
-    Query insert_query = db.query();
-    insert_query << " INSERT into observations(data_loc,filename,prefix,obsfreq,bw,nbin,nchan,npol,nsub,present) ";
-    insert_query << " VALUES( \"" << entry.data_loc << "\", ";
-    insert_query << " \"" << entry.filename << "\", ";
-    if( entry.prefix != '\0' )
-    	insert_query << " '" << entry.prefix << "', ";
+    
+    unsigned long each_file_size = filesize( entry.files.front().c_str() );
+    details.file_size_bytes = each_file_size * entry.files.size();
+  }
+}
+
+
+/// This function takes an ObservationEntry that is either for an observation
+/// or calibration and checks to see if an entry already exists in the database
+/// for it. If not, it will create a new entry.
+
+void psrdb::CheckObsCals( const ObservationEntry &entry, string table_name )
+{
+  try
+  {
+    Query q = db.query();
+  
+    q << " SELECT file_id ";
+    q << " FROM " << table_name << " ";
+    q << " WHERE data_loc = \"" << entry.data_loc << "\"";
+    q << " AND filename = \"" << entry.filename << "\"";
+    q << " AND prefix = '" << entry.prefix << "' ";
+    q << " AND deleted = 0 ";
+
+    Result res = q.store();
+  
+    if( res.num_rows() != 0 )
+    {
+      Row row = res.fetch_row();
+      string file_id = row["file_id"].get_string();
+
+      Query present_query = db.query();
+      present_query << " UPDATE " << table_name << " SET present = 1 WHERE file_id = " << file_id;
+      present_query.execute();
+    }
     else
-        insert_query << " '\\0', ";
-    insert_query << details.obsfreq << ", ";
-    insert_query << details.bw << ", ";
-    insert_query << details.nbin << ", ";
-    insert_query << details.nchan << ", ";
-    insert_query << details.npol << ", ";
-    insert_query << details.nsub << ", ";
-    insert_query << " 1 )"; // present = 1
-    
-	cerr << "executing query " << insert_query.preview() << endl;
-
-    insert_query.execute();
-    
-    cerr << entry.data_loc << "/" << entry.filename << " not in the database " << endl;
-  }
-  }catch( Error e )
-  {
-  	cerr << "received an exception from psrchive on file " << entry.data_loc << entry.filename << endl;
-  }
-}
-
+    {
+      ObservationDetails details;
+      GetObsDetails( entry, details );
+    
+      Query insert_query = db.query();
+      insert_query << " INSERT into " << table_name << "(data_loc,filename,prefix,obsfreq,bw,scanlen,nbin,nchan,npol,nsub,raj,decj,rajd,decjd,mjd,bmaj,bmin,bpa,dm,period,tsamp,nbits,nbeam,cnfg,rcvr,hdrver,survey,telescope,site,obsrvr,date,ut,src_name,project_id,data_type,file_size_bytes,gl,gb,present) ";
+      insert_query << " VALUES( \"" << entry.data_loc << "\", ";
+      insert_query << " \"" << entry.filename << "\", ";
+      insert_query << " '" << entry.prefix << "', ";
+      insert_query << details.obsfreq << ", ";
+      insert_query << details.bw << ", ";
+      insert_query << details.scanlen << ", ";
+      insert_query << details.nbin << ", ";
+      insert_query << details.nchan << ", ";
+      insert_query << details.npol << ", ";
+      insert_query << details.nsub << ", ";
+      insert_query << "'" << details.raj << "', ";
+      insert_query << "'" << details.decj << "', ";
+      insert_query << details.rajd << ", ";
+      insert_query << details.decjd << ", ";
+      insert_query << details.MJD << ", ";
+      insert_query << details.bmaj << ", ";
+      insert_query << details.bmin << ", ";
+      insert_query << details.bpa << ", ";
+      insert_query << details.dm << ", ";
+      insert_query << details.period << ", ";
+      insert_query << details.tsamp << ", ";
+      insert_query << details.nbits << ", ";
+      insert_query << details.nbeam << ", ";
+      insert_query << "'" << details.cnfg << "', ";
+      insert_query << "'" << details.rcvr << "', ";
+      insert_query << "'" << details.hdrver << "', ";
+      insert_query << "'" << details.survey << "', ";
+      insert_query << "'" << details.telescope << "', ";
+      insert_query << "'" << details.site << "', ";
+      insert_query << "'" << details.observer << "', ";
+      insert_query << "'" << details.date << "', ";
+      insert_query << "'" << details.ut << "', ";
+      insert_query << "'" << details.src_name << "', ";
+      insert_query << "'" << details.project_id << "', ";
+      insert_query << "'" << details.data_type << "', ";
+      insert_query << details.file_size_bytes << ", ";
+      insert_query << details.gl << ", ";
+      insert_query << details.gb << ", ";
+      insert_query << " 1 )"; // present = 1
+    
+      cerr << "insert query is " << insert_query.preview() << endl << endl;
+      
+      insert_query.execute();
+    }
+  }
+  catch( Error e )
+  {
+    cerr << "received an exception from psrchive on file " << entry.data_loc << entry.filename << endl;
+  }
+}
+
+
Index: /trunk/DBUpdater/psrdb.h
===================================================================
--- /trunk/DBUpdater/psrdb.h	(revision 422)
+++ /trunk/DBUpdater/psrdb.h	(revision 423)
@@ -23,5 +23,5 @@
   string filename;
   string data_loc;
-  char prefix;
+  string prefix;
   vector<string> files;
 };
@@ -29,10 +29,40 @@
 struct ObservationDetails
 {
+  string src_name;
+  string project_id;
+  string raj;
+  string decj;
+  string data_type;
   double obsfreq;
   double bw;
+  double scanlen;
+  string date;
+  string ut;
+  double MJD;
+  double rajd;
+  double decjd;
+  double gl;
+  double gb;
+  float bmaj;
+  float bmin;
+  float bpa;
+  double dm;
+  double period;
+  int nchan;
+  int npol;
   int nbin;
-  int npol;
-  int nchan;
   int nsub;
+  float tsamp;
+  int nbits;
+  int nbeam;
+  string cnfg;
+//   string inst;
+  string rcvr;
+  string hdrver;
+  string survey;
+  string telescope;
+  string site;
+  string observer;
+  long file_size_bytes;
 };
 
@@ -76,7 +106,11 @@
     void GetObsDetails( const ObservationEntry &entry, ObservationDetails &details );
     
-    void CheckObservation( const ObservationEntry &details );
+    void CheckObservation( const ObservationEntry &entry ) { CheckObsCals( entry, "observations" ); }
+    void CheckCalibration( const ObservationEntry &entry ) { CheckObsCals( entry, "cals" ); }
+  private:
+    void CheckObsCals( const ObservationEntry &entry, string table_name );
     
-  private:
+    void CalculateGalactic( double rajd, double decjd, double &gl, double &gb );
+    
     Connection db;
 };
