Index: /trunk/DBUpdater/DBUpdater.c
===================================================================
--- /trunk/DBUpdater/DBUpdater.c	(revision 420)
+++ /trunk/DBUpdater/DBUpdater.c	(revision 420)
@@ -0,0 +1,260 @@
+/**
+ * Written by Dvaid Smith
+ * 
+ * This program scans the directories given on the command line for any pulsar observations
+ * For every observation it finds, it checks that it exists in the database
+ * If an observation exists on disk but not in the database, it is added
+ * If an observation is in the database but not on disk, then we delete it
+ * 
+ * An observation can be in many forms
+ * 1) A single PSRFITS archive ending in rf or cf
+ * 2) A set of files in a CPSR2 directory ending with cfb and beginning with either m, n or o ( 1 file for each sub integration)
+ * 3) If an observation is in a directory that ends with _R then it is a calibration
+ * 4) If the fits file has a pulsar src_name of CAL then it is also a calibration file (check with dick about this)
+ **/
+
+
+#include <iostream>
+#include <vector>
+#include <string>
+#include <dirutil.h>
+#include <strutil.h>
+#include <boost/regex.hpp>
+#include "psrdb.h"
+
+
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::vector;
+using std::string;
+using mysqlpp::Connection;
+using mysqlpp::Query;
+using mysqlpp::Exception;
+
+
+const char *database = "psrchive";
+const char *db_username = "psrdba";
+const char *db_password = "lighthouse";
+const char *db_hostname = "localhost";
+
+psrdb db;
+
+
+
+
+/**
+ * Take the files given to be a calibration and add an entry to the database
+ * if none exists yet, otherwise mark the calibartion entry as present.
+ **/
+
+void AddCalEntry( const vector<string> &files )
+{
+  /**
+   * if we have a single file
+   *   search the cals table for that file
+   *   if we have a match
+   *     mark it as present
+   *   otherwise
+   *     create a new entry in the database for that file
+   * otherwise
+   *   determine the band prefix from the first filename
+   *   search the cals table for a matching directory/band prefix
+   *   if we have a match
+   *     mark it as present
+   *   otherwise
+   *     create a new entry ni the database for that file
+   **/
+  
+  cerr << "Adding a cal entry for " << files.size() << " files under " << pathname(files.front()) << endl;
+}
+
+void AddCalEntry( const string &single_file )
+{
+  vector<string> c;
+  c.push_back( single_file );
+  AddCalEntry( c );
+}
+
+
+
+/**
+ * Take the files given to be an observation and add an entry in the database
+ * if none exists yet, otherwise mark the observation entry as present
+ **/
+
+void AddObservationEntry( const vector<string> &files )
+{
+  cerr << "Adding a observation entry for " << files.size() << " files under " << files.front() << endl;
+  
+  ObservationEntry entry;
+  
+  string example_datafile = basename( files.front() );
+  
+  string full_path = pathname( files.front() );
+  string stripped_path = full_path.substr( 0, full_path.size() - 1 );
+  
+  string new_data_loc = pathname( stripped_path );
+  string new_filename = basename( stripped_path );
+  
+  entry.filename = new_filename;
+  entry.data_loc = new_data_loc;
+  entry.prefix = example_datafile[0];
+  entry.files = files;
+  
+  db.CheckObservation( entry );
+}
+
+void AddObservationEntry( const string &single_file )
+{
+  ObservationEntry entry;
+  
+  entry.filename = basename( single_file );
+  entry.data_loc = pathname( single_file );
+  entry.prefix = '\0';
+
+  db.CheckObservation( entry );
+}
+
+
+
+/**
+ * This function iteratively examines a directory and calls itself to examine
+ * any subdirectories
+ **/
+
+void ExamineDirectory( const string path )
+{
+  vector<string> rf_files;
+  vector<string> cf_files;
+  vector<string> m_fb_files;
+  vector<string> n_fb_files;
+  vector<string> m_cfb_files;
+  vector<string> n_cfb_files;
+  vector<string> r_cfb_files;
+  
+  boost::regex const rf_matcher(".*\\.rf");
+  boost::regex const cf_matcher(".*\\.cf");
+  boost::regex const m_fb_matcher("m.*\\.fb");
+  boost::regex const n_fb_matcher("n.*\\.fb");
+  boost::regex const m_cfb_matcher("m.*\\.cfb");
+  boost::regex const n_cfb_matcher("n.*\\.cfb");
+  boost::regex const r_cfb_matcher("r.*\\.cfb");
+
+  vector<string> contents;
+  dirglob( &contents, path + string( "/*" ) );
+  vector<string>::iterator it;
+  for( it = contents.begin(); it != contents.end(); it ++ )
+  {
+    string filename = basename( (*it) );
+    if( file_is_directory( (*it).c_str() ) )
+    {
+      if( filename.find("_S") != string::npos )
+        cerr << "ignoring scrunch directory " << (*it) << endl;
+      else
+        ExamineDirectory( (*it) );
+    }
+    else
+    {
+      if( boost::regex_match( filename, rf_matcher ) )
+        rf_files.push_back( (*it) );
+      else if( boost::regex_match( filename, cf_matcher ) )
+        cf_files.push_back( (*it) );
+      else if( boost::regex_match( filename, m_fb_matcher ) )
+        m_fb_files.push_back( (*it) );
+      else if( boost::regex_match( filename, n_fb_matcher ) )
+        n_fb_files.push_back( (*it) );
+      else if( boost::regex_match( filename, m_cfb_matcher ) )
+        m_cfb_files.push_back( (*it) );
+      else if( boost::regex_match( filename, n_cfb_matcher ) )
+        n_cfb_files.push_back( (*it) );
+      else if( boost::regex_match( filename, r_cfb_matcher ) )
+        r_cfb_files.push_back( (*it) );
+//       else
+//         cerr << "don't know how to handle file " << filename << endl;
+    }
+  }
+//   cerr << "Found " << rf_files.size() << " rf files " << endl;
+//   cerr << "Found " << cf_files.size() << " cf files " << endl;
+//   cerr << "Found " << m_fb_files.size() << " m*.fb files " << endl;
+//   cerr << "Found " << n_fb_files.size() << " n*.fb files " << endl;
+//   cerr << "Found " << m_cfb_files.size() << " m*.cfb files " << endl;
+//   cerr << "Found " << n_cfb_files.size() << " n*.cfb files " << endl;
+//   cerr << "Found " << r_cfb_files.size() << " r*.cfb files " << endl;
+  
+  /**
+   *   for each cf file
+   *     add this file as a calibration
+   *   for each rf file
+   *     if this directory ends in _R
+   *       add this file as a calibration
+   *     otherwise
+   *       add this file as an observation
+   * for each of the vectors n*.cfb m*.cfb r*.cfb m*.fb n*.fb
+   *   if we have any m*.cfb files
+   *     if this directory ends in _R
+   *       add these files as a calibration
+   *     otherwise
+   *       add these files as an observation
+   **/
+  
+  for( it = cf_files.begin(); it != cf_files.end(); it ++ )
+    AddCalEntry( (*it) );
+  
+  for( it = rf_files.begin(); it != rf_files.end(); it ++ )
+    AddObservationEntry( (*it) );
+  
+  vector< vector<string> > cpsr2_observations;
+  cpsr2_observations.push_back( m_cfb_files );
+  cpsr2_observations.push_back( n_cfb_files );
+  cpsr2_observations.push_back( r_cfb_files );
+  cpsr2_observations.push_back( m_fb_files );
+  cpsr2_observations.push_back( n_fb_files );
+  
+  boost::regex const r_matcher( "/.*_R\\//" );
+  
+  vector< vector<string> >::iterator oit;
+  for( oit = cpsr2_observations.begin(); oit != cpsr2_observations.end(); oit ++ )
+  {
+    if( (*oit).size() != 0 )
+    {
+      if( boost::regex_match( (*oit).front(), r_matcher ) )
+      {
+        AddCalEntry( (*oit) );
+      }
+      else
+      {
+        AddObservationEntry( (*oit) );
+      }
+    }
+  }
+  
+}
+
+
+
+int main( int argc, char *argv[] )
+{
+  try
+  {
+    db.Connect( database, db_hostname, db_username, db_password );
+    
+    db.MarkObservations();
+    
+    vector<string> target_directories;
+    for( int i = 1; i < argc; i ++ )
+      ExamineDirectory( string( argv[i] ) );
+
+    db.PurgeMissingObservations();
+    db.PurgeMissingCals();
+  }
+  catch( Exception e )
+  {
+    cerr << "The following exception was caught: ";
+    cerr << e.what() << endl;
+  }
+        
+  return 0;
+}
+
Index: /trunk/DBUpdater/Makefile
===================================================================
--- /trunk/DBUpdater/Makefile	(revision 420)
+++ /trunk/DBUpdater/Makefile	(revision 420)
@@ -0,0 +1,20 @@
+
+PSRHOME = /usr/local/work/psrhome/linux/
+
+
+CC = g++
+LL = g++
+CCFLAGS = -I${PSRHOME}/include/ -I/usr/include/mysql++/ -I/usr/include/mysql/
+LLFLAGS = -L${PSRHOME}/lib/
+LDLIBS = -lpsrutil -lpsrbase -lpsrutil -lfftw3f -lcfitsio -lmysqlclient -lmysqlpp -lboost_regex -lfrtbegin -lg2c -lm
+
+all : DBUpdater
+
+DBUpdater : DBUpdater.o psrdb.o
+	$(LL) $(LLFLAGS) -o DBUpdater DBUpdater.o psrdb.o $(LDLIBS)
+
+
+.c.o :
+	$(CC) $(CCFLAGS) -c $<
+
+psrdb.o : psrdb.c psrdb.h
Index: /trunk/DBUpdater/psrdb.c
===================================================================
--- /trunk/DBUpdater/psrdb.c	(revision 420)
+++ /trunk/DBUpdater/psrdb.c	(revision 420)
@@ -0,0 +1,177 @@
+
+
+
+#include "psrdb.h"
+#include <mysql++.h>
+#include <Pulsar/Archive.h>
+
+
+using mysqlpp::Query;
+using mysqlpp::Result;
+using mysqlpp::Row;
+using std::cerr;
+using std::endl;
+using Pulsar::Archive;
+
+
+
+
+
+/// Constructor
+
+psrdb::psrdb()
+{
+}
+
+
+/**
+ * Connect to the pulsararchive database using the parameters given,
+ * If we failed to connect for any reason, simply throw an exception.
+ **/
+
+void psrdb::Connect( const char *db_name,
+                     const char *db_hostname,
+                     const char *db_username,
+                     const char *db_password )
+{
+  if( !db.connect( db_name, db_hostname, db_username, db_password ) )
+    throw( FailedConnection() );
+}
+
+
+
+/**
+ * Mark all the entries in the observations and cals tables as being
+ * asbent. As we update entries we will mark the as present, and we
+ * will know any that don't get marked as present can be deleted from
+ * the database.
+ **/
+
+void psrdb::MarkObservations( void )
+{
+  Query obs_query = db.query();
+  
+  obs_query << "UPDATE observations SET present = 0 WHERE deleted = 0";
+  obs_query.execute();
+  
+  Query cals_query = db.query();
+  cals_query << "UPDATE cals SET present = 0 WHERE deleted = 0";
+  cals_query.execute();
+}
+
+
+
+/**
+ * Purge any records in the observations table that haven't been marked as present.
+ **/
+
+void psrdb::PurgeMissingObservations()
+{
+  Query purge_query = db.query();
+  
+  purge_query << "UPDATE observations SET deleted = 1 WHERE present = 0 AND deleted = 0";
+  
+  purge_query.execute();
+}
+
+
+/**
+ * Purge any records in the cals table that haven't been marked as present
+ **/
+
+void psrdb::PurgeMissingCals()
+{
+  Query purge_query = db.query();
+  
+  purge_query << "UPDATE cals SET deleted = 1 WHERE present = 0 AND deleted = 0";
+  
+  purge_query.execute();
+}
+
+
+void psrdb::GetArchiveDetails( ObservationDetails &details, Reference::To<Archive> arch )
+{
+  int nbin = arch->get_nbin();
+  int npol = arch->get_npol();
+  int nchan = arch->get_nchan();
+  
+  details.obsfreq = arch->get_centre_frequency();
+  details.bw = arch->get_bandwidth();
+}
+
+/**
+ * This function retreives the details of an observation given the data_loc/filename/prefix
+ **/
+
+void psrdb::GetObsDetails( const ObservationEntry &entry, ObservationDetails &details )
+{
+  if( entry.prefix == '\0' )
+  {
+    string archive_path = entry.data_loc + entry.filename;
+    Reference::To<Archive> arch = Archive::load( archive_path );
+    if( !arch )
+      throw( GenericException( (string("Failed to load archive") + archive_path).c_str() ) );
+    
+    GetArchiveDetails( details, arch );
+  }
+  else
+  {
+    Reference::To<Archive> first_arch = Archive::load( entry.files.front() );
+    if( !first_arch )
+      throw( GenericException( (string("Failed to load archive") + entry.files.front()).c_str() ) );
+
+    GetArchiveDetails( details, first_arch );
+  }
+}
+
+
+
+/**
+ * This function checks that an observation already exists in the database, if
+ * not it will create an entry for it.
+ **/
+
+void psrdb::CheckObservation( const ObservationEntry &entry )
+{
+  Query q = db.query();
+  
+  cerr << "checking that " << entry.filename << " exists" << endl;
+  
+  q << " SELECT file_id ";
+  q << " FROM observations ";
+  q << " WHERE data_loc = \"" << entry.data_loc << "\"";
+  q << " AND filename = \"" << entry.filename << "\"";
+  q << " AND prefix = \"" << entry.prefix << "\"";
+  q << " AND deleted = 0 ";
+  
+  Result res = q.store();
+  
+  if( res.num_rows() != 0 )
+  {
+    Row row = res.fetch_row();
+    string file_id = row["file_id"].get_string();
+
+    Query present_query = db.query();
+    present_query << " UPDATE observations SET present = 1 WHERE file_id = " << file_id;
+    present_query.execute();
+  }
+  else
+  {
+    ObservationDetails details;
+    GetObsDetails( entry, details );
+    
+    Query insert_query = db.query();
+    insert_query << " INSERT into observations(data_loc,filename,prefix,obsfreq,bw,present) ";
+    insert_query << " VALUES( \"" << entry.data_loc << "\", ";
+    insert_query << " \"" << entry.filename << "\", ";
+    insert_query << " '" << entry.prefix << "', ";
+    insert_query << details.obsfreq << ", ";
+    insert_query << details.bw << ", ";
+    insert_query << " 1 )"; // present = 1
+    
+    insert_query.execute();
+    
+    cerr << entry.data_loc << "/" << entry.filename << " not in the database " << endl;
+  }
+}
+
Index: /trunk/DBUpdater/psrdb.h
===================================================================
--- /trunk/DBUpdater/psrdb.h	(revision 420)
+++ /trunk/DBUpdater/psrdb.h	(revision 420)
@@ -0,0 +1,86 @@
+/**
+ * This file represents the pulsararchive database in mysql++
+ **/
+
+
+#include <mysql++.h>
+#include <Pulsar/Archive.h>
+
+
+
+using mysqlpp::Exception;
+using mysqlpp::Connection;
+using mysqlpp::sql_int;
+using mysqlpp::sql_bigint;
+using mysqlpp::sql_char;
+using mysqlpp::sql_double;
+using mysqlpp::sql_float;
+using std::string;
+using std::vector;
+using Pulsar::Archive;
+
+
+
+
+
+struct ObservationEntry
+{
+  string filename;
+  string data_loc;
+  char prefix;
+  vector<string> files;
+};
+
+struct ObservationDetails
+{
+  double obsfreq;
+  double bw;
+};
+
+
+
+
+/**
+ * Exceptions the database might throw
+ **/
+
+class MYSQLPP_EXPORT FailedConnection : public Exception
+{
+  public:
+    explicit FailedConnection(const char* w = "Unable to connect to database") : Exception(w) { }
+};
+
+class MYSQLPP_EXPORT GenericException : public Exception
+{
+  public:
+    explicit GenericException( const char *w = "Unknown Expception" ) : Exception(w) { }
+};
+
+
+
+
+
+class psrdb
+{
+  public:
+    psrdb();
+    
+    void Connect( const char *db_name, const char *db_hostname, const char *db_username, const char *db_password );
+    
+    void MarkObservations();
+    
+    void GetArchiveDetails( ObservationDetails &details, Reference::To<Archive> arch );
+    
+    void PurgeMissingObservations();
+    void PurgeMissingCals();
+    
+    void GetObsDetails( const ObservationEntry &entry, ObservationDetails &details );
+    
+    void CheckObservation( const ObservationEntry &details );
+    
+  private:
+    Connection db;
+};
+
+
+
