#! /usr/bin/perl -w

# find_archives.pl
# Author: Albert Teoh
# Date: December 2005
# ATNF, CSIRO

# Crawls through data archive directories (given as an input parameter),
# searches for all observation and/or calibration files, enquires about
# their pulsar parameters and inputs these details as well as their
# location into a database


use strict;
use File::Find ();
#use DBI qw(:sql_types);
use POSIX;

# Set the variable $File::Find::dont_use_nlink if you're using AFS,
# since AFS cheats.

# for the convenience of &wanted calls, including -eval statements:
use vars qw/*name *dir *prune @archive_dirs $archive_dir @files
    $vap_cmd $vap_out $params $num_vap_params $db_handle
    $filename $raj $decj $dm $period $bw $cnfg 
    $freq $inst $mjdint $mjdfrac $npol $nchan $nbin $nsub
    $rcvr $site $length $obsrvr $data_loc $data_type
    $file_size $MJD $raH $raM $raS $decD $decM $decS $rajd $decjd $decstring
    $pi $pion180 $NGP_RA $NGP_DEC $ASC_NODE $draR $decR 
    $sinb $sinl $cosl $gb $gl $gl_raw 
    $bmaj $bmin $bpa $hdrver $survey $nbeam
    $output $vap $archive_extensions $ext	
    $default
    @result_params $FILE_SIZE_INDEX $FB $DFB $WBC $CPSR2 $NULLFIELD
    $params_NAME    
    $params_PROJID  
    $params_RAJ     
    $params_DECJ    
    $params_FREQ    
    $params_BW      
    $params_LENGTH  
    $params_DATE    
    $params_TIME    
    $params_MJD
    $params_BMAJ    
    $params_BMIN    
    $params_BPA     
    $params_DM      
    $params_PERIOD  
    $params_NCHAN   
    $params_NPOL    
    $params_NBIN    
    $params_NSUB    
    $params_NBITS   
    $params_TSAMP   
    $params_NBEAM
    $params_CNFG    
    $params_INST    
    $params_RCVR    
    $params_HDRVER
    $params_TELESCOP
    $params_SITE    
    $params_OBSRVR  
    /;  # $machine
						
*name   = *File::Find::name;
*dir    = *File::Find::dir;
*prune  = *File::Find::prune;

# The data types
$FB = "FB";
$WBC = "WBC";
$DFB = "DFB";
$CPSR2 = "CPSR2";

# How to write NULL fields to the database
$NULLFIELD = "NULL";

# Pulsar parameters (listed in "vap -H")
#$params  = "name raj decj dm period bw cnfg freq inst intmjd fracmjd npol nchan nbin nsub rcvr site length obsrvr";
#$params  = "name projid ra dec freq bw length stt_date stt_time intmjd fracmjd bmaj bmin bpa dm period nchan npol nbin nsub nbits tsamp nbeam cnfg backend rcvr telescop site obsrvr";
$params  = "name projid ra dec freq bw length stt_date stt_time mjd bmaj bmin bpa dm period nchan npol nbin nsub nbits tsamp nbeam beconfig backend rcvr hdrver telescop asite observer";

# vap result parameter indices. Used to locate the resulting 
# parameter's location
$params_NAME    = 0;
$params_PROJID  = 1;
$params_RAJ     = 2;
$params_DECJ    = 3;
$params_FREQ    = 4;
$params_BW      = 5;
$params_LENGTH  = 6;
$params_DATE    = 7;
$params_TIME    = 8;
$params_MJD     = 9;
$params_BMAJ    = 10;
$params_BMIN    = 11;
$params_BPA     = 12;
$params_DM      = 13;
$params_PERIOD  = 14;
$params_NCHAN   = 15;
$params_NPOL    = 16;
$params_NBIN    = 17;
$params_NSUB    = 18;
$params_NBITS   = 19;
$params_TSAMP   = 20;
$params_NBEAM   = 21;
$params_CNFG    = 22;
$params_INST    = 23;
$params_RCVR    = 24;
$params_HDRVER  = 25;
$params_TELESCOP= 26;
$params_SITE    = 27;
$params_OBSRVR  = 28;

# filename extensions
#$archive_extensions = "ar rf cf cfb fb";
$archive_extensions = "rf cf cfb fb";

# The machine to do all the processing on
#$machine = "tycho"; 

# The location of vap
$vap = "/pulsar/psr/linux/bin/vap";

#####################
# Begin main method
#
my @vap_params = split(/ /, $params);
$num_vap_params = scalar(@vap_params);

# The column number (starting from 0) of the file 
# size when calling ls -l
$FILE_SIZE_INDEX = 4;

# Use the default operation which is to firstly delete any
# stale records, then crawl through and insert new records
$default = 1;

print "ARGV = @ARGV\n";

if (scalar(@ARGV) == 0) {
	print "find_archives.pl: A crawler that populates a relational database\n
 				with indexes to observational data along with their respective\n
				cal files.\n\n
				Usage: find_archives.pl [options] directory1 [directory2 ...]\n\n
				If no options are provided, it will default to do both -d and -p\n
				Options:\n
				\t-d Delete stale records\n
				\t-p Populate database with new records\n";
	
}

# Connect to the database
$db_handle = connectdb();

#########################################################
# Step 1. First remove any stale entries in the database
#         (remove file locations that no longer exist)
#########################################################

if ($ARGV[0] eq "-d") {
#	deleteStaleRecords();
	$default = 0;
	shift @ARGV;
}


#########################################################
# Step 2. Crawl through each input directory and insert
#         any new archives

if ($ARGV[0] eq "-p") {
	shift @ARGV;
	@archive_dirs = @ARGV;

	foreach $archive_dir (@archive_dirs) {

		# Traverse desired filesystems
		File::Find::find({wanted => \&wanted}, $archive_dir);
	}
	$default = 0;
}

# Else do both
if ($default) {

#	deleteStaleRecords();

	@archive_dirs = @ARGV;

	foreach $archive_dir (@archive_dirs) {
		
		# Check that the input directory is legit
		
		# Traverse desired filesystems
		File::Find::find({wanted => \&wanted}, $archive_dir);
	}

}
#if ($db_handle->disconnect()) {
#	print "Successfully Disconnected from database\n";
#}
#else {
#	print "Error: Failed to disconnect from the database\n";
#	exit;
#}

#######################################################
# Subroutine definitions
#
sub wanted {
  
	my @files;
	my @all_files;
	my @cpsr2_obs_dirs;
	my $file;
	my $cpsr2_obs_dir;
	my $search_str = "";
	my $num_dumps = 0;
	
	if (-d and /^[a-zA-Z]{0,1}[0-9]{4}[+-][0-9]{4}.*/s ) {

		print "\n\nIn directory $_\n\n";
		print "\$File::Find::name = $File::Find::name\n";
		
		foreach $ext (split (/\s+/, $archive_extensions) ) {
			$search_str .= $File::Find::name . "/*.$ext "
		}
		print "$search_str\n";

		@all_files = glob($search_str);
		
		if (scalar(@all_files) == 0) {
		
			print "This is a CPSR2 directory: $_\n";
			
			# Foreach observation
			@cpsr2_obs_dirs = glob($File::Find::name . "/????-??-??-??:??:??");
			
			foreach $cpsr2_obs_dir (@cpsr2_obs_dirs) {
				print "dir is $cpsr2_obs_dir\n";
				
				# Need to check if this entry already exists
				$cpsr2_obs_dir =~ s/\s//g;

				if (isDuplicate($cpsr2_obs_dir) == 1) {
					print "$cpsr2_obs_dir is a duplicate entry. Skipping....\n\n";
					next;
				}				
				#else {
					#print "$cpsr2_obs_dir NOT a duplicate. Inserting into database\n";
				#}
				
				
				@all_files = glob($cpsr2_obs_dir . "/m*fb " . $cpsr2_obs_dir . "/m*ar");
				
				if (scalar(@all_files) == 0) {
					next;
				}
				
				#print "first file is " . $all_files[0] . "\n";
				
				# Work out the length of each dump
				# $vap_cmd = "ssh $machine $vap -n -c \\\"$params\\\" ".$all_files[0]. " |";
				$vap_cmd = "$vap -n -c \"$params\" ".$all_files[0]. " | grep -v filename | grep -v -x \"\" |";

				print "Calling vap...$vap_cmd\n";

				open(VAP,$vap_cmd);
				
				while ($vap_out = <VAP>) {
				    
				    print "vap_out = $vap_out\n";

				    #@result_params = split(/\s+/, $vap_out);
				    @result_params = split(" ", $vap_out);
				    
				    $length = $result_params[$params_LENGTH+1];
				    $result_params[$params_BW+1] *= 2;
					
				    #print "length = $length secs\n";
				    
				    $num_dumps = `ls -1 $cpsr2_obs_dir/m*fb $cpsr2_obs_dir/m*ar | wc -l`;
				    
				    #print "num dumps = $num_dumps\n";
				    
				    $result_params[$params_LENGTH+1] = $length*$num_dumps;
				    
				    #print "total length = " . $result_params[$params_LENGTH+1] . "\n";
				    
				    #print "new bw = " . $result_params[$params_BW+1] . "\n";
				    
				    # +1 because first column is filename
				    if (scalar(@result_params) == ($num_vap_params + 1)) {
					print "@result_params\n";
					
					populate_observations($cpsr2_obs_dir, $num_dumps);
				    }
				    else {
					print "Incorrect number of cols = ".scalar(@result_params).". Expected " . ($num_vap_params+1) ."\n";
				    }
				    
				}
				print "\n";
				

				# Count the number of files
			}
		}
		else {
			foreach $file (@all_files) {

				# Need to check if this entry already exists
				$_ =~ s/\s//g;
				$file =~ s/\s//g;

				if (isDuplicate($file) == 1) {
					#print "$file is a duplicate entry. Skipping....\n\n";
					next;
				}				
				else {
					#print "$file NOT a duplicate. Inserting into database\n";
				}

			#	$vap_cmd = "ssh $machine $vap -n -c \\\"$params\\\" $file |";
				$vap_cmd = "$vap -n -c \"$params\" $file  | grep -v filename | grep -v -x \"\" |";

				print "Calling vap...$vap_cmd\n";

				open(VAP,$vap_cmd);

				while ($vap_out = <VAP>) {
					$vap_out =~ s/^\s+//g; #remove any leading space
					@result_params = split(/\s+/, $vap_out);

					# +1 because first column is filename
					if (scalar(@result_params) == ($num_vap_params + 1)) {
					    print "@result_params\n";
					    
					    populate_observations($File::Find::name);
					}
					else {
					    print "Incorrect number of cols = ".scalar(@result_params).". Expected " . $num_vap_params+1 ."\n";
					    print "@result_params\n";
					}
				    }
				print "\n";
			}
		}
		# don't traverse into this directory
		$File::Find::prune = 1;	
	}
}

sub isDuplicate {

#DEBUGGING 
    return 0;

#	my $filename = $_[0];
#	my $search_data_loc;
#	my $search_filename;
#	my $pulsar_name;
#	
#	if ( $filename =~ m/(.*(J[0-9]{4}[+-][0-9]{1,4}.*))[\/]+([a-z][0-9]{6}_[0-9]{6}\.([a-z]{2,}))$/i ) {
#		$search_data_loc = $1;
#		$search_filename = $3;
#		$pulsar_name = $2;
#	}
#	
#	elsif ($filename =~ m/(.*(J[0-9]{4}[+-][0-9]{1,4}.*))[\/]+([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}:[0-9]{2}:[0-9]{2})$/i ) {
#		$search_data_loc = $1;
#		$search_filename = $3;
#		$pulsar_name = $2;
#	}
#	else {
#		return 1;
#	}
#	
#	
#	my $sql;
#	
#	if ($pulsar_name =~ m/.*_R$/) {
#		
#		$sql = qq{ SELECT * from cals 
#							  WHERE data_loc LIKE ? AND 
#								      filename    = ?
#						};
#	}
#	else {
#		print "This is an observation\n";
#		$sql = qq{ SELECT * from observations 
#							  WHERE data_loc LIKE ? AND 
#								      filename    = ?
#						};
#	}
#	
#	my $sth = $db_handle->prepare( $sql );
#
#  eval {
#    print "like $search_data_loc and filename = $search_filename \n";
#    
#    $sth->bind_param( 1, $search_data_loc, SQL_VARCHAR );
#    $sth->bind_param( 2, $search_filename, SQL_VARCHAR );
#    $sth->execute();
#
#	};
#
#  if( $@ ) {
#    warn "Database error: $DBI::errstr\n";
#    $db_handle->rollback(); #just die if rollback is failing
#  }
#	
#	# read the records
#	my @data = $sth->fetchrow_array();
#		
#	$sth->finish();
#	
#	if (scalar(@data) > 0) {
#		return 1;
#	}
#	
#	return 0;
}

sub populate_observations {


	print "populate_obs(".$_[0].")\n";
	$data_loc = $_[0];
	my $display_data_loc = $data_loc;
	
	if ($data_loc =~ m/(.*)\/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}:[0-9]{2}:[0-9]{2}$/i) {
		print "Actual data loc is $1\n";
		$display_data_loc = $1;
	}
		
	$filename = shift(@result_params);
	
	# remove leading and trailing whitespace
	$filename =~ s/ //g;
	
	my $display_filename = $filename;
	print "display_filename = $display_filename\n";
	my $ls = `ls -l $data_loc/$filename`;
	
	my @file_details = split(/\s+/, $ls);
	
	$file_size = $file_details[$FILE_SIZE_INDEX];
	
	# Get the total filesize
	if (scalar (@_) > 1) {
		my $num_dumps = $_[1];
	
		$file_size *= $num_dumps*2; # *2 because of other band
		print "Multiplying filesize by number files = $num_dumps with total $file_size\n";
	}


	# Try to find the data type
#	my $file_is_good = true;

	if ($filename =~ /^a.*\.rf/ || $filename =~ /^a.*\.cf/ || 
	    $filename =~ /^r.*\.rf/ || $filename =~ /^r.*\.cf/) {
	    $data_type = $DFB;
	}
	elsif ($filename =~ /^w.*\.rf/ || $filename =~ /^w.*\.cf/) {
	    $data_type = $WBC;
	}
	elsif ($filename =~ /^m.*\.cfb/ || $filename =~ /^m.*\.fb/ ||
	       $filename =~ /^n.*\.cfb/ || $filename =~ /^n.*\.fb/ ||
	       $filename =~ /^o.*\.cfb/ || $filename =~ /^o.*\.fb/ ) {
	    $data_type = $CPSR2;

	    # Use the directory of observation instead of filename instance
	    my @dirs = split(/\//, $data_loc);
	    print "The new filename will be " . $dirs[$#dirs] . " from $data_loc\n";
	    $display_filename = $dirs[$#dirs];
#			# For some reason, the name of the pulsar cals for CPSR2 files are all "CAL"
#			if ($data_loc =~ /\_R/) {
#				my @jname_parts = split(/J/i, $dirs[$#dirs-1]);
#				$result_params[$params_NAME] = $jname_parts[1];
#			}
			
	}
	else {
	    $data_type = $NULLFIELD;
	}

#	# Try to find the data type
#	if ($filename =~ /\.ar$/) {
#		$data_type = $FB;
#	}
#	elsif ($filename =~ /\.rf/ || $filename =~ /\.cf/ || $filename =~ /fb/) {
#		if ($filename =~ /^a/) { # DFB file
#			$data_type = $DFB;
#		}
#		
#		elsif ($filename =~ /^w/) { # Wide Band Correlator file
#			$data_type = $WBC;
#		}
#		elsif ($filename =~ /^m/ || $filename =~ /^n/ || $filename =~ /^r/) {
#			$data_type = $CPSR2;
#			
#			# Use the directory of observation instead of filename instance
#			my @dirs = split(/\//, $data_loc);
#			
#			print "The new filename will be " . $dirs[$#dirs] . " from $data_loc\n";
#			
#			$display_filename = $dirs[$#dirs];
#			
#			# For some reason, the name of the pulsar cals for CPSR2 files are all "CAL"
#			if ($data_loc =~ /\_R/) {
#				my @jname_parts = split(/J/i, $dirs[$#dirs-1]);
#				$result_params[$params_NAME] = $jname_parts[1];
#			}
#
#		}
#		else {
#			$data_type = undef;
#		}
#	}
	print "display_filename = $display_filename\n";
	
	# Check that N/A values are set to NULL

	# Get the total MJD
#	if ( $result_params[$params_MJDINT] eq "N/A" && 
#	     $result_params[$params_MJDFRAC] eq "N/A" ) {
#		$MJD = undef;
#	}
#	else {
#		$MJD = $result_params[$params_MJDINT] + $result_params[$params_MJDFRAC];
#	}
	if($result_params[$params_MJD] eq "INVALID"){
	    $MJD = $NULLFIELD;
	}
	else{
	    $MJD = $result_params[$params_MJD];
	}
	
	print "MJD = $MJD\n";

	#### Calculate the RA and Dec in decimal degrees.

	($raH,$raM,$raS) = split(':', $result_params[$params_RAJ]);
	$rajd = ($raH + $raM/60. + $raS/3600.) * 15.;

	($decD,$decM,$decS) = split(':', $result_params[$params_DECJ]);
	$decjd = (abs($decD) + $decM/60. + $decS/3600.);
	my @decstring = split(/ */,$result_params[$params_DECJ]);
	if($decstring[0] eq '-'){
	    $decjd = -1. * $decjd; 
	}

	### Calculate the Galactic Longitude and latitude
	$pi=asin(1) * 2.;
	$pion180 = $pi/180.;
	$NGP_RA = 192.859508 * $pion180; # location of NGP
	$NGP_DEC= 27.128336 * $pion180;
	$ASC_NODE=32.932;
	
	$draR = $rajd*$pion180 - $NGP_RA;
	$decR = $decjd*$pion180;
	$sinb = cos($decR) * cos($NGP_DEC) * cos($draR) + sin($decR) * sin($NGP_DEC);
	$gb = asin($sinb); # this is the latitude, but in radians.
	
	$sinl = (sin($decR) * cos($NGP_DEC) - cos($decR) * cos($draR) * sin($NGP_DEC)) / cos($gb);
	$cosl = cos($decR) * sin($draR) / cos($gb);
	
	# Need to get the correct quadrant, as this isn't preserved by
	# atan, which returns angle between -90 and 90.
	$gl_raw = atan($sinl/$cosl);
	if($sinl > 0){
	    if($cosl > 0 ){ $gl = $gl_raw; }
	    else { $gl = $gl_raw + $pi; }
	}
	else {
	    if($cosl > 0){ $gl = $gl_raw + 2.*$pi; }
	    else{ $gl = $gl_raw + $pi; }
	}
	# Now put them into degrees.
	$gb = $gb / $pion180;
	$gl = ($gl / $pion180) + $ASC_NODE;

	print "rajd = $rajd, decjd=$decjd, gl=$gl, gb=$gb\n";


	# Beam information
	$bmaj = $result_params[$params_BMAJ];
	$bmin = $result_params[$params_BMIN];
	$bpa = $result_params[$params_BPA];
	if($bmaj eq "UNDEF" || $bmaj eq "*" || $bmaj == 0.){
	    # could not get beam info from vap output. 
	    # Need to calculate manually.
	    if( $result_params[$params_FREQ] != 0.){
		# if the frequency is zero, do not calculate these...
		$bmaj = (1.2*(299792458./($result_params[$params_FREQ] * 1.e6))/64.) / $pion180;
		$bmin = (1.2*(299792458./($result_params[$params_FREQ] * 1.e6))/64.) / $pion180;
		$bpa = 0.;
	    }
	    else{
		$bmaj = $NULLFIELD;
		$bmin = $NULLFIELD;
		$bpa = $NULLFIELD;
	    }
	}

	if($result_params[$params_NBITS] eq "UNDEF" || $result_params[$params_NBITS] == 0){
	    $result_params[$params_NBITS] = $NULLFIELD;
	}
	if($result_params[$params_TSAMP] eq "UNDEF" || $result_params[$params_TSAMP] == 0){
	    $result_params[$params_TSAMP] = $NULLFIELD;
	}


	# Dud things that aren't required for this set of data.
	$survey = $NULLFIELD;
	$nbeam = 1;
	
	my $i;	
	for ($i = 0; $i <= $#result_params; $i++) {
		if ($result_params[$i] eq "N/A" || $result_params[$i] eq "UNDEF" || $result_params[$i] eq "INVALID" || $result_params[$i] eq "*error*" || $result_params[$i] eq "*") {
			$result_params[$i] = $NULLFIELD;
		}
	}
	
	# print out the list of parameters 
	print "filename = $display_filename\n";
	print "src_name = " . $result_params[$params_NAME] . "\n";
	print "projid = " . $result_params[$params_PROJID] . "\n";
	print "raj = " . $result_params[$params_RAJ] . "\n";
	print "dec = " . $result_params[$params_DECJ] . "\n";
	print "data_type = $data_type\n";
	print "freq = " . $result_params[$params_FREQ] . "\n";
	print "bw = " . $result_params[$params_BW] . "\n";
	print "scanlen = " . $result_params[$params_LENGTH] . "\n";
	print "date = " . $result_params[$params_DATE] ."\n";
	print "ut = " . $result_params[$params_TIME] ."\n";
	print "MJD = " . $MJD ."\n";
	print "rajd = " . $rajd ."\n";
	print "decjd = " . $decjd ."\n";
	print "gl = " . $gl ."\n";
	print "gb = " . $gb ."\n";
	print "bmaj = " . $bmaj ."\n";
	print "bmin = " . $bmin ."\n";
	print "bpa = " . $bpa ."\n";
       	print "dm = " . $result_params[$params_DM] . "\n";
	print "period = " . $result_params[$params_PERIOD] . "\n";
	print "nchan = " . $result_params[$params_NCHAN] . "\n";
	print "npol = " . $result_params[$params_NPOL] . "\n";
	print "nbin = " . $result_params[$params_NBIN] . "\n";
	print "nsub = " . $result_params[$params_NSUB] . "\n";
	print "tsamp = " . $result_params[$params_TSAMP] . "\n";
	print "nbits = " . $result_params[$params_NBITS] . "\n";
#	print "nbeam = " . $result_params[$params_NBEAM] . "\n";
	print "nbeam = " . $nbeam . "\n";
	print "cnfg = " . $result_params[$params_CNFG] . "\n";
	print "inst = " . $result_params[$params_INST] . "\n";
	print "rcvr = " . $result_params[$params_RCVR] . "\n";
	print "hdrver = " . $result_params[$params_HDRVER] . "\n";
	print "survey = " . $survey . "\n";
	print "telescope = " . $result_params[$params_TELESCOP] . "\n";
	print "site = " . $result_params[$params_SITE] . "\n";
	print "obsrvr = " . $result_params[$params_OBSRVR] . "\n";
	print "data_loc = $display_data_loc\n";
	print "file_size = $file_size bytes\n";

#	my $sql;
#	
#	if ($data_loc =~ /\_R/ || $) {
#	    print "Inserting cal file $display_filename\n";
#		
#	    $sql = qq{ INSERT INTO cals 
#			   (filename, src_name, project_id,
#			    raj, decj, data_type, obsfreq, bw,
#			    scanlen, date, ut, MJD, 
#			    rajd, decjd, gl, gb, bmaj, bmin, bpa,
#			    dm, period, nchan, npol, nbin, nsub, 
#			    tsamp, nbits, nbeam,
#			    cnfg, inst, rcvr, hdrver, survey,
#			    telescope, site, obsrvr, 
#			    data_loc, file_size_bytes
#			    ) 
#			   
#			   VALUES 
#			   ( ?, ?, ?
#			     ?, ?, ?, ?, ?,
#			     ?, ?, ?, ?, 
#			     ?, ?, ?, ?, ?, ?, ?, 
#			     ?, ?, ?, ?, ?, ?, 
#			     ?, ?, ?, 
#			     ?, ?, ?, ?, ?, 
#			     ?, ?, ?, 
#			     ?, ?, 
#			     ) 
#		       };
#	}
#	else {
#	    
#	    print "Inserting obs file $display_filename\n";
#	    
#	    $sql = qq{ INSERT INTO observations 
#			   (filename, src_name, project_id,
#			    raj, decj, data_type, obsfreq, bw,
#			    scanlen, date, ut, MJD, 
#			    rajd, decjd, gl, gb, bmaj, bmin, bpa,
#			    dm, period, nchan, npol, nbin, nsub, 
#			    tsamp, nbits, nbeam,
#			    cnfg, inst, rcvr, hdrver, survey,
#			    telescope, site, obsrvr, 
#			    data_loc, file_size_bytes
#			    ) 
#			   
#			   VALUES 
#			   ( ?, ?, ?
#			     ?, ?, ?, ?, ?,
#			     ?, ?, ?, ?, 
#			     ?, ?, ?, ?, ?, ?, ?, 
#			     ?, ?, ?, ?, ?, ?, 
#			     ?, ?, ?, 
#			     ?, ?, ?, ?, ?, 
#			     ?, ?, ?, 
#			     ?, ?, 
#			     ) 
#		       };
#	}
##	my $sth = $db_handle->prepare( $sql );
##DEBUGGING	
#	my $sth;
#
#	#for( @records ) {
#  eval {
#
#    $sth->bind_param( 1,  $display_filename,                SQL_VARCHAR );
#    $sth->bind_param( 2,  $result_params[$params_NAME],     SQL_VARCHAR );
#    $sth->bind_param( 3,  $result_params[$params_RAJ],      SQL_VARCHAR );
#    $sth->bind_param( 4,  $result_params[$params_DECJ],     SQL_VARCHAR );
#    $sth->bind_param( 5,  $data_type,                       SQL_VARCHAR );
#    $sth->bind_param( 6,  $result_params[$params_FREQ],     SQL_NUMERIC );
#    $sth->bind_param( 7,  $result_params[$params_BW],       SQL_NUMERIC );
#    $sth->bind_param( 8,  $result_params[$params_LENGTH],   SQL_NUMERIC );
#    $sth->bind_param( 9,  $result_params[$params_DATE],     SQL_NUMERIC );
#    $sth->bind_param( 10, $result_params[$params_TIME],     SQL_NUMERIC );
#    $sth->bind_param( 11, $MJD,                             SQL_NUMERIC );
#    $sth->bind_param( 12, $rajd,                            SQL_NUMERIC );
#    $sth->bind_param( 13, $decjd,                           SQL_NUMERIC );
#    $sth->bind_param( 14, $gl,                              SQL_NUMERIC );
#    $sth->bind_param( 15, $gb,                              SQL_NUMERIC );
#    $sth->bind_param( 16, $bmaj,                            SQL_NUMERIC );
#    $sth->bind_param( 17, $bmin,                            SQL_NUMERIC );
#    $sth->bind_param( 18, $bpa,                             SQL_NUMERIC );
#    $sth->bind_param( 19, $result_params[$params_DM],       SQL_NUMERIC );
#    $sth->bind_param( 20, $result_params[$params_PERIOD],   SQL_NUMERIC );    
#    $sth->bind_param( 21, $result_params[$params_NCHAN],    SQL_INTEGER );
#    $sth->bind_param( 22, $result_params[$params_NPOL],     SQL_INTEGER );
#    $sth->bind_param( 23, $result_params[$params_NBIN],     SQL_INTEGER );    
#    $sth->bind_param( 24, $result_params[$params_NSUB],     SQL_INTEGER );
#    $sth->bind_param( 25, $result_params[$params_NBITS],    SQL_INTEGER );
#    $sth->bind_param( 26, $result_params[$params_TSAMP],    SQL_INTEGER );
#    $sth->bind_param( 27, $nbeam,                           SQL_INTEGER );
#    $sth->bind_param( 28, $result_params[$params_CNFG],     SQL_VARCHAR );
#    $sth->bind_param( 29, $result_params[$params_INST],     SQL_VARCHAR );
#    $sth->bind_param( 30, $result_params[$params_RCVR],     SQL_VARCHAR );
#    $sth->bind_param( 31, $hdrver,                          SQL_VARCHAR );
#    $sth->bind_param( 32, $survey,                          SQL_VARCHAR );
#    $sth->bind_param( 33, $result_params[$params_TELESCOP], SQL_VARCHAR );
#    $sth->bind_param( 34, $result_params[$params_SITE],     SQL_INTEGER );
#    $sth->bind_param( 35, $result_params[$params_OBSRVR],   SQL_VARCHAR );
#    $sth->bind_param( 13, $display_data_loc,                SQL_VARCHAR );
#    $sth->bind_param( 34, $file_size,                       SQL_INTEGER );
#
##    $sth->execute();
##DEBUGGING
#    DBI::dump_results($sth);
#
#    # $db_handle->commit(); # Autocommit is already ON so no need for a manual commit
#  };

  if( $@ ) {
      print "warning statement\n";
#DEBUGGING
#    warn "Database error: $DBI::errstr\n";
#    $db_handle->rollback(); #just die if rollback is failing
  }
	
	#}

#DEBUGGING
#	$sth->finish();
    }

# Removes 
##sub deleteStaleRecords {
##
##	my $sql, my $sth, my @data, my $file;
##	
##	$sql = qq{ SELECT data_loc, filename from cals };
##	$sth = $db_handle->prepare( $sql );
##
##  eval {
##
##    $sth->execute();
##
##	};
##	
##  if( $@ ) {
##    warn "Database error: $DBI::errstr\n";
##    $db_handle->rollback(); #just die if rollback is failing
##  }
##
##	my @cals2delete;
##	
##	while (@data = $sth->fetchrow_array()) {
##		
##		if (scalar(@data) == 2) {
##			$file = $data[0]."/".$data[1];
##			
##			if (!-e $file) {
##				 print "$file does NOT exist\n\n";
##				
##				# Push the row into the cals to delete
##				# So array will look like:
##				# {/full/path/to/dir, filename.cf, /full/path/to/dir, filename2.cf, etc..}
##				push (@cals2delete, @data);
##			}
##		}
##		        
##  }
##
##	$sql = qq{ SELECT data_loc, filename from observations };
##	
##	$sth = $db_handle->prepare( $sql );
##  eval {
##
##    $sth->execute();
##
##	};
##	
##  if( $@ ) {
##    warn "Database error: $DBI::errstr\n";
##    $db_handle->rollback(); #just die if rollback is failing
##  }
##	
##	my @obs2delete;
##	
##	while (@data = $sth->fetchrow_array()) {
##  	
##		if (scalar(@data) == 2) {
##			$file = $data[0]."/".$data[1];
##			
##			if (!-e $file) {
##				print "$file does NOT exist\n\n";
##				
##				# Push the row into the cals to delete
##				# So array will look like:
##				# {/full/path/to/dir, filename.cf, /full/path/to/dir, filename2.cf, etc..}
##				push (@obs2delete, @data);
##			}
##		}	          
##  }
##
##	print "There are " . scalar(@cals2delete)/2 . " stale cals (" . 
##	      scalar(@cals2delete) . "/2) and " . 
##	      scalar(@obs2delete)/2 . " stale obs to delete\n";
##
##	# Delete the rows
##	my $data_loc, my $filename;
##	
##	while (@cals2delete) {
##		$data_loc = shift @cals2delete;
##		$filename = shift @cals2delete;
##		
##		$sql = qq{ DELETE FROM cals WHERE data_loc = ? AND filename = ?};
##
##		$sth = $db_handle->prepare( $sql );
##  	eval {
##
##    	$sth->bind_param( 1, $data_loc, SQL_VARCHAR );
##    	$sth->bind_param( 2, $filename, SQL_VARCHAR );
##    	$sth->execute();
##
##		};
##		
##		if( $@ ) {
##    	warn "Database error: $DBI::errstr\n";
##    	$db_handle->rollback(); #just die if rollback is failing
##  	}
##		else {
##			print "Successfully deleted $data_loc/$filename\n";
##		}
##
##	}
##	
##	while (@obs2delete) {
##		$data_loc = shift @obs2delete;
##		$filename = shift @obs2delete;
##		
##		$sql = qq{ DELETE FROM observations WHERE data_loc = ? AND filename = ?};
##
##		$sth = $db_handle->prepare( $sql );
##		
##  	eval {
##
##    	$sth->bind_param( 1, $data_loc, SQL_VARCHAR );
##    	$sth->bind_param( 2, $filename, SQL_VARCHAR );
##    	$sth->execute();
##
##		};
##		
##		if( $@ ) {
##    	warn "Database error: $DBI::errstr\n";
##    	$db_handle->rollback(); #just die if rollback is failing
##  	}
##		else {
##			print "Successfully deleted $data_loc/$filename\n";
##		}
##	}
##
##	$sth->finish();
##	
##}

sub connectdb {
	
	# Connect to the MySQL server
#	my $dbh = DBI->connect("dbi:mysql:database=psrchive;host=localhost", "psrdba", "lighthouse")
#    	or die "Couldn't connect to database: $DBI::errstr\n";

#	print "Successfully connected to db\n" if $dbh;

# DEBUGGING	
    print "We would normally connect to the DB here\n";
    return 1;
#	return $dbh;
}