Context Navigation

source: trunk/htdocs/perl/find_archives.pl @ 206

Last change on this file since 206 was 206, checked in by MatthewWhiting, 16 years ago
Moving towards a working find_archives.pl. Still need to fix CPSR2 case, but DFB/WBC cases appear fine. Removed a debug message in database.php.
Property svn:executable set to ``*
File size: 25.8 KB

Line
1	#! /usr/bin/perl -w
2
3	# find_archives.pl
4	# Author: Albert Teoh
5	# Date: December 2005
6	# ATNF, CSIRO
7
8	# Crawls through data archive directories (given as an input parameter),
9	# searches for all observation and/or calibration files, enquires about
10	# their pulsar parameters and inputs these details as well as their
11	# location into a database
12
13
14	use strict;
15	use File::Find ();
16	#use DBI qw(:sql_types);
17	use POSIX;
18
19	# Set the variable $File::Find::dont_use_nlink if you're using AFS,
20	# since AFS cheats.
21
22	# for the convenience of &wanted calls, including -eval statements:
23	use vars qw/name dir *prune @archive_dirs $archive_dir @files
24	$vap_cmd $vap_out $params $num_vap_params $db_handle
25	$filename $raj $decj $dm $period $bw $cnfg
26	$freq $inst $mjdint $mjdfrac $npol $nchan $nbin $nsub
27	$rcvr $site $length $obsrvr $data_loc $data_type
28	$file_size $MJD $raH $raM $raS $decD $decM $decS $rajd $decjd $decstring
29	$pi $pion180 $NGP_RA $NGP_DEC $ASC_NODE $draR $decR
30	$sinb $sinl $cosl $gb $gl $gl_raw
31	$bmaj $bmin $bpa $hdrver $survey $nbeam
32	$output $vap $archive_extensions $ext
33	$default
34	@result_params $FILE_SIZE_INDEX $FB $DFB $WBC $CPSR2 $NULLFIELD
35	$params_NAME
36	$params_PROJID
37	$params_RAJ
38	$params_DECJ
39	$params_FREQ
40	$params_BW
41	$params_LENGTH
42	$params_DATE
43	$params_TIME
44	$params_MJD
45	$params_BMAJ
46	$params_BMIN
47	$params_BPA
48	$params_DM
49	$params_PERIOD
50	$params_NCHAN
51	$params_NPOL
52	$params_NBIN
53	$params_NSUB
54	$params_NBITS
55	$params_TSAMP
56	$params_NBEAM
57	$params_CNFG
58	$params_INST
59	$params_RCVR
60	$params_HDRVER
61	$params_TELESCOP
62	$params_SITE
63	$params_OBSRVR
64	/; # $machine
65
66	name = File::Find::name;
67	dir = File::Find::dir;
68	prune = File::Find::prune;
69
70	# The data types
71	$FB = "FB";
72	$WBC = "WBC";
73	$DFB = "DFB";
74	$CPSR2 = "CPSR2";
75
76	# How to write NULL fields to the database
77	$NULLFIELD = "NULL";
78
79	# Pulsar parameters (listed in "vap -H")
80	#$params = "name raj decj dm period bw cnfg freq inst intmjd fracmjd npol nchan nbin nsub rcvr site length obsrvr";
81	#$params = "name projid ra dec freq bw length stt_date stt_time intmjd fracmjd bmaj bmin bpa dm period nchan npol nbin nsub nbits tsamp nbeam cnfg backend rcvr telescop site obsrvr";
82	$params = "name projid ra dec freq bw length stt_date stt_time mjd bmaj bmin bpa dm period nchan npol nbin nsub nbits tsamp nbeam beconfig backend rcvr hdrver telescop asite observer";
83
84	# vap result parameter indices. Used to locate the resulting
85	# parameter's location
86	$params_NAME = 0;
87	$params_PROJID = 1;
88	$params_RAJ = 2;
89	$params_DECJ = 3;
90	$params_FREQ = 4;
91	$params_BW = 5;
92	$params_LENGTH = 6;
93	$params_DATE = 7;
94	$params_TIME = 8;
95	$params_MJD = 9;
96	$params_BMAJ = 10;
97	$params_BMIN = 11;
98	$params_BPA = 12;
99	$params_DM = 13;
100	$params_PERIOD = 14;
101	$params_NCHAN = 15;
102	$params_NPOL = 16;
103	$params_NBIN = 17;
104	$params_NSUB = 18;
105	$params_NBITS = 19;
106	$params_TSAMP = 20;
107	$params_NBEAM = 21;
108	$params_CNFG = 22;
109	$params_INST = 23;
110	$params_RCVR = 24;
111	$params_HDRVER = 25;
112	$params_TELESCOP= 26;
113	$params_SITE = 27;
114	$params_OBSRVR = 28;
115
116	# filename extensions
117	#$archive_extensions = "ar rf cf cfb fb";
118	$archive_extensions = "rf cf cfb fb";
119
120	# The machine to do all the processing on
121	#$machine = "tycho";
122
123	# The location of vap
124	$vap = "/pulsar/psr/linux/bin/vap";
125
126	#####################
127	# Begin main method
128	#
129	my @vap_params = split(/ /, $params);
130	$num_vap_params = scalar(@vap_params);
131
132	# The column number (starting from 0) of the file
133	# size when calling ls -l
134	$FILE_SIZE_INDEX = 4;
135
136	# Use the default operation which is to firstly delete any
137	# stale records, then crawl through and insert new records
138	$default = 1;
139
140	print "ARGV = @ARGV\n";
141
142	if (scalar(@ARGV) == 0) {
143	print "find_archives.pl: A crawler that populates a relational database\n
144	with indexes to observational data along with their respective\n
145	cal files.\n\n
146	Usage: find_archives.pl [options] directory1 [directory2 ...]\n\n
147	If no options are provided, it will default to do both -d and -p\n
148	Options:\n
149	\t-d Delete stale records\n
150	\t-p Populate database with new records\n";
151
152	}
153
154	# Connect to the database
155	$db_handle = connectdb();
156
157	#########################################################
158	# Step 1. First remove any stale entries in the database
159	# (remove file locations that no longer exist)
160	#########################################################
161
162	if ($ARGV[0] eq "-d") {
163	# deleteStaleRecords();
164	$default = 0;
165	shift @ARGV;
166	}
167
168
169	#########################################################
170	# Step 2. Crawl through each input directory and insert
171	# any new archives
172
173	if ($ARGV[0] eq "-p") {
174	shift @ARGV;
175	@archive_dirs = @ARGV;
176
177	foreach $archive_dir (@archive_dirs) {
178
179	# Traverse desired filesystems
180	File::Find::find({wanted => \&wanted}, $archive_dir);
181	}
182	$default = 0;
183	}
184
185	# Else do both
186	if ($default) {
187
188	# deleteStaleRecords();
189
190	@archive_dirs = @ARGV;
191
192	foreach $archive_dir (@archive_dirs) {
193
194	# Check that the input directory is legit
195
196	# Traverse desired filesystems
197	File::Find::find({wanted => \&wanted}, $archive_dir);
198	}
199
200	}
201	if ($db_handle->disconnect()) {
202	print "Successfully Disconnected from database\n";
203	}
204	else {
205	print "Error: Failed to disconnect from the database\n";
206	exit;
207	}
208
209	#######################################################
210	# Subroutine definitions
211	#
212	sub wanted {
213
214	my @files;
215	my @all_files;
216	my @cpsr2_obs_dirs;
217	my $file;
218	my $cpsr2_obs_dir;
219	my $search_str = "";
220	my $num_dumps = 0;
221
222	if (-d and /^[a-zA-Z]{0,1}[0-9]{4}[+-][0-9]{4}.*/s ) {
223
224	print "\n\nIn directory $_\n\n";
225	print "\$File::Find::name = $File::Find::name\n";
226
227	foreach $ext (split (/\s+/, $archive_extensions) ) {
228	$search_str .= $File::Find::name . "/*.$ext "
229	}
230
231	@all_files = glob($search_str);
232
233	if (scalar(@all_files) == 0) {
234
235	print "This is a CPSR2 directory: $_\n";
236
237	# Foreach observation
238	@cpsr2_obs_dirs = glob($File::Find::name . "/????-??-??-??:??:??");
239
240	foreach $cpsr2_obs_dir (@cpsr2_obs_dirs) {
241	print "dir is $cpsr2_obs_dir\n";
242
243	# Need to check if this entry already exists
244	$cpsr2_obs_dir =~ s/\s//g;
245
246	if (isDuplicate($cpsr2_obs_dir) == 1) {
247	print "$cpsr2_obs_dir is a duplicate entry. Skipping....\n\n";
248	next;
249	}
250	#else {
251	#print "$cpsr2_obs_dir NOT a duplicate. Inserting into database\n";
252	#}
253
254
255	@all_files = glob($cpsr2_obs_dir . "/mfb " . $cpsr2_obs_dir . "/mar");
256
257	if (scalar(@all_files) == 0) {
258	next;
259	}
260
261	#print "first file is " . $all_files[0] . "\n";
262
263	# Work out the length of each dump
264	# $vap_cmd = "ssh $machine $vap -n -c \\\"$params\\\" ".$all_files[0]. " \|";
265	$vap_cmd = "$vap -n -c \"$params\" ".$all_files[0]. " \| grep -v filename \| grep -v -x \"\" \|";
266
267	print "Calling vap...$vap_cmd\n";
268
269	open(VAP,$vap_cmd);
270
271	while ($vap_out = <VAP>) {
272
273	print "vap_out = $vap_out\n";
274
275	#@result_params = split(/\s+/, $vap_out);
276	@result_params = split(" ", $vap_out);
277
278	$length = $result_params[$params_LENGTH+1];
279	$result_params[$params_BW+1] *= 2;
280
281	#print "length = $length secs\n";
282
283	$num_dumps = `ls -1 $cpsr2_obs_dir/mfb $cpsr2_obs_dir/mar \| wc -l`;
284
285	#print "num dumps = $num_dumps\n";
286
287	$result_params[$params_LENGTH+1] = $length*$num_dumps;
288
289	#print "total length = " . $result_params[$params_LENGTH+1] . "\n";
290
291	#print "new bw = " . $result_params[$params_BW+1] . "\n";
292
293	# +1 because first column is filename
294	if (scalar(@result_params) == ($num_vap_params + 1)) {
295	print "@result_params\n";
296
297	populate_observations($cpsr2_obs_dir, $num_dumps);
298	}
299	else {
300	print "Incorrect number of cols = ".scalar(@result_params).". Expected " . ($num_vap_params+1) ."\n";
301	}
302
303	}
304	print "\n";
305
306
307	# Count the number of files
308	}
309	}
310	else {
311	foreach $file (@all_files) {
312
313	# Need to check if this entry already exists
314	$_ =~ s/\s//g;
315	$file =~ s/\s//g;
316
317	if (isDuplicate($file) == 1) {
318	#print "$file is a duplicate entry. Skipping....\n\n";
319	next;
320	}
321	else {
322	#print "$file NOT a duplicate. Inserting into database\n";
323	}
324
325	# $vap_cmd = "ssh $machine $vap -n -c \\\"$params\\\" $file \|";
326	$vap_cmd = "$vap -n -c \"$params\" $file \| grep -v filename \| grep -v -x \"\" \|";
327
328	print "Calling vap...$vap_cmd\n";
329
330	open(VAP,$vap_cmd);
331
332	while ($vap_out = <VAP>) {
333	$vap_out =~ s/^\s+//g; #remove any leading space
334	@result_params = split(/\s+/, $vap_out);
335
336	# +1 because first column is filename
337	if (scalar(@result_params) == ($num_vap_params + 1)) {
338	print "@result_params\n";
339
340	populate_observations($File::Find::name);
341	}
342	else {
343	print "Incorrect number of cols = ".scalar(@result_params).". Expected " . $num_vap_params+1 ."\n";
344	print "@result_params\n";
345	}
346	}
347	print "\n";
348	}
349	}
350	# don't traverse into this directory
351	$File::Find::prune = 1;
352	}
353	}
354
355	sub isDuplicate {
356
357	#DEBUGGING
358	return 0;
359
360	# my $filename = $_[0];
361	# my $search_data_loc;
362	# my $search_filename;
363	# my $pulsar_name;
364	#
365	# if ( $filename =~ m/(.(J[0-9]{4}[+-][0-9]{1,4}.))[\/]+([a-z][0-9]{6}_[0-9]{6}\.([a-z]{2,}))$/i ) {
366	# $search_data_loc = $1;
367	# $search_filename = $3;
368	# $pulsar_name = $2;
369	# }
370	#
371	# elsif ($filename =~ m/(.(J[0-9]{4}[+-][0-9]{1,4}.))[\/]+([0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}:[0-9]{2}:[0-9]{2})$/i ) {
372	# $search_data_loc = $1;
373	# $search_filename = $3;
374	# $pulsar_name = $2;
375	# }
376	# else {
377	# return 1;
378	# }
379	#
380	#
381	# my $sql;
382	#
383	# if ($pulsar_name =~ m/.*_R$/) {
384	#
385	# $sql = qq{ SELECT * from cals
386	# WHERE data_loc LIKE ? AND
387	# filename = ?
388	# };
389	# }
390	# else {
391	# print "This is an observation\n";
392	# $sql = qq{ SELECT * from observations
393	# WHERE data_loc LIKE ? AND
394	# filename = ?
395	# };
396	# }
397	#
398	# my $sth = $db_handle->prepare( $sql );
399	#
400	# eval {
401	# print "like $search_data_loc and filename = $search_filename \n";
402	#
403	# $sth->bind_param( 1, $search_data_loc, SQL_VARCHAR );
404	# $sth->bind_param( 2, $search_filename, SQL_VARCHAR );
405	# $sth->execute();
406	#
407	# };
408	#
409	# if( $@ ) {
410	# warn "Database error: $DBI::errstr\n";
411	# $db_handle->rollback(); #just die if rollback is failing
412	# }
413	#
414	# # read the records
415	# my @data = $sth->fetchrow_array();
416	#
417	# $sth->finish();
418	#
419	# if (scalar(@data) > 0) {
420	# return 1;
421	# }
422	#
423	# return 0;
424	}
425
426	sub populate_observations {
427
428
429	print "populate_obs(".$_[0].")\n";
430	$data_loc = $_[0];
431	my $display_data_loc = $data_loc;
432
433	if ($data_loc =~ m/(.*)\/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{2}:[0-9]{2}:[0-9]{2}$/i) {
434	print "Actual data loc is $1\n";
435	$display_data_loc = $1;
436	}
437
438	$filename = shift(@result_params);
439
440	# remove leading and trailing whitespace
441	$filename =~ s/ //g;
442
443	my $display_filename = $filename;
444	print "display_filename = $display_filename\n";
445	my $ls = `ls -l $data_loc/$filename`;
446
447	my @file_details = split(/\s+/, $ls);
448
449	$file_size = $file_details[$FILE_SIZE_INDEX];
450
451	# Get the total filesize
452	if (scalar (@_) > 1) {
453	my $num_dumps = $_[1];
454
455	$file_size = $num_dumps2; # *2 because of other band
456	print "Multiplying filesize by number files = $num_dumps with total $file_size\n";
457	}
458
459
460	# Try to find the data type
461	# my $file_is_good = true;
462
463	if ($filename =~ /^a.\.rf/ \|\| $filename =~ /^a.\.cf/ \|\|
464	$filename =~ /^r.\.rf/ \|\| $filename =~ /^r.\.cf/) {
465	$data_type = $DFB;
466	}
467	elsif ($filename =~ /^w.\.rf/ \|\| $filename =~ /^w.\.cf/) {
468	$data_type = $WBC;
469	}
470	elsif ($filename =~ /^m.\.cfb/ \|\| $filename =~ /^m.\.fb/ \|\|
471	$filename =~ /^n.\.cfb/ \|\| $filename =~ /^n.\.fb/ \|\|
472	$filename =~ /^o.\.cfb/ \|\| $filename =~ /^o.\.fb/ ) {
473	$data_type = $CPSR2;
474
475	# Use the directory of observation instead of filename instance
476	my @dirs = split(/\//, $data_loc);
477	print "The new filename will be " . $dirs[$#dirs] . " from $data_loc\n";
478	$display_filename = $dirs[$#dirs];
479	# # For some reason, the name of the pulsar cals for CPSR2 files are all "CAL"
480	# if ($data_loc =~ /\_R/) {
481	# my @jname_parts = split(/J/i, $dirs[$#dirs-1]);
482	# $result_params[$params_NAME] = $jname_parts[1];
483	# }
484
485	}
486	else {
487	$data_type = $NULLFIELD;
488	}
489
490	# # Try to find the data type
491	# if ($filename =~ /\.ar$/) {
492	# $data_type = $FB;
493	# }
494	# elsif ($filename =~ /\.rf/ \|\| $filename =~ /\.cf/ \|\| $filename =~ /fb/) {
495	# if ($filename =~ /^a/) { # DFB file
496	# $data_type = $DFB;
497	# }
498	#
499	# elsif ($filename =~ /^w/) { # Wide Band Correlator file
500	# $data_type = $WBC;
501	# }
502	# elsif ($filename =~ /^m/ \|\| $filename =~ /^n/ \|\| $filename =~ /^r/) {
503	# $data_type = $CPSR2;
504	#
505	# # Use the directory of observation instead of filename instance
506	# my @dirs = split(/\//, $data_loc);
507	#
508	# print "The new filename will be " . $dirs[$#dirs] . " from $data_loc\n";
509	#
510	# $display_filename = $dirs[$#dirs];
511	#
512	# # For some reason, the name of the pulsar cals for CPSR2 files are all "CAL"
513	# if ($data_loc =~ /\_R/) {
514	# my @jname_parts = split(/J/i, $dirs[$#dirs-1]);
515	# $result_params[$params_NAME] = $jname_parts[1];
516	# }
517	#
518	# }
519	# else {
520	# $data_type = undef;
521	# }
522	# }
523	print "display_filename = $display_filename\n";
524
525	# Check that N/A values are set to NULL
526
527	# Get the total MJD
528	# if ( $result_params[$params_MJDINT] eq "N/A" &&
529	# $result_params[$params_MJDFRAC] eq "N/A" ) {
530	# $MJD = undef;
531	# }
532	# else {
533	# $MJD = $result_params[$params_MJDINT] + $result_params[$params_MJDFRAC];
534	# }
535	if($result_params[$params_MJD] eq "INVALID"){
536	$MJD = $NULLFIELD;
537	}
538	else{
539	$MJD = $result_params[$params_MJD];
540	}
541
542	print "MJD = $MJD\n";
543
544	#### Calculate the RA and Dec in decimal degrees.
545
546	($raH,$raM,$raS) = split(':', $result_params[$params_RAJ]);
547	$rajd = ($raH + $raM/60. + $raS/3600.) * 15.;
548
549	($decD,$decM,$decS) = split(':', $result_params[$params_DECJ]);
550	$decjd = (abs($decD) + $decM/60. + $decS/3600.);
551	my @decstring = split(/ */,$result_params[$params_DECJ]);
552	if($decstring[0] eq '-'){
553	$decjd = -1. * $decjd;
554	}
555
556	### Calculate the Galactic Longitude and latitude
557	$pi=asin(1) * 2.;
558	$pion180 = $pi/180.;
559	$NGP_RA = 192.859508 * $pion180; # location of NGP
560	$NGP_DEC= 27.128336 * $pion180;
561	$ASC_NODE=32.932;
562
563	$draR = $rajd*$pion180 - $NGP_RA;
564	$decR = $decjd*$pion180;
565	$sinb = cos($decR) * cos($NGP_DEC) * cos($draR) + sin($decR) * sin($NGP_DEC);
566	$gb = asin($sinb); # this is the latitude, but in radians.
567
568	$sinl = (sin($decR) * cos($NGP_DEC) - cos($decR) * cos($draR) * sin($NGP_DEC)) / cos($gb);
569	$cosl = cos($decR) * sin($draR) / cos($gb);
570
571	# Need to get the correct quadrant, as this isn't preserved by
572	# atan, which returns angle between -90 and 90.
573	$gl_raw = atan($sinl/$cosl);
574	if($sinl > 0){
575	if($cosl > 0 ){ $gl = $gl_raw; }
576	else { $gl = $gl_raw + $pi; }
577	}
578	else {
579	if($cosl > 0){ $gl = $gl_raw + 2.*$pi; }
580	else{ $gl = $gl_raw + $pi; }
581	}
582	# Now put them into degrees.
583	$gb = $gb / $pion180;
584	$gl = ($gl / $pion180) + $ASC_NODE;
585
586	print "rajd = $rajd, decjd=$decjd, gl=$gl, gb=$gb\n";
587
588
589	# Beam information
590	$bmaj = $result_params[$params_BMAJ];
591	$bmin = $result_params[$params_BMIN];
592	$bpa = $result_params[$params_BPA];
593	if($bmaj eq "UNDEF" \|\| $bmaj eq "*" \|\| $bmaj == 0.){
594	# could not get beam info from vap output.
595	# Need to calculate manually.
596	$bmaj = (1.2(299792458./($result_params[$params_FREQ] 1.e6))/64.) / $pion180;
597	$bmin = (1.2(299792458./($result_params[$params_FREQ] 1.e6))/64.) / $pion180;
598	$bpa = 0.;
599	}
600
601	if($result_params[$params_NBITS] == 0){
602	$result_params[$params_NBITS] = $NULLFIELD;
603	}
604	if($result_params[$params_TSAMP] == 0){
605	$result_params[$params_TSAMP] = $NULLFIELD;
606	}
607
608
609	# Dud things that aren't required for this set of data.
610	$survey = $NULLFIELD;
611	$nbeam = 1;
612
613	my $i;
614	for ($i = 0; $i <= $#result_params; $i++) {
615	if ($result_params[$i] eq "N/A" \|\| $result_params[$i] eq "UNDEF" \|\| $result_params[$i] eq "INVALID") {
616	$result_params[$i] = $NULLFIELD;
617	}
618	}
619
620	# print out the list of parameters
621	print "filename = $display_filename\n";
622	print "src_name = " . $result_params[$params_NAME] . "\n";
623	print "projid = " . $result_params[$params_PROJID] . "\n";
624	print "raj = " . $result_params[$params_RAJ] . "\n";
625	print "dec = " . $result_params[$params_DECJ] . "\n";
626	print "data_type = $data_type\n";
627	print "freq = " . $result_params[$params_FREQ] . "\n";
628	print "bw = " . $result_params[$params_BW] . "\n";
629	print "scanlen = " . $result_params[$params_LENGTH] . "\n";
630	print "date = " . $result_params[$params_DATE] ."\n";
631	print "ut = " . $result_params[$params_TIME] ."\n";
632	print "MJD = " . $MJD ."\n";
633	print "rajd = " . $rajd ."\n";
634	print "decjd = " . $decjd ."\n";
635	print "gl = " . $gl ."\n";
636	print "gb = " . $gb ."\n";
637	print "bmaj = " . $bmaj ."\n";
638	print "bmin = " . $bmin ."\n";
639	print "bpa = " . $bpa ."\n";
640	print "dm = " . $result_params[$params_DM] . "\n";
641	print "period = " . $result_params[$params_PERIOD] . "\n";
642	print "nchan = " . $result_params[$params_NCHAN] . "\n";
643	print "npol = " . $result_params[$params_NPOL] . "\n";
644	print "nbin = " . $result_params[$params_NBIN] . "\n";
645	print "nsub = " . $result_params[$params_NSUB] . "\n";
646	print "tsamp = " . $result_params[$params_TSAMP] . "\n";
647	print "nbits = " . $result_params[$params_NBITS] . "\n";
648	# print "nbeam = " . $result_params[$params_NBEAM] . "\n";
649	print "nbeam = " . $nbeam . "\n";
650	print "cnfg = " . $result_params[$params_CNFG] . "\n";
651	print "inst = " . $result_params[$params_INST] . "\n";
652	print "rcvr = " . $result_params[$params_RCVR] . "\n";
653	print "hdrver = " . $result_params[$params_HDRVER] . "\n";
654	print "survey = " . $survey . "\n";
655	print "telescope = " . $result_params[$params_TELESCOP] . "\n";
656	print "site = " . $result_params[$params_SITE] . "\n";
657	print "obsrvr = " . $result_params[$params_OBSRVR] . "\n";
658	print "data_loc = $display_data_loc\n";
659	print "file_size = $file_size bytes\n";
660
661	# my $sql;
662	#
663	# if ($data_loc =~ /\_R/ \|\| $) {
664	# print "Inserting cal file $display_filename\n";
665	#
666	# $sql = qq{ INSERT INTO cals
667	# (filename, src_name, project_id,
668	# raj, decj, data_type, obsfreq, bw,
669	# scanlen, date, ut, MJD,
670	# rajd, decjd, gl, gb, bmaj, bmin, bpa,
671	# dm, period, nchan, npol, nbin, nsub,
672	# tsamp, nbits, nbeam,
673	# cnfg, inst, rcvr, hdrver, survey,
674	# telescope, site, obsrvr,
675	# data_loc, file_size_bytes
676	# )
677	#
678	# VALUES
679	# ( ?, ?, ?
680	# ?, ?, ?, ?, ?,
681	# ?, ?, ?, ?,
682	# ?, ?, ?, ?, ?, ?, ?,
683	# ?, ?, ?, ?, ?, ?,
684	# ?, ?, ?,
685	# ?, ?, ?, ?, ?,
686	# ?, ?, ?,
687	# ?, ?,
688	# )
689	# };
690	# }
691	# else {
692	#
693	# print "Inserting obs file $display_filename\n";
694	#
695	# $sql = qq{ INSERT INTO observations
696	# (filename, src_name, project_id,
697	# raj, decj, data_type, obsfreq, bw,
698	# scanlen, date, ut, MJD,
699	# rajd, decjd, gl, gb, bmaj, bmin, bpa,
700	# dm, period, nchan, npol, nbin, nsub,
701	# tsamp, nbits, nbeam,
702	# cnfg, inst, rcvr, hdrver, survey,
703	# telescope, site, obsrvr,
704	# data_loc, file_size_bytes
705	# )
706	#
707	# VALUES
708	# ( ?, ?, ?
709	# ?, ?, ?, ?, ?,
710	# ?, ?, ?, ?,
711	# ?, ?, ?, ?, ?, ?, ?,
712	# ?, ?, ?, ?, ?, ?,
713	# ?, ?, ?,
714	# ?, ?, ?, ?, ?,
715	# ?, ?, ?,
716	# ?, ?,
717	# )
718	# };
719	# }
720	## my $sth = $db_handle->prepare( $sql );
721	##DEBUGGING
722	# my $sth;
723	#
724	# #for( @records ) {
725	# eval {
726	#
727	# $sth->bind_param( 1, $display_filename, SQL_VARCHAR );
728	# $sth->bind_param( 2, $result_params[$params_NAME], SQL_VARCHAR );
729	# $sth->bind_param( 3, $result_params[$params_RAJ], SQL_VARCHAR );
730	# $sth->bind_param( 4, $result_params[$params_DECJ], SQL_VARCHAR );
731	# $sth->bind_param( 5, $data_type, SQL_VARCHAR );
732	# $sth->bind_param( 6, $result_params[$params_FREQ], SQL_NUMERIC );
733	# $sth->bind_param( 7, $result_params[$params_BW], SQL_NUMERIC );
734	# $sth->bind_param( 8, $result_params[$params_LENGTH], SQL_NUMERIC );
735	# $sth->bind_param( 9, $result_params[$params_DATE], SQL_NUMERIC );
736	# $sth->bind_param( 10, $result_params[$params_TIME], SQL_NUMERIC );
737	# $sth->bind_param( 11, $MJD, SQL_NUMERIC );
738	# $sth->bind_param( 12, $rajd, SQL_NUMERIC );
739	# $sth->bind_param( 13, $decjd, SQL_NUMERIC );
740	# $sth->bind_param( 14, $gl, SQL_NUMERIC );
741	# $sth->bind_param( 15, $gb, SQL_NUMERIC );
742	# $sth->bind_param( 16, $bmaj, SQL_NUMERIC );
743	# $sth->bind_param( 17, $bmin, SQL_NUMERIC );
744	# $sth->bind_param( 18, $bpa, SQL_NUMERIC );
745	# $sth->bind_param( 19, $result_params[$params_DM], SQL_NUMERIC );
746	# $sth->bind_param( 20, $result_params[$params_PERIOD], SQL_NUMERIC );
747	# $sth->bind_param( 21, $result_params[$params_NCHAN], SQL_INTEGER );
748	# $sth->bind_param( 22, $result_params[$params_NPOL], SQL_INTEGER );
749	# $sth->bind_param( 23, $result_params[$params_NBIN], SQL_INTEGER );
750	# $sth->bind_param( 24, $result_params[$params_NSUB], SQL_INTEGER );
751	# $sth->bind_param( 25, $result_params[$params_NBITS], SQL_INTEGER );
752	# $sth->bind_param( 26, $result_params[$params_TSAMP], SQL_INTEGER );
753	# $sth->bind_param( 27, $nbeam, SQL_INTEGER );
754	# $sth->bind_param( 28, $result_params[$params_CNFG], SQL_VARCHAR );
755	# $sth->bind_param( 29, $result_params[$params_INST], SQL_VARCHAR );
756	# $sth->bind_param( 30, $result_params[$params_RCVR], SQL_VARCHAR );
757	# $sth->bind_param( 31, $hdrver, SQL_VARCHAR );
758	# $sth->bind_param( 32, $survey, SQL_VARCHAR );
759	# $sth->bind_param( 33, $result_params[$params_TELESCOP], SQL_VARCHAR );
760	# $sth->bind_param( 34, $result_params[$params_SITE], SQL_INTEGER );
761	# $sth->bind_param( 35, $result_params[$params_OBSRVR], SQL_VARCHAR );
762	# $sth->bind_param( 13, $display_data_loc, SQL_VARCHAR );
763	# $sth->bind_param( 34, $file_size, SQL_INTEGER );
764	#
765	## $sth->execute();
766	##DEBUGGING
767	# DBI::dump_results($sth);
768	#
769	# # $db_handle->commit(); # Autocommit is already ON so no need for a manual commit
770	# };
771
772	if( $@ ) {
773	print "warning statement\n";
774	#DEBUGGING
775	# warn "Database error: $DBI::errstr\n";
776	# $db_handle->rollback(); #just die if rollback is failing
777	}
778
779	#}
780
781	#DEBUGGING
782	# $sth->finish();
783	}
784
785	# Removes
786	##sub deleteStaleRecords {
787	##
788	## my $sql, my $sth, my @data, my $file;
789	##
790	## $sql = qq{ SELECT data_loc, filename from cals };
791	## $sth = $db_handle->prepare( $sql );
792	##
793	## eval {
794	##
795	## $sth->execute();
796	##
797	## };
798	##
799	## if( $@ ) {
800	## warn "Database error: $DBI::errstr\n";
801	## $db_handle->rollback(); #just die if rollback is failing
802	## }
803	##
804	## my @cals2delete;
805	##
806	## while (@data = $sth->fetchrow_array()) {
807	##
808	## if (scalar(@data) == 2) {
809	## $file = $data[0]."/".$data[1];
810	##
811	## if (!-e $file) {
812	## print "$file does NOT exist\n\n";
813	##
814	## # Push the row into the cals to delete
815	## # So array will look like:
816	## # {/full/path/to/dir, filename.cf, /full/path/to/dir, filename2.cf, etc..}
817	## push (@cals2delete, @data);
818	## }
819	## }
820	##
821	## }
822	##
823	## $sql = qq{ SELECT data_loc, filename from observations };
824	##
825	## $sth = $db_handle->prepare( $sql );
826	## eval {
827	##
828	## $sth->execute();
829	##
830	## };
831	##
832	## if( $@ ) {
833	## warn "Database error: $DBI::errstr\n";
834	## $db_handle->rollback(); #just die if rollback is failing
835	## }
836	##
837	## my @obs2delete;
838	##
839	## while (@data = $sth->fetchrow_array()) {
840	##
841	## if (scalar(@data) == 2) {
842	## $file = $data[0]."/".$data[1];
843	##
844	## if (!-e $file) {
845	## print "$file does NOT exist\n\n";
846	##
847	## # Push the row into the cals to delete
848	## # So array will look like:
849	## # {/full/path/to/dir, filename.cf, /full/path/to/dir, filename2.cf, etc..}
850	## push (@obs2delete, @data);
851	## }
852	## }
853	## }
854	##
855	## print "There are " . scalar(@cals2delete)/2 . " stale cals (" .
856	## scalar(@cals2delete) . "/2) and " .
857	## scalar(@obs2delete)/2 . " stale obs to delete\n";
858	##
859	## # Delete the rows
860	## my $data_loc, my $filename;
861	##
862	## while (@cals2delete) {
863	## $data_loc = shift @cals2delete;
864	## $filename = shift @cals2delete;
865	##
866	## $sql = qq{ DELETE FROM cals WHERE data_loc = ? AND filename = ?};
867	##
868	## $sth = $db_handle->prepare( $sql );
869	## eval {
870	##
871	## $sth->bind_param( 1, $data_loc, SQL_VARCHAR );
872	## $sth->bind_param( 2, $filename, SQL_VARCHAR );
873	## $sth->execute();
874	##
875	## };
876	##
877	## if( $@ ) {
878	## warn "Database error: $DBI::errstr\n";
879	## $db_handle->rollback(); #just die if rollback is failing
880	## }
881	## else {
882	## print "Successfully deleted $data_loc/$filename\n";
883	## }
884	##
885	## }
886	##
887	## while (@obs2delete) {
888	## $data_loc = shift @obs2delete;
889	## $filename = shift @obs2delete;
890	##
891	## $sql = qq{ DELETE FROM observations WHERE data_loc = ? AND filename = ?};
892	##
893	## $sth = $db_handle->prepare( $sql );
894	##
895	## eval {
896	##
897	## $sth->bind_param( 1, $data_loc, SQL_VARCHAR );
898	## $sth->bind_param( 2, $filename, SQL_VARCHAR );
899	## $sth->execute();
900	##
901	## };
902	##
903	## if( $@ ) {
904	## warn "Database error: $DBI::errstr\n";
905	## $db_handle->rollback(); #just die if rollback is failing
906	## }
907	## else {
908	## print "Successfully deleted $data_loc/$filename\n";
909	## }
910	## }
911	##
912	## $sth->finish();
913	##
914	##}
915
916	sub connectdb {
917
918	# Connect to the MySQL server
919	# my $dbh = DBI->connect("dbi:mysql:database=psrchive;host=localhost", "psrdba", "lighthouse")
920	# or die "Couldn't connect to database: $DBI::errstr\n";
921
922	# print "Successfully connected to db\n" if $dbh;
923
924	# DEBUGGING
925	print "We would normally connect to the DB here\n";
926	return 1;
927	# return $dbh;
928	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: