Index: /trunk/docs/VOrequirements.tex
===================================================================
--- /trunk/docs/VOrequirements.tex	(revision 74)
+++ /trunk/docs/VOrequirements.tex	(revision 75)
@@ -143,4 +143,12 @@
 are not defined for any file: date, ut, gl, gb, tsamp, nbeam. These
 may need to be defined before this is made public.
+
+The existing archive has a trawling script%
+\footnote{\texttt{herschel:/var/www/vhosts/pulsararchive.atnf.csiro.au/htdocs/perl/find\_archives.pl}}
+that searches for new files and incorporates them into the SQL
+archive. Such a script will not be necessary for the survey data, as
+there is a fixed amount of data for each survey, so, provided the data
+has all been converted, such a search will only need to be done once
+(and not periodically through a cron job).
 
 \subsection{Output considerations}
@@ -218,7 +226,9 @@
 option), or, if the hours/degrees, minutes and seconds must be stored
 separately, also have a decj\_sign column that indicates the sign of
-the declination (equal to +1 or $-1$).
-
-Currently this is done incorrectly even in the ATNF Pulsar
+the declination (equal to +1 or $-1$). The positions could also be
+stored as a string (\ie ``18:23:45.3''), although a numerical value
+will also be needed for angular separation calculations.
+
+Currently the positions are stored incorrectly even in the ATNF Pulsar
 Database\footnote{See
 http://www.atnf.csiro.au/research/pulsar/psrcat/}. A search ``within
@@ -230,4 +240,21 @@
 
 
+\subsection{Interacting with the mySQL database}
+
+It is possible to directly query the database via batch mode:
+\begin{quote}
+\texttt{> mysql -u psrdba -p < input-file.sql}
+\end{quote}
+although one still needs to enter the password for psrdba (although it
+can be specified with the option \texttt{--password=<password>}
+instead of \texttt{-p} -- a bit insecure though?). This method appears
+like a good way of interacting with the database, particularly for
+entering the data. It would be a simple matter to have a .sql file
+that contains the commands necessary to create a table and read data
+from a .csv file into that table.
+
+For querying the database, \eg via a conesearch, it would be similar,
+but would need some way of accessing the query parameters.
+
 \end{document}
 % LocalWords:  conesearch metadata
Index: /trunk/docs/pvoconvertGuide.tex
===================================================================
--- /trunk/docs/pvoconvertGuide.tex	(revision 74)
+++ /trunk/docs/pvoconvertGuide.tex	(revision 75)
@@ -130,11 +130,10 @@
 differences between the surveys. The S70 data\footnote{I have been
 examining the data in the directory
-\texttt{/pulsar/archive00/S70\_TAPES/}.} appears to follow the format
-given by the file\\ \texttt{%
-/pulsar/psr/cvshome/pulsar/soft\_swin\_old/soft\_swin/search/util/sc\_head.inc
-}.
-
-The PM data, however, seems to follow the file \\
-\texttt{/pulsar/psr/cvshome/pulsar/soft\_atnf/search/sc\_td/sc\_pmhead.inc},\\
+\texttt{/pulsar/archive00/S70\_TAPES/}} appears to follow the format
+given by this file\footnote{\texttt{%
+/pulsar/psr/cvshome/pulsar/soft\_swin\_old/soft\_swin/search/util/sc\_head.inc}}.
+
+The PM data, however, seems to follow this file\footnote{%
+\texttt{/pulsar/psr/cvshome/pulsar/soft\_atnf/search/sc\_td/sc\_pmhead.inc}},
 which is somewhat newer and has slightly different headers in some
 cases (for instance, where there are comments in the S70 data, there
@@ -166,6 +165,5 @@
 Total Survey Pointings  &44,299  &2,670           &6,456\\
 Total Survey Beams      &44,299  &34,710          &83,928\\
-Disk Size of 1 pointing (1 beam)
-                        &15.5~MB &96.1~MB         &24.3~MB\\
+Disk Size of 1 file     &15.5~MB &96.1~MB         &24.3~MB\\
 Disk Size of Survey     &692~GB  &3.18~TB         &1.94~TB\\
 \end{tabular}
@@ -449,4 +447,10 @@
 \end{quote}
 
+More than one file can be done at once, and wildcards can be accepted
+according to the usual shell rules \eg
+\begin{quote}
+\texttt{> pvoconvert S31201 S31204 S3131*}
+\end{quote}
+
 If the datafile is from the PH Survey, you need to say so explicitly
 so that \pvoconvert knows to apply the PH settings (as the PH files
@@ -454,10 +458,14 @@
 the \texttt{-H} flag:
 \begin{quote}
-\texttt{> pvoconvert -H <file>}
+\texttt{> pvoconvert -H <file(s)>}\hspace{1cm}\\
+\texttt{> pvoconvert --highlat <file(s)>}
 \end{quote}
-
-Other possible flags are \texttt{-h} to provide a help message, and
-\texttt{-v} to set verbose mode for maximum information for any error
-or warning messages.
+If a list of files is provided, and some are S70 files, they will be
+processed according to the S70 setup. However, you cannot provide a
+mix of PH and PM files, as all will get processed in the same way. 
+
+Other possible flags are \texttt{-h} or \texttt{--help} to provide a
+help message, and \texttt{-v} or \texttt{--verbose} to set verbose
+mode for maximum information for any error or warning messages.
 
 \section{Other PulsarVO programs}
Index: /trunk/sql/angsep_query-formatted.sql
===================================================================
--- /trunk/sql/angsep_query-formatted.sql	(revision 75)
+++ /trunk/sql/angsep_query-formatted.sql	(revision 75)
@@ -0,0 +1,24 @@
+-- suitable for batch processing: mysql -u psrdba -p < angsep_query-formatted.sql
+use psrchive;
+-- Performing a conesearch around position RA=272.46=18:09:50.4 and DEC=-19.73=-19:43:48
+-- This should pick up at least J1809-1943 and J1810-2005 (within 0.5deg of that position).
+SELECT '<TD>',pulsar_name,'</TD>','<TD>',period,'</TD>','<TD>', dm,'</TD>','<TD>', freq,'</TD>','<TD>',
+--	raj_hour, raj_minute, raj_second,
+--	(raj_hour+raj_minute/60.+raj_second/3600.)*15 as RA,
+--	decj_degree, decj_minute, decj_second,
+--	(abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree) as DECL,
+        ( acos(  cos( ( (raj_hour+raj_minute/60.+raj_second/3600.)*15 - 272.46) * pi() / 180. )
+	       * cos( (abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree) * pi() / 180. )
+	       * cos( -19.73 * pi() / 180. )
+	       + sin( (abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree) * pi() / 180. )
+	       * sin( -19.73 * pi() / 180. ) 
+	     ) * 180. / pi() ) as angsep, '</TD>'
+FROM observations
+WHERE ( acos(  cos( ( (raj_hour+raj_minute/60.+raj_second/3600.)*15 - 272.46) * pi() / 180. )
+	       * cos( (abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree) * pi() / 180. )
+	       * cos( -19.73 * pi() / 180. )
+	       + sin( (abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree) * pi() / 180. )
+	       * sin( -19.73 * pi() / 180. ) 
+	     ) * 180. / pi() ) <  0.5
+ORDER by angsep;
+-- period;
Index: /trunk/sql/angsep_query-s70-alt.sql
===================================================================
--- /trunk/sql/angsep_query-s70-alt.sql	(revision 75)
+++ /trunk/sql/angsep_query-s70-alt.sql	(revision 75)
@@ -0,0 +1,14 @@
+use psrchive;
+-- Use the following file to create a temporary s70 table based on a csv file
+--  produced by running pvosummary -c in the $S70V directory.
+source /DATA/SITAR_1/whi550/VO/PulsarVO/sql/create_table_s70.sql;
+--
+-- Performing a conesearch around position RA=0.=00:00:00 and DEC=-49.=-49:00:00 for the S70 data
+-- This should pick up four files.
+-- This uses the haversin formula for the angular separation.
+SELECT filename, raj, decj, 0.5*(bmaj+bmin) as beamsize, obsfreq,
+(degrees( asin( sqrt( sin(radians((rajd-0.)/2.))*sin(radians((rajd-0.)/2.))*cos(radians(decjd))*cos(radians(-49.))+sin(radians((decjd+49.)/2.))*sin(radians((decjd+49.)/2.)) ))*2.)) as angsep
+FROM s70
+WHERE (degrees( asin( sqrt( sin(radians((rajd-0.)/2.))*sin(radians((rajd-0.)/2.))*cos(radians(decjd))*cos(radians(-49.))+sin(radians((decjd+49.)/2.))*sin(radians((decjd+49.)/2.)) ))*2.) - 0.25*(bmaj+bmin) ) <  1.
+ORDER by angsep;
+-- period;
Index: /trunk/sql/angsep_query-s70.sql
===================================================================
--- /trunk/sql/angsep_query-s70.sql	(revision 75)
+++ /trunk/sql/angsep_query-s70.sql	(revision 75)
@@ -0,0 +1,13 @@
+use psrchive;
+-- Use the following file to create a temporary s70 table based on a csv file
+--  produced by running pvosummary -c in the $S70V directory.
+source /DATA/SITAR_1/whi550/VO/PulsarVO/sql/create_table_s70.sql;
+--
+-- Performing a conesearch around position RA=0.=00:00:00 and DEC=-49.=-49:00:00 for the S70 data
+-- This should pick up four files.
+SELECT filename, raj, decj, 0.5*(bmaj+bmin) as beamsize, obsfreq,
+(degrees( acos( cos(radians(rajd-0.))*cos(radians(decjd))*cos(radians(-49.))+sin(radians(decjd))*sin(radians(-49.)) ) )) as angsep
+FROM s70
+WHERE (degrees( acos( cos(radians(rajd-0.))*cos(radians(decjd))*cos(radians(-49.))+sin(radians(decjd))*sin(radians(-49.)) ) ) - 0.25*(bmaj+bmin) ) <  1.
+ORDER by angsep;
+-- period;
Index: /trunk/sql/angsep_query.sql
===================================================================
--- /trunk/sql/angsep_query.sql	(revision 75)
+++ /trunk/sql/angsep_query.sql	(revision 75)
@@ -0,0 +1,25 @@
+use psrchive;
+-- Performing a conesearch around position RA=272.46=18:09:50.4 and DEC=-19.73=-19:43:48
+-- This should pick up at least J1809-1943 and J1810-2005 (within 0.5deg of that position).
+SELECT pulsar_name, period, dm, freq,
+--	raj_hour, raj_minute, raj_second,
+--	(raj_hour+raj_minute/60.+raj_second/3600.)*15 as RA,
+--	decj_degree, decj_minute, decj_second,
+--	(abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree) as DECL,
+        (degrees(acos(  cos(radians( (raj_hour+raj_minute/60.+raj_second/3600.)*15 - 272.46) )
+			* cos(radians(abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree))
+			* cos(radians(-19.73) )
+			+ sin(radians(abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree))
+			* sin( radians(-19.73) ) 
+		     )  
+		) ) as angsep
+FROM observations
+WHERE ( degrees(acos( cos( radians( (raj_hour+raj_minute/60.+raj_second/3600.)*15 - 272.46 ) )
+			* cos(radians(abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree))
+			* cos(radians(-19.73))
+			+ sin(radians(abs(decj_degree)+decj_minute/60.+decj_second/3600.)*sign(decj_degree))
+			* sin(radians(-19.73)) 
+		    )
+		 )) <  0.5
+ORDER by angsep;
+-- period;
Index: /trunk/sql/create_table_s70.sql
===================================================================
--- /trunk/sql/create_table_s70.sql	(revision 75)
+++ /trunk/sql/create_table_s70.sql	(revision 75)
@@ -0,0 +1,29 @@
+DROP TEMPORARY TABLE IF EXISTS s70;
+
+CREATE TEMPORARY TABLE s70 (
+	file_id SMALLINT UNSIGNED AUTO_INCREMENT,
+	filename VARCHAR(30),
+	fileloc VARCHAR(100),
+	hdrver VARCHAR(10),
+	src_name VARCHAR(20),
+	raj VARCHAR(15),
+	decj VARCHAR(15),
+	rajd FLOAT,
+	decjd FLOAT,
+	bmaj FLOAT,
+	bmin FLOAT,
+	bpa FLOAT,
+	date_obs VARCHAR(30),
+	obsfreq FLOAT,
+	chanbw FLOAT,
+	nchan FLOAT,
+	scanlen FLOAT,
+	tbin FLOAT,
+	PRIMARY KEY (file_id)
+);
+
+LOAD DATA LOCAL
+INFILE "/DATA/SITAR_1/whi550/VO/PulsarVO/s70v_summary.csv"
+INTO TABLE s70
+FIELDS TERMINATED BY ","
+(filename,fileloc,hdrver,src_name,raj,decj,rajd,decjd,bmaj,bmin,bpa,date_obs,obsfreq,chanbw,nchan,scanlen,tbin);
