Index: /trunk/docs/pvoconvertGuide.tex
===================================================================
--- /trunk/docs/pvoconvertGuide.tex	(revision 65)
+++ /trunk/docs/pvoconvertGuide.tex	(revision 65)
@@ -0,0 +1,1124 @@
+\documentclass[10pt,a4paper]{article}
+\usepackage{natbib}
+\usepackage{fancyheadings}
+\usepackage{array}
+\usepackage{xspace}
+
+%\textwidth=165 mm
+%\textheight=248 mm
+%\topmargin=-15 mm
+%\oddsidemargin=0 mm
+%\parindent=6 mm
+
+\pagestyle{fancy}
+\rhead{}
+\lhead{}
+\lfoot{\today}
+\cfoot{}
+\rfoot{\thepage}
+\setlength{\headrulewidth}{0in}
+
+%Define a test for doing PDF format -- use different code below
+\newif\ifPDF
+\ifx\pdfoutput\undefined\PDFfalse 
+\else\ifnum\pdfoutput > 0\PDFtrue 
+     \else\PDFfalse 
+     \fi 
+\fi
+% If we are creating a PDF, use different options for graphicx, hyperref.
+\ifPDF
+  \usepackage[pdftex]{graphicx,color}
+  \usepackage[pdftex,bookmarks]{hyperref}
+  \hypersetup{colorlinks=true,% 	    
+              citecolor=red,%
+              filecolor=red,%
+              linkcolor=red,%
+              urlcolor=red,%
+              }
+\else
+  \usepackage[dvips]{graphicx} 
+  \usepackage[dvips]{hyperref} 
+\fi
+
+\newcommand{\eg}{e.g.\xspace}
+\newcommand{\ie}{i.e.\xspace}
+\newcommand{\fdg}{\hbox{$.\!\!^\circ$}}
+\newcommand{\cfitsio}{\textsc{cfitsio}\xspace}
+\newcommand{\subint}{\textsc{subint}\xspace}
+\newcommand{\history}{\textsc{history}\xspace}
+\newcommand{\move}{\textsc{move}\xspace}
+\newcommand{\pvoconvert}{\textsc{pvoconvert}\xspace}
+\newcommand{\pvosummary}{\textsc{pvosummary}\xspace}
+\newcommand{\pvoconesearch}{\textsc{pvoconeseach}\xspace}
+\newcommand{\hi}{H\textsc{i}\xspace}
+\begin{document}
+%\titlepage
+\begin{flushright}
+{\huge \pvoconvert }\\
+\vspace{5mm}{\LARGE A Guide}\\
+\vspace{5mm}{\Large Matthew Whiting}
+\end{flushright}
+\tableofcontents
+
+\newpage
+\section{Summary}
+
+This document provides a guide to the motivation behind \pvoconvert
+and the details of how it works. The nature of the existing data is
+briefly described, primarily so that the conversion process can be
+understood. The algorithms involved are described, along with
+particular elements of the code that had to be developed. We also
+describe where the code itself resides and how it should be
+implemented. 
+
+Note: this document supersedes \textit{conversionCodeDetails.tex},
+which was written in the course of the development and so has
+information that is no longer of interest.
+
+\section{Motivation behind \pvoconvert}
+
+This software has been developed as part of a project to make all
+the data from various Parkes pulsar surveys available through an
+online archive. In doing so, we would like the data to be available in
+a standard format, and accessible through standard Virtual Observatory
+(VO) techniques. This has necessitated converting the data from the
+compact tape structures they are stored in currently to a FITS-based
+format that provides standard ways of accessing the meta-data (\ie the
+FITS header information), while preserving the data itself in an
+efficiently stored manner.
+
+There are three different surveys that we will be dealing with. The
+Parkes Southern Pulsar survey, or the 70-cm survey
+\citep[S70,][]{manchester96} is a survey of the entire southern sky
+made with a single beam at a wavelength of 70~cm. The Parkes
+Multi-beam survey \citep[PM,][]{manchester01} is a more sensitive
+survey, done at 21~cm with the 13-beam \hi multi-beam receiver. This
+survey covered only a portion of the Galactic plane. The final survey
+under consideration is the Parkes High-Latitude survey
+\citep[PH,][]{burgay06}, which used the same multi-beam receiver but
+searched out of the Galactic plane in a relatively narrow band of
+longitude (and with different integration times to the PM survey).
+
+
+\newpage
+\section{Existing data format}
+
+The current formats of the data files for all three surveys are very
+similar. Within each file, the data are stored in blocks, where the
+first 640 bytes consist of header information, detailing positional
+and instrumental information, and the next 48kB is the 1-bit digitised
+data. For the S70 data, there is only one beam, so the data block
+consists of successive samples of the $n$ frequency channels (it is
+possible for there to be two filterbanks, in which case the second set
+of channels follows the first -- all the S70 survey data \emph{should}
+have just one filterbank). There must be a whole number of samples to
+fit in the 48kbytes.
+
+For the PM and PH data, there are 13 beams for each sample, and these
+are stored in successive blocks. This means a given file switches
+continually from beam to beam as you move from block to block (and this
+has implications for the way the FITS file will be written).
+
+A given file can have many blocks in it, and they don't all have to be
+at the same sky position. The way the surveys work is to point at a
+given location for a fixed time and then move to the next position,
+taking data as they go. The data that is taken while the telescope is
+moving is flagged as such.
+
+The header information is crucial in the conversion process. As noted,
+the format of this information is largely the same, with slight
+differences between the surveys. The S70 data\footnote{I have been
+examining the data in the directory
+\texttt{/pulsar/archive00/S70\_TAPES/}.} appears to follow the format
+given by the file\\ \texttt{%
+/pulsar/psr/cvshome/pulsar/soft\_swin\_old/soft\_swin/search/util/sc\_head.inc
+}.
+
+The PM data, however, seems to follow the file \\
+\texttt{/pulsar/psr/cvshome/pulsar/soft\_atnf/search/sc\_td/sc\_pmhead.inc},\\
+which is somewhat newer and has slightly different headers in some
+cases (for instance, where there are comments in the S70 data, there
+is space for attenuator settings).
+
+
+We summarise in Table~\ref{tab-surveys} the details of each
+survey. The last two lines indicate the amount of disk space required
+for all data for one pointing, and for the whole survey. Note that the
+headers are not included here, but the size of the PSRFITS header is
+about 80~kB, so this would add about 3-6~GB to the total Survey size
+for each survey.
+
+\begin{center}
+\begin{table}[h]
+\begin{tabular}{l|lll}
+                        &S70     &PM              &PH\\ \hline
+Telescope               &Parkes  &Parkes          &Parkes\\
+Receiver                &70-cm   &21-cm multibeam &21-cm multibeam\\
+\# Beams/pointing       &1       &13              &13\\
+HPBW [arcmin]           &45      &14.2            &14.2\\
+Frequency [MHz]         &436     &1374            &1374\\
+Bandpass [MHz]          &32      &288             &288\\
+\# Channels             &256     &96              &96\\
+\# Polarisations        &1       &1               &1\\
+Sampling time [$\mu$s]  &300     &250             &125\\
+Digitisation            &1 bit   &1 bit           &1 bit\\
+Time/pointing [sec]     &152     &2100 = 35~min   &265\\
+Total Survey Pointings  &44,299  &2,670           &6,456\\
+Total Survey Beams      &44,299  &34,710          &83,928\\
+Disk Size of 1 pointing (1 beam)
+                        &15.5~MB &96.1~MB         &24.3~MB\\
+Disk Size of Survey     &692~GB  &3.18~TB         &1.94~TB\\
+\end{tabular}
+\caption{Comparison of the three surveys under consideration.}
+\label{tab-surveys}
+\end{table} 
+\end{center}
+
+\newpage
+\section{Output format}
+
+We convert the data into a FITS format, using an updated version of
+the PSRFITS format \citep{hotan04}. This has a Primary Header Data
+Unit (PHU) that records a lot of the basic metadata for the FITS file:
+information such as time \& date of observation, RA, Dec, observing
+frequency, integration time, instrument setup, and so on. There are
+also a number of binary table extensions that follow the PHU. The only
+two that are used by \pvoconvert are the \history (containing a record
+of the processing of the data) and \subint (containing the actual data)
+tables. 
+
+The survey observations were performed by pointing at a fixed point on
+the sky for a given integration time, then slewing the telescope to
+the next point. The data is continually being taken in this
+process. Each of the existing tape files will have more than one sky
+position in them. To make archive searching simpler, we have stored
+the data such that each sky position is stored in its own FITS
+file. Note that this means each \emph{beam} for multibeam observations
+is given a separate file, and so the multibeam data will need to be
+broken up according to the beam number.
+
+The resultant FITS files will then be unique according to the time
+at which they were observed, and so we use the time and date to form
+the filename, together with the survey type. The format of the
+filenames are:
+\begin{itemize}
+\item S70 data: \texttt{S70\_YYMMDD\_hhmmss.fits}
+\item PM  data: \texttt{PM\_YYMMDD\_hhmmss\_BB.fits}
+\item PH  data: \texttt{PH\_YYMMDD\_hhmmss\_BB.fits}
+\end{itemize}
+where the date of observations was YY/MM/DD (the year is given in
+two-digit format), the UT time was hh:mm:ss, and, for the multibeam
+case, the beam number is BB (from 01 to 13).
+
+The actual data (that is, the 1-bit digitised measurements) will be
+written to the ``data'' field of the \subint binary
+table. Each block (of length 48kbytes) is given a separate entry
+in the table, as this is the smallest chunk of data that we have
+information about (\eg time and position information from the
+header). The dimensions of each entry will be (\# samples per
+block)$\times$(\# channels per sample). Since there is a fixed time
+per pointing for each survey, there will be a fixed number of blocks
+for each pointing position, and hence a fixed number of rows in the
+\subint table. The following table summarises the size of the
+table and the dimensions of each data entry (note that there is no
+polarisation information in any of this data):
+
+\begin{table}[h]
+\begin{tabular}{lll}
+Survey  &Number of \subint rows &Dimensions of data table\\ 
+\hline
+S70          &330                            &$1536\times256$\\
+PM           &2051                           &$4096\times96$\\
+PH           &517                            &$4096\times96$\\
+\end{tabular}
+\caption{The size of the \subint data table for each survey.}
+\label{tab-datasize}
+\end{table}
+
+\newpage
+\section{Algorithm}
+
+Here we summarise the \pvoconvert algorithm, looking at what variables
+need to be examined and what functions will need to be called. Some
+classes and functions are referred to -- these are detailed in the
+next section.
+
+\begin{enumerate}
+
+\item Open Survey file. 
+  \begin{itemize}
+  \item Use \texttt{std::ifstream::open()} to open it. 
+  \end{itemize}
+
+\item Determine survey from first few bytes of file. Note that there
+  is no difference between the PM and PH surveys, as far as the file
+  format goes, just a difference in some parameter values.
+  \begin{itemize}
+  \item Use this information to define the Block setup (\eg
+    \texttt{DataBlock block("S70");}).
+  \end{itemize} 
+
+\item Read first block and obtain beam and moving state for that
+  block. 
+  \begin{itemize}
+  \item \texttt{block.readBlock(fin);}
+  \item \texttt{block.telescopeOK();}
+  \item This information will be used to determine future changes in
+  the beam number or the moving state of the telescope.
+  \end{itemize} 
+
+\item\label{step-init} Create a PSRFITS file and initialise based on
+the block information.
+  \begin{itemize}
+  \item Does this with \texttt{PSRFits::initialise(DataBlock \&)}.
+  \item Sets the time, date, beam, initial block and FITS filename. 
+  \item Opens the FITS file, calling the
+    \texttt{PSRFits::fixPSRFITS()} function that corrects any incorrect
+    header keywords.
+  \item Write the PHU of the fits file with
+    \texttt{PSRFits::writePHU(DataBlock \&);} 
+  \item Setup the \subint table with the necessary keywords and
+    DAT\_... arrays.
+  \end{itemize} 
+
+\item Write the current command-line command to the HISTORY table. 
+
+\item\label{step-data} If the file is open, write the block of data to
+the \subint binary table.
+  \begin{itemize}
+  \item Use the function \texttt{PSRFits::writeSubint(DataBlock \&);}
+  \item Obviously the first time round the file will be open, but it
+  may not be on subsequent occasions.
+  \end{itemize} 
+
+\item\label{step-readNew} Read next block header. If we are at the end
+  of the file, go to Step~\#\ref{step-closefile}.
+
+\item If the file is still going (so that the block is valid), check
+  beam information and moving flag.
+  \begin{itemize}
+  \item Read in the block as before.
+  \item Read the current beam and moving flags (with
+    \texttt{block.getKeyword} and \texttt{block.telescopeOK()}) and
+    compare to local versions.
+  \end{itemize} 
+
+\item If the FITS file is deemed to be full, or the beam number has
+  changed, close the file.
+  \begin{itemize}
+  \item See \S\ref{sec-moving} for details on what ``full'' means.
+  \item This is done with \texttt{PSRFits::closeFile()}.
+  \end{itemize}
+
+\item If we were moving, but are now pointing correctly, set up for a
+new file.
+  \begin{itemize}
+  \item This just means setting a flag in the PSRFits class to record
+  the fact that the file is a new one. We do not open a file on disk
+  yet.
+  \end{itemize}
+
+\item If the file is closed, and we are pointing correctly, go to
+  Step~\#\ref{step-init}.
+
+\item If the input file is still open (\ie we have read in another
+  block at Step~\#\ref{step-readNew}), go back to
+  Step~\#\ref{step-data}. If it is not, we are at the end of the
+  file. Go to next step.
+
+\item\label{step-closefile} Close PSRFITS file.
+  \begin{itemize}
+  \item \texttt{PSRFits::closeFile()}.
+  \item Need to close the existing survey file as well: \texttt{fin.close()}
+  \end{itemize} 
+  
+\end{enumerate}
+
+\newpage
+\section{Implementation}
+
+In this section, we discuss in more detail aspects of the \pvoconvert
+code: how it is implemented and what particular methods were required
+due to the peculiarities of the data.
+
+\subsection{Design of code}
+
+The code is written in C++: this was chosen so that several features
+in particular could be used, namely the classes and namespaces that
+C++ does better than C. The code makes use of the \cfitsio library to
+create and write to FITS files, using the psrheader.fits template
+file\footnote{\texttt{/pulsar/psr/linux/share/psrheader.fits}, pointed
+to by environment variable \texttt{\$PSRFITSDEFN}.} to define the
+structure of the PSRFITS file.
+
+There are three main classes that are used to manipulate the data and
+metadata. The primary one is the \texttt{PVO::FITS::PSRFits} class. This holds
+the information about the PSRFITS file: filenames of new and old
+files, time and date of the start of the data, beam number if needed,
+and a class containing data for the \subint table. This class,
+\texttt{PVO::FITS::Subint} holds all the information needed to populate the
+\subint binary table, both the header and the table cells themselves,
+with the exception of the actual observation data.
+
+Another class, \texttt{PVO::Block::DataBlock} keeps all information
+for a given block (640 byte header + 48kbyte data) in the old
+file. The header is stored as an STL string, while the data is stored
+as bytes in an array of chars. The class also records information
+about the type of survey and the rules for reading the header string
+(which differ for each survey). These rules are stored using the final
+class, \texttt{PVO::Header::SurveySchema}. It records the location and
+length of each header parameter in the existing header blocks, and
+provides functionality to extract a desired keyword (thereby hiding
+the location from the user). It also records certain parameters that
+are not stored in the header block, but are fixed for a given survey
+and are required by the PSRFITS PHU (such as the beam size, the
+instrumental configurations, the antenna locations).
+
+
+\subsection{Opening/reading existing data files}
+
+The different data formats for the different surveys can be
+distinguished by the first few bytes of the file. The S70 data has the
+first four bytes as \texttt{b6 6d 6d db} in hex (note that \texttt{6d}
+is \texttt{``m''} in ASCII., while the multibeam data files commence
+with \texttt{``PMD''} (in ASCII). So it is possible to
+discriminate between the different types upon reading the first few
+bytes of the file.
+
+\pvoconvert then reads the first few bytes, determines the survey
+type, and uses this to set up the \texttt{SurveySchema} class. This
+defines how the tape files' headers will be read and interpreted, to
+ensure the correct metadata is extracted and the correct FITS keywords
+are populated. Summaries of which headers go to which keywords are
+given in Appendix~\ref{app-headers}. The \texttt{SurveySchema} class
+uses \texttt{enum} labels to associate a given header parameter with a
+location in the header string. These labels are listed in
+Appendix~\ref{app-headerFields}, together with the header parameters
+they correspond to for each survey and their location in the 640-byte
+string. 
+
+\subsection{How we know when a FITS file is finished}
+\label{sec-moving}
+
+The S70 survey includes many pointings in a given (existing) data
+file. It has a \move flag in the block headers to tell you
+whether the telescope is pointing (01) or moving (02) (or is not doing
+a GRID pattern -- 00). 
+
+The S70 data is taken in a such a way that, after 152sec (or 330
+blocks), the move flag is set to 02, to indicate the telescope is
+about to move. However, the subsequent data is still good.
+
+The number of samples should be at least 512K ($=524288=2^{19}$),
+which requires 342 blocks. We therefore keep reading blocks and
+writing to the \subint table until the later of: \move is set
+to 02; and number of samples $>$ 521K.
+
+An important point to note is that the position, as stored in the
+\textsc{RA} and \textsc{DEC} keywords, will change a few blocks before
+the number of samples reaches 512K, so that the last few rows in the
+\subint table will have different positions to the rest.
+
+
+\newpage
+\section{Location of code and usage}
+
+\subsection{Location}
+
+The software has been developed in my local directory:
+\texttt{/DATA/SITAR\_1/whi550/VO/PulsarVO/src}. This has been kept
+under revision control using the Subversion system, and visible on the
+ATNF sourcecode website or through the Trac webpage
+\href{http://sourcecode.atnf.csiro.au/cgi-bin/trac\_pulsarvo.cgi}%
+{http://sourcecode.atnf.csiro.au/cgi-bin/trac\_pulsarvo.cgi}
+
+The working code, however, has been installed in the pulsar
+filesystem, in the directory \texttt{/pulsar/psr/pvo}, where a
+Makefile uses the standard make system to install the executables in
+the directory \texttt{/psr/linux/bin} (in the case of installation on
+a linux machine).
+
+\subsection{How to run \pvoconvert}
+
+The program is run very simply. If you have an existing tape file, \eg
+S31201, then you can convert it by typing
+\begin{quote}
+\texttt{> pvoconvert S31201}
+\end{quote}
+
+If the datafile is from the PH Survey, you need to say so explicitly
+so that \pvoconvert knows to apply the PH settings (as the PH files
+have the same basic structure as the PM ones). This is done by using
+the \texttt{-H} flag:
+\begin{quote}
+\texttt{> pvoconvert -H <file>}
+\end{quote}
+
+Other possible flags are \texttt{-h} to provide a help message, and
+\texttt{-v} to set verbose mode for maximum information for any error
+or warning messages.
+
+\section{Other PulsarVO programs}
+
+We briefly detail here two utility programs that have been developed
+for use with the newly created FITS files. They are not sufficiently
+complex to warrant their own document.
+
+\subsection{\pvosummary}
+
+\subsection{\pvoconesearch}
+
+
+\bibliographystyle{mn2e}
+\bibliography{mnrasmnemonic,pulsars}
+
+
+\newpage
+\section{Step-by-step guide to algorithm}
+
+
+\subsection{Writing the FITS file}
+
+Here we summarise the main \cfitsio routines that will be used
+to open, create, write, and close the FITS file:
+
+\begin{itemize}
+\item \verb|fits_create_template| -- uses the template FITS definition
+  to create a FITS file with all the correct keywords.
+\item \verb|fits_moveabs_hdu| -- use this to move to the primary HDU.
+\item \verb|fits_movenam_hdu| -- use this to move to particular
+  (named) HDUs in the FITS file \eg SUBINT, HISTORY, ...
+\item \verb|fits_update_key| -- add the value of keywords (the
+  keywords should already be there, but if not they get appended).
+\item \verb|fits_get_num_rows| -- get the number of rows in the binary
+  table.
+\item \verb|fits_insert_rows| -- add a new row to the binary table
+  (used when adding the next block).
+\item \verb|fits_write_tdim| -- write the TDIM\# information for a
+  binary table entry that is a multidimensional array.
+\item \verb|fits_get_colnum| -- get the column number of a named
+  column in a binary table.
+\item \verb|fits_write_col| -- write data to a column of a binary
+  table. May need to do this one row at a time, so want to specify the
+  last row (\ie the highest row number present in the table) as the
+  \texttt{firstrow} argument.
+\item \verb|fits_write_col_bit| -- write 1-bit data to a column of a
+  binary table (see note in next subsection about writing 1-bit data).
+\item \verb|fits_close_file| -- obviously, closes the file once we're
+  done. 
+\end{itemize}
+
+
+\subsection{Writing the data to the FITS file's binary table}
+\label{sec-1bit1st}
+
+When the actual data (as opposed to the header information) gets
+written to the binary table, it needs to be stored as 1-bit data (else
+the storage requirements will blow out), and the datatype given by the
+TFORM keyword needs to 'X'. The writing can be accomplished in a
+couple of ways:
+
+\begin{enumerate}
+\item Use \verb|fits_write_col_bit|, giving an array of characters
+  that are either true (\ie not equal to zero) for a bit value of 1,
+  or false (\ie equal to zero) for a bit value of 0.
+\item Use \verb|fits_write_col|, writing 8 bits at a time, giving an
+  array of 8-bit integers by specifying the TBYTE datatype.
+\end{enumerate}
+
+The latter method might be the best option, as it is much less memory
+intensive. For instance, if one read a character from the existing
+data, that is 1 byte or 8 bits, and so 8 separate measurements. This
+will translate into 8 characters (and hence 8 \emph{bytes}) to be
+passed to \verb|fits_write_col_bit|. It is just a matter of making
+sure there is no loss of information.
+
+
+\newpage
+\appendix
+\newpage
+\section{Conversion of header keywords}
+\label{app-headers}
+
+The following pages show how the header parameters listed in the
+existing survey data map to header keywords in the PSRFITS file. The
+parameter names are listed as they are described in the definition
+files, along with their size in bytes (and their length, if they are
+arrays -- this is done in FORTRAN format). 
+
+Alongside the parameter names are the PSRFITS keywords that each
+parameter maps to. These are labelled by the header unit that the
+keyword belongs to: \eg \verb|PHU -> STT_DATE| means the keyword
+\verb|STT_DATE| in the Primary HDU (abbreviated herein as PHU).
+
+Not all parameters have a place in the PSRFITS structure. Those
+without a clear place to go have their comments bracketed by \verb|**|
+-- many of these are simply irrelevant for the PSRFITS data structure,
+while some may need the appropriate keyword to be added to the PSRFITS
+definition. 
+
+The third sub-section of this appendix goes the other way, and
+indicates what keywords in the PSRFITS header are defined and which
+will need some input from elsewhere. This input, where necessary, will
+be of a generic nature -- such as telescope information -- that will
+most likely be able to be hard-coded into the conversion program. We
+only discuss the primary HDU and the \textsc{subint} binary table, as
+these are the two most important extensions.
+
+\newpage
+\subsection{S70 survey}
+{\footnotesize
+\begin{verbatim}
+ synch_hd*8,         :  Used to determine Survey type
+ version_hd*4,       :  
+ tapenr_hd*4,        :  
+ scan_cntr_hd*4,     :  
+ blk_cntr_hd*8,      :  Used to record the initial block of a FITS file.
+ date_hd*8,          :  PHU -> STT_DATE, plus FITS filename
+ mjd_hd*8,           :  PHU -> STT_IMJD
+ ut_strt_hd*16,      :  PHU -> STT_TIME
+ lst_strt_hd*12,     :  PHU -> STT_LST	
+ coord_type_hd*2     :  PHU -> COORD_MD
+	
+ ra_strt_hd*16,      :  PHU -> STT_CRD1
+ dec_strt_hd*16,     :  PHU -> STT_CRD2
+ gl_strt_hd*8,       :  PHU -> STT_CRD1
+ gb_strt_hd*8,       :  PHU -> STT_CRD2
+ scan_rate_hd*8,     :
+ scan_len_hd*8,      :
+ comment_hd*64,      :  PHU -> SRC_NAME (only the Field ID Gxxxxx)
+ nfilter_hd*2,       :  PHU -> NRCVR
+ chanbw_hd(2)*8,     :  SUBINT -> FREQ,     PHU->OBSFREQ
+ nchan_hd(2)*4,      :  SUBINT -> NCH_FILE, PHU->OBSFREQ
+ freq_chan1_hd(2)*12 :  SUBINT -> DAT_FREQ, PHU->OBSFREQ
+		      
+ samp_int_hd(2)*12,  :  SUBINT -> TBIN, TSAMP
+ samp_grp_hd(2)*4,   :  Used to keep track of the number of samples
+ grp_blk_hd*8,       :  Used to keep track of the number of samples
+ blk_sec_hd*8,       :  SUBINT -> TSUBINT, OFFSUB
+ data1_hd*4,         :
+ ut_hd*16,           :  PHU -> DATE_OBS, plus FITS filename
+ lst_hd*12,          :  SUBINT -> LST_SUB   
+ RA_hd*16,           :  SUBINT -> RA_SUB     
+ Dec_hd*16,          :  SUBINT -> DEC_SUB   
+ gl_hd*8,            :  SUBINT -> GLON_SUB 
+ gb_hd*8,            :  SUBINT -> GLAT_SUB 
+ tel_zen_hd*8        :  SUBINT -> TEL_AZ     
+		      						       
+ tel_az_hd*8,        :  SUBINT -> TEL_ZEN 
+ status_hd(4)*8,     :
+ tpower_hd(8)*8,     :
+ nblk_read_hd*8,     :
+ scan_ratel_hd*8,    :
+ scan_rateb_hd*8,    :
+ byte_grp_hd*4,      :
+ ibeam_hd*4,         :  Not used for S70 data -- only one beam.
+ psrname_hd*12,      :
+ data2_hd*4,         :
+ obs_filen_hd*16,    :
+ nbits_hd(2)*2,      :  SUBINT -> TFORM for DATA field
+ ddm_hd*8,           :
+ nddch_hd(2)*4,      :
+ move_hd*2,          :  Used to determine whether scan has finished.
+ pnterr_hd*6,        :
+ tree_hd*2,          :
+ nsys_hd*2,          :
+ telid_hd*10,        :  PHU -> TELESCOP
+ nbeam_hd*4,         :
+ reserve*38          :  ** blank field to fill up 640 bytes **
+\end{verbatim}
+}
+\newpage
+\subsection{Multibeam data}
+{\footnotesize
+\begin{verbatim}
+ program_hd*6        :  Used to determine Survey type
+ version_hd*4        :
+ tapenr_hd*6         :
+ file_cntr_hd*4      :
+ blk_cntr_hd*8       :  Used to record the initial block of a FITS file.
+ date_hd*8           :  PHU -> STT_DATE
+ mjd_hd*8            :  PHU -> STT_IMJD
+ ut_strt_hd*16       :  PHU -> STT_TIME
+ lst_strt_hd*12      :  PHU -> STT_LST
+ coord_type_hd*2     :  PHU -> COORD_MD
+ ra_strt_hd*16       :  PHU -> STT_CRD1  (if coord_type=J2000)
+ dec_strt_hd*16      :  PHU -> STT_CRD2
+ gl_strt_hd*8        :  PHU -> STT_CRD1  (if coord_type=Galactic)
+ gb_strt_hd*8        :  PHU -> STT_CRD2	
+ feed_angle_hd*8     :  PHU -> FA_REQ, SUBINT->FD_ANG
+ obs_time_hd*8       : 
+ comment_hd*64       : 
+ nfilter_hd*2        :  PHU -> NRCVR
+ chanbw_hd(2)*8      :  SUBINT -> FREQ,     PHU->OBSFREQ
+ nchan_hd(2)*4       :  SUBINT -> NCH_FILE, PHU->OBSFREQ
+ freq_chan1_hd(2)*12 :  SUBINT -> DAT_FREQ, PHU->OBSFREQ
+
+ samp_int_hd(2)*12   :  SUBINT -> TBIN, TSAMP			   
+ samp_grp_hd(2)*4    :	Used to keep track of the number of samples
+ grp_blk_hd*8        :	Used to keep track of the number of samples
+ blk_sec_hd*8        :  SUBINT -> TSUBINT 
+ fdcntrl_hd*2        :  PHU -> FD_MODE
+ data_hd(2)*1        :
+ ut_hd*16            :  PHU -> DATE_OBS, plus FITS filename
+ lst_hd*12           :  SUBINT -> LST_SUB  
+ RA_hd*16            :  SUBINT -> RA_SUB   
+ Dec_hd*16           :  SUBINT -> DEC_SUB  
+ gl_hd*8             :  SUBINT -> GLON_SUB 
+ gb_hd*8             :  SUBINT -> GLAT_SUB 
+ tel_zen_hd*8        :  SUBINT -> TEL_AZ   
+					   	   
+ tel_az_hd*8         :  SUBINT -> TEL_ZEN   
+ atten_hd(4)*4       :
+ tpower_hd(20)*4     :
+ nblk_read_hd*8      :
+ scan_ratel_hd*8     :
+ scan_rateb_hd*8     :
+ nbeam_hd*4          :
+ ibeam_hd*4          :  Goes in FITS filename.
+ pname_hd*16         :  PHU -> SRC_NAME  (not necessarily a pulsar name though)
+ obs_filen_hd*16     :
+ nbits_hd(2)*2       :  SUBINT -> TFORM for DATA entry
+ ddm_hd*8            :
+ nddch_hd(2)*4       :
+ move_hd*2           :  Used to determine if scan has finished
+ pnterr_hd*6         :
+ tree_hd*2           :
+ nsys_hd*2           :
+ telid_hd*10         :  PHU -> TELESCOP
+ pangle_hd*8         :  SUBINT -> PAR_ANG 
+ smp_bsw_hd*8        :
+ cal_cycle_hd*4      :  PHU -> CAL_DCYC
+\end{verbatim}
+}
+
+\newpage
+\section{Identification of fields for namespace}
+\label{app-headerFields}
+
+{\scriptsize
+\begin{tabular}{l|c|lll|lll}
+ENUM keyword &used? &S70 parameter &start &width 
+                                      &PM parameter &start &width\\
+ \hline
+
+CODE         &  &synch       &0   &8    &program     &0   &6    \\
+VERSION      &  &version     &8   &4    &version     &6   &4    \\
+TAPENR	     &  &tapenr	     &12  &4    &tapenr      &10  &6    \\
+SCAN\_CNTR   &  &scan\_cntr  &16  &4    &scan\_cntr  &16  &4    \\
+BLK\_CNTR    &  &blk\_cntr   &20  &8    &blk\_cntr   &20  &8    \\
+DATE	     &* &date	     &28  &8    &date	     &28  &8     \\
+MJD	     &* &mjd	     &36  &8    &mjd	     &36  &8     \\
+UT\_STRT     &* &ut\_strt    &44  &16   &ut\_strt    &44  &16    \\
+LST\_STRT    &  &lst\_strt   &60  &12   &lst\_strt   &60  &12    \\
+COORD\_TYPE  &* &coord\_type &72  &2    &coord\_type &72  &2     \\
+RA\_STRT     &  &ra\_strt    &74  &16   &ra\_strt    &74  &16    \\
+DEC\_STRT    &  &dec\_strt   &90  &16   &dec\_strt   &90  &16    \\
+GL\_STRT     &  &gl\_strt    &106 &8    &gl\_strt    &106 &8     \\
+GB\_STRT     &  &gb\_strt    &114 &8    &gb\_strt    &114 &8     \\
+FEED\_ANGLE  &* &[RESERVE]   &--- &---  &feed\_angle &122 &8     \\
+OBS\_TIME    &  &[RESERVE]   &--- &---  &obs\_time   &130 &8    \\
+SCAN\_RATE   &  &scan\_rate  &122 &8    &[RESERVE]   &--- &---  \\
+SCAN\_LEN    &  &scan\_len   &130 &8    &[RESERVE]   &--- &---  \\
+COMMENT      &  &comment     &138 &64   &comment     &138 &64   \\
+NFILTER	     &* &nfilter     &202 &2    &nfilter     &202 &2     \\
+CHANBW	     &* &chanbw	     &204 &2x8  &chanb       &204 &2x8   \\
+NCHAN	     &* &nchan	     &220 &2x4  &nchan	     &220 &2x4   \\
+FREQ\_CHAN1  &* &freq\_chan1 &228 &2x12 &freq\_chan1 &228 &2x12  \\
+SAMP\_INT    &* &samp\_int   &252 &2x12 &samp\_int   &252 &2x12 \\
+SAMP\_GRP    &* &samp\_grp   &276 &2x4  &samp\_grp   &276 &2x4  \\
+GRP\_BLK     &* &grp\_blk    &284 &8    &grp\_blk    &284 &8    \\
+BLK\_SEC     &* &blk\_sec    &292 &8    &blk\_sec    &292 &8     \\
+FDCNTRL	     &  &[RESERVE]   &--- &---  &fdcntrl     &300 &2     \\
+DATA\_TYPE   &  &data1	     &300 &4    &data        &302 &2x1  \\
+UT	     &* &ut	     &304 &16   &ut  	     &304 &16   \\
+LST	     &* &lst	     &320 &12   &lst	     &320 &12    \\
+RA	     &* &ra	     &332 &16   &RA	     &332 &16    \\
+DEC	     &* &dec	     &348 &16   &Dec	     &348 &16    \\
+GL	     &  &gl	     &364 &8    &gl	     &364 &8     \\
+GB	     &  &gb	     &372 &8    &gb	     &372 &8     \\
+TEL\_ZEN     &* &tel\_zen    &380 &8    &tel\_zen    &380 &8     \\
+TEL\_AZ	     &* &tel\_az     &388 &8    &tel\_az     &388 &8     \\
+STATUS	     &  &status	     &396 &4x8  &[RESERVE]   &--- &---  \\
+ATTEN	     &? &[RESERVE]   &--- &---  &atten	     &396 &4x4   \\
+TPOWER	     &  &tpower	     &428 &8x8  &tpower      &412 &20x4 \\
+NBLK\_READ   &  &nblk\_read  &492 &8    &nblk\_read  &492 &8    \\
+SCAN\_RATEL  &  &scan\_ratel &500 &8    &scan\_ratel &500 &8    \\
+SCAN\_RATEB  &  &scan\_rateb &508 &8    &scan\_rateb &508 &8    \\
+BYTE\_GRP    &  &byte\_grp   &516 &4    &[RESERVE]   &--- &---  \\
+NBEAM	     &  &nbeam	     &598 &4    &nbeam	     &516 &4    \\
+IBEAM	     &* &ibeam	     &520 &4    &ibeam	     &520 &4    \\
+PSRNAME	     &* &psrname     &524 &12   &pname	     &524 &16   \\
+DATA\_TWO    &  &data2	     &536 &4    &[RESERVE]   &--- &---  \\
+OBS\_FILEN   &  &obs\_filen  &540 &16   &obs\_filen  &540 &16   \\
+NBITS	     &* &nbits	     &556 &2x2  &nbits	     &556 &2x2  \\
+DDM	     &  &ddm	     &560 &8    &ddm	     &560 &8    \\
+NDDCH	     &  &nddch	     &568 &2x4  &nddch	     &568 &2x4  \\
+MOVE	     &* &move	     &576 &2    &move	     &576 &2    \\
+PNTERR	     &  &pnterr	     &578 &6    &pnterr      &578 &6    \\
+TREE	     &  &tree	     &584 &2    &tree	     &584 &2    \\
+NSYS	     &  &nsys	     &586 &2    &nsys	     &586 &2    \\
+TELID	     &* &telid	     &588 &10   &telid	     &588 &10   \\
+PANGLE	     &* &[RESERVE]   &--- &---  &pangle      &598 &8    \\
+SMP\_BSW     &  &[RESERVE]   &--- &---  &smp\_bsw    &606 &8    \\
+CAL\_CYCLE   &  &[RESERVE]   &--- &---  &cal\_cycle  &614 &4    \\
+RESERVE      &  &reserve     &602 &38   &reserve     &618 &22   
+\end{tabular}
+}
+
+\section{How keywords are defined.}
+
+In this section we summarise the keywords and other header parameters
+that go into the PSRFITS file, together with their comment field, and
+detail how the value is arrived at. The value will be generated by:
+\begin{itemize}
+\item Some fixed value that is a constant for the particular survey --
+  either a numeric value or a string.
+\item A direct substitution of the existing file's header values
+(written as \eg \verb|<NFILTER>|, using the enumeration setup detailed
+in the previous setction)
+\item A value calculated from one or more header values via some
+  formula. 
+\end{itemize}
+
+In the case of the first option, we sometimes separate according to
+the survey under consideration -- the options for each survey are then
+listed separately.
+
+The keywords are divided according to the extension, where we show the
+PHU and the \subint extensions. We also show new keywords that are
+added to the PHU, that involve information not in the PSRFITS
+definition that will be useful for VO purposes. These are added using
+the \texttt{PSRFits::addNewKeywords()} function.
+
+For the \subint table, we give the column values that are set for each
+block -- the ``Column Title'' is the \texttt{TTFORM\#} parameter in
+the header. Where the columns are multi-valued, we show how the $i$th
+value is calculated.
+
+\subsection{Existing PHU keywords}
+
+\begin{tabular}{|l|>{\setlength{\rightskip}{0pt plus 2cm}}p{7cm}|>{\setlength{\rightskip}{0pt plus 2cm}}p{5cm}|}\hline
+FITS Keyword & Comment field from FITS header & Value \\ \hline\hline
+
+DATE     & File creation date (\mbox{YYYY-MM-DDThh:mm:ss} UTC) & Date of file creation, from \texttt{fits\_get\_system\_time()}.\\\hline
+
+OBSERVER & Observer name(s)                         & \textbf{not set}\\ \hline
+
+PROJID   & Project name                             & Survey-specific project name.\\ \hline
+
+TELESCOP & Telescope name                           & \verb|<TELID>|\\\hline
+
+ANT\_X    & [m] Antenna ITRF X-coordinate (D)       & -4554231.5 \\\hline
+
+ANT\_Y    & [m] Antenna ITRF Y-coordinate (D)       & 2816759.1 \\\hline
+
+ANT\_Z    & [m] Antenna ITRF Z-coordinate (D)       & -3454036.3 \\\hline
+
+FRONTEND & Rx and feed ID                           & \emph{S70}: ``70cm\_Cavity\_Disk'', \emph{PM/PH}: ``20cm\_MB''\\\hline
+
+FD\_POLN  & LIN or CIRC                             & ``LIN'' \\\hline
+
+FD\_HAND  & +/- 1. +1 is LIN:A=X,B=Y, CIRC:A=L,B=R (I) &  \emph{S70}: 0, \emph{PM/PH}: $+1$\\\hline
+
+FD\_SANG  & [deg] FA of E vect for equal sig in A\&B (E) &  \emph{S70}: 0, \emph{PM/PH}: $-90$\\\hline
+
+FD\_XYPH  & [deg] Phase of A\^* B for injected cal (E) &  \emph{S70}: 0, \emph{PM/PH}: 0\\\hline
+
+BACKEND  & Backend ID                               & ``FB\_1BIT''\\ \hline
+
+BECONFIG & Backend configuration file name          & ``'' \\ \hline
+
+BE\_PHASE & 0/+1/-1 BE cross-phase:0 unknown,+/-1 std/rev & 0 \\\hline
+
+BE\_DCC   & 0/1 BE downconversion conjugation corrected & 0 \\\hline
+
+TCYCLE   & [s] On-line cycle time (D)               & 0 \\\hline
+
+NRCVR    & Number of receiver channels (I)          & \verb|<NFILTER>|\\\hline
+
+OBS\_MODE & (PSR, CAL, SEARCH)                      & ``SEARCH'' \\\hline
+
+SRC\_NAME & Source or scan ID                       & \verb|<PSRNAME>| (Note that this will be blank for survey data!)\\\hline
+
+COORD\_MD & Coordinate mode (J2000, Gal, Ecliptic, etc.)& \verb|<COORD_TYPE>|, translated into text.\\ \hline
+
+STT\_CRD1 & Start coord 1 (hh:mm:ss.sss or ddd.ddd) & \verb|<RA>|\\\hline
+
+STT\_CRD2 & Start coord 2 (-dd:mm:ss.sss or -dd.ddd)& \verb|<DEC>|\\\hline
+
+TRK\_MODE & Track mode (TRACK, SCANGC, SCANLAT)     & ``TRACK'' \\\hline
+
+STP\_CRD1 & Stop coord 1 (hh:mm:ss.sss or ddd.ddd)  & \verb|<RA>|\\\hline
+
+STP\_CRD2 & Stop coord 2 (-dd:mm:ss.sss or -dd.ddd) & \verb|<DEC>|\\\hline
+
+SCANLEN  & [s] Requested scan length (E)            & \emph{S70}: 152, \emph{PM}: 2100, \emph{PH}: 265 \\ \hline
+
+FD\_MODE  & Feed track mode - FA, CPA, SPA, TPA     & \emph{S70}: ``FIXED'', \emph{PM/PH}: ``CPA'' \\ \hline
+
+FA\_REQ   & [deg] Feed/Posn angle requested (E)     & \emph{S70}: 0, \emph{PM/PH}: 30 \\ \hline
+
+CAL\_MODE & Cal mode (OFF, SYNC, EXT1, EXT2)        & ``OFF'' \\ \hline
+
+CAL\_FREQ & [Hz] Cal modulation frequency (E)       &  0 \\\hline
+
+CAL\_DCYC & Cal duty cycle (E)                      & 0 \\\hline
+
+CAL\_PHS & Cal phase (wrt start time) (E)           & 0 \\\hline
+
+STT\_DATE & Start UT date (YYYY-MM-DD)              & \verb|<DATE>|, with the century added on to the start, and '/' changed to '-' \\\hline
+
+STT\_TIME & Start UT (hh:mm:ss)                     & \verb|<UT>|\\ \hline
+
+STT\_IMJD & Start MJD (UTC days) (J - long integer) & \verb|<MJD>|\\ \hline
+
+STT\_SMJD & [s] Start time (sec past UTC 00h) (J)   & \verb|<UT>| as a decimal $\times$ 3600.\\ \hline
+
+STT\_OFFS & [s] Start time offset (D)               & 0 \textbf{NEED TO CHECK THIS.}\\ \hline
+
+STT\_LST  & [s] Start LST (D)                       & \verb|<LST>| as a decimal $\times$ 3600.\\ \hline
+\end{tabular}
+
+\subsection{New PHU keywords}
+
+\begin{tabular}%
+{|l|>{\setlength{\rightskip}{0pt plus 2cm}}p{7cm}|>{\setlength{\rightskip}{0pt plus 2cm}}p{5cm}|}\hline
+FITS Keyword & Comment field from FITS header & Value \\ \hline\hline
+
+OBSFREQ  & [MHz] Observing frequency 
+& \verb|<FREQ_CHAN1>| $+$ ((\verb|<NCHAN>|$-1)/2) \times$
+\verb|<CHANBW>| \\ \hline
+
+RA       & Right Ascension & \verb|<RA>| \\ \hline
+
+DEC      & Declination     & \verb|<DEC>| \\ \hline
+
+DATE-OBS & Date of observation (\mbox{YY-MM-DDThh:mm:ss UTC})  &\verb|<DATE>|, with the century added on to the start, and '/'  changed to '-' \\\hline
+
+EQUINOX  & Equinox         & Either ``1950.0'' or ``2000.0'', if \verb|<COORD_TYPE>| is of one of
+these. \\ \hline
+
+BMAJ     &                 &\emph{S70}: 0.75, \emph{PM/PH}: 0.24 \\ \hline
+
+BMIN     &                 &\emph{S70}: 0.75, \emph{PM/PH}: 0.24 \\ \hline
+
+BPA      &                 &\emph{S70}: 0., \emph{PM/PH}: 0. \\ \hline
+\end{tabular}
+
+
+
+\subsection{SUBINT keywords}
+
+
+\begin{tabular}%
+{|l|>{\setlength{\rightskip}{0pt plus 2cm}}p{7cm}|>{\setlength{\rightskip}{0pt plus 2cm}}p{5cm}|}\hline
+FITS Keyword & Comment field from FITS header & Value \\ \hline\hline
+
+INT\_TYPE & Time axis (TIME, BINPHSPERI, BINLNGASC, etc)   & ``TIME'' \\\hline
+
+INT\_UNIT & Unit of time axis (SEC, PHS (0-1), DEG)        & ``SEC'' \\ \hline
+
+TSAMP     & [s] Sample interval for SEARCH-mode data       & \verb|<SAMP_INT>| $\times$ 1000. \\ \hline
+
+NBIN      & Nr of bins (1 for SEARCH-mode data)            & 1 \\ \hline
+
+NCH\_FILE & Number of channels/sub-bands in this file      & \verb|<NCHAN>| \\ \hline
+
+NCH\_STRT & Start channel/sub-band number (0 to NCHAN-1)   & 0 \\ \hline
+
+NPOL      & Nr of polarisations in table                   & 1 \\ \hline
+
+NSBLK     & Samples/row (each NPOL*NCH\_FILE, SEARCH mode) & \verb|<SAMP_GRP>| $\times$ \verb|<GRP_BLK>| \\ 
+\hline
+\end{tabular}
+
+\subsection{SUBINT columns}
+
+\begin{tabular}%
+{|l|>{\setlength{\rightskip}{0pt plus 2cm}}p{7cm}|>{\setlength{\rightskip}{0pt plus 2cm}}p{5cm}|}\hline
+Column title & Comment field from FITS header & Value \\ 
+\hline\hline
+
+ISUBINT  & Subint number
+& Incremental counter of the number of rows.\\ \hline
+TSUBINT  & Length of subintegration                         & \verb|<BLK_SEC>| \\ \hline
+
+OFFS\_SUB & Offset from Start of subint centre              &$\frac{1}{2}\times(2\times(\verb|<BLK_CNTR>|-(\verb|blk_initial|))+1)\times \verb|<BLK_SEC>|$ \\ \hline
+
+LST\_SUB  & LST at subint centre                            & \verb|<LST>| \\ \hline
+
+RA\_SUB   & RA (J2000) at subint centre                     & \verb|<RA>| \\ \hline
+
+DEC\_SUB  & [turns] Dec (J2000) at subint centre            & \verb|<DEC>| \\ \hline
+
+GLON\_SUB & [deg] Gal longitude at subint centre            & \verb|<GL>| \\ \hline
+
+GLAT\_SUB & [deg] Gal latitude at subint centre             & \verb|<GB>| \\ \hline
+
+FD\_ANG   & [deg] Feed angle at subint centre               & \verb|<FEED_ANGLE>| \\ \hline
+
+POS\_ANG  & [deg] Position angle of feed at subint centre   & \verb|<PANGLE>| \\ \hline
+
+PAR\_ANG  & [deg] Parallactic angle at subint centre        & $(\verb|<FEED_ANGLE>| + \verb|<PANGLE>|) \bmod 360$ \\ \hline
+
+TEL\_AZ   & [deg] Telescope azimuth at subint centre        & \verb|<TEL_AZ>| \\ \hline
+
+TEL\_ZEN  & [deg] Telescope zenith angle at subint centre   & \verb|<TEL_ZEN>| \\ \hline
+
+DAT\_FREQ & [MHz] Centre frequency for each channel         & $\verb|<FREQ_CHAN1>| + i\times\verb|<CHANBW>|$ \\ \hline
+
+DAT\_WTS  & Weights for each channel                        & 1 $\forall i \in\left[1,\verb|<NCHAN>|\right]$ \\ \hline
+
+DAT\_OFFS & Data offset for each channel                    & 0 $\forall i \in\left[1,\verb|<NCHAN>|\right]$ \\ \hline
+
+DAT\_SCL  & Data scale factor for each channel              & 1 $\forall i \in\left[1,\verb|<NCHAN>|\right]$ \\ \hline
+\end{tabular}
+
+\newpage
+\section{Details of Implementation of Header analysis}
+
+We list here some important details about how the header information
+will be implemented in the code:
+
+\begin{itemize}
+\item There will be an enum structure that enumerates the keywords
+  given in the first column of the preceeding table. These will be
+  used to refer to the relevant header keyword throughout the code.
+\item There will be arrays of starting position and length of
+  field. These will need to be implemented differently for each
+  survey, due to the different starting positions for some keywords
+  (although most are the same, some that we will need are either in
+  different positions or not present in one or other of the
+  setups). The different implementations can be done through a single
+  function call within a class (with a switch to the necessary setup,
+  depending on the argument of the function).
+\item Some fields include more than one value, and so there will need
+  to be an array recording the dimension of each field (\eg NCHAN,
+  giving the number of channels, is 2$\times$4 characters in size, 
+\item There will be a standard function (templated to cope with int,
+  float and string variables) that extracts the relevant substring and
+  reads the value in that string. It will also need an argument
+  indicating the element required, if the keyword has more than one
+  dimension (this should be an optional parameter). \eg:
+  \begin{verbatim}
+    string utStartTime = readParameter(headerString, UT_STRT, 0);
+    float ra = readParameter(headerString, RA);
+    int bandwidth2 = readParameter(headerString, CHANBW, 1);
+  \end{verbatim}
+\item Enumerated keywords that are not present for a given survey
+  should point to the RESERVE location, and be given a length of
+  0. This provides a good test as to whether something is present
+  (\eg testing to see if the fdcntrl parameter is present, as it will
+  only be in the PM survey data).
+\end{itemize}
+
+The class that we have designed to implement this scheme is as follows:
+{\small
+\begin{verbatim}
+  class SurveySchema
+  {
+  public:
+    SurveySchema(){};
+    SurveySchema(std::string surveyName);
+    virtual ~SurveySchema(){};
+    int getStartPosition(int field){return startPosition[field];};
+    int getFieldLength(int field){return fieldLength[field];};
+    int getFieldDimension(int field){return fieldDimension[field];};
+    template <class T> T readParameter(std::string headerString, int field,
+				       int elementNumber = 0);
+    class xBadSurvey{
+    public:
+      xBadSurvey(std::string s){surveyName=s;};
+      virtual ~xBadSurvey(){};
+      std::string getSurvey(){return surveyName;};
+    private:
+      std::string surveyName;
+    };
+ 
+  private:
+    int startPosition[numKeywords];
+    int fieldLength[numKeywords];
+    int fieldDimension[numKeywords];
+  };
+
+\end{verbatim}
+}
+
+\newpage
+\section{Implementation of existing data file}
+
+The fundamental unit of data from the existing files is going to be
+the \emph{block}. This is the 640-character-long header string, plus
+the 48-kbyte data array. It will have the following elements:
+
+\begin{itemize}
+\item A string indicating which survey type it belongs to
+\item A header string -- the 640-character long string with all the
+  header information in it.
+\item A SurveyHeader object, which has the encoding of the header
+  string recorded, and provides the functionality to extract keywords
+  from the header string.
+\item A data array, stored as characters (?).
+\item A function that acts as a front-end to the readParameter
+  function in the header class above.
+\end{itemize}
+
+\newpage
+\section{Implementation of FITS file access}
+
+The FITS file will be accessed through a special C++ Class that has
+functions that will control the writing of data to the PHU and to the
+SUBINT binary table. These functions will be front-ends to the
+updateKeyword and writeColumnValue functions that access the
+individual keywords and table columns.
+
+\subsection{Positional information}
+
+WARNING! Contrary to first thought, the existing header parameters
+\textsc{ra\_strt} and \textsc{dec\_strt} are not the appropriate ones
+for the \textsc{stt\_crd1,2} FITS keywords. They are the starting
+positions \emph{as far as the file is concerned}, and so are the same
+even when the telescope has moved to a different position. They are
+not the position of the start of the subints for a given sky location! 
+
+\subsection{Storing 1-bit binary data}
+
+As discussed earlier (Section~\ref{sec-1bit1st}), there are a couple of
+ways to write 1-bit data to a binary table using \cfitsio. However, we
+have run up against several problems. 
+
+First, the \texttt{TFORMn} keyword that goes with the data column
+needs to be `X', but the PSRFITS definition file currently has it as
+`I'. This is not a problem -- in fact, one probably does not want the
+general definition file to be 1-bit by default. It can easily be
+changed to `X' by the \verb|fits_update_key| function. 
+
+However, a word of caution. The implicit size of the row needs to be
+preserved, and so since the `I' datatype is 2-bytes, we need to
+replace `I' by `16X' (since there are 16 bits in 2 bytes). This is
+incorporated into a \texttt{fixPSRFITS} function in the PSRFits class.
+
+Next, however, there is a bug in \cfitsio that incorrectly calculates
+the number of bytes for a given number of bits when the `X' datatype
+is being used. This bug is in \texttt{editcol.c} in the \cfitsio
+distribution and is easily fixed:
+\begin{quote}
+delbyte = ((newveclen + 1) / 8) - ((repeat + 1) / 8);
+\end{quote}
+should be replaced by
+\begin{quote}
+delbyte = ((newveclen + 7) / 8) - ((repeat + 7) / 8);
+\end{quote}
+
+Another bug/feature to be wary of: when writing the first block to a
+FITS file, it seems to want to write a second row with nothing in
+it. This doesn't happen for any subsequent rows/blocks, so I'm not
+really sure what is happening. It is easy to delete that extra row on
+the first time round (for the first block, we also need to redefine
+the data cell's dimensions and size, so this can be done in the same
+way). 
+
+
+\end{document}
+% LocalWords:  cfitsio
Index: /trunk/src/print_pm_data.cc
===================================================================
--- /trunk/src/print_pm_data.cc	(revision 64)
+++ /trunk/src/print_pm_data.cc	(revision 65)
@@ -5,5 +5,4 @@
 #include <vector>
 #include <string>
-using namespace std;
 
 enum FIELDS 
Index: /trunk/src/print_s70_data.cc
===================================================================
--- /trunk/src/print_s70_data.cc	(revision 64)
+++ /trunk/src/print_s70_data.cc	(revision 65)
@@ -6,5 +6,4 @@
 #include <string>
 #include "pvo-utils.hh"
-using namespace std;
 using namespace PVO;
 
Index: /trunk/src/pvo-FITS.hh
===================================================================
--- /trunk/src/pvo-FITS.hh	(revision 64)
+++ /trunk/src/pvo-FITS.hh	(revision 65)
@@ -279,15 +279,15 @@
       STATE filestate;          ///< OPEN or CLOSED?
       std::string UT_initial;   ///< The UT at the start of the FITS
-      ///   file data.
+                                ///   file data.
       std::string DATE_initial; ///< The UT date at the start of the
-      ///   data.
+				///   data.
       int beam_number;          ///< What beam are we using? Necessary
-      ///   for FITS file name
-      int blk_initial;          ///< The block number of the first block
-      ///   in the file.
-      long numSamples;          ///< The number of samples stored in the
-      ///   FITS file.
+				///   for FITS file name
+      int blk_initial;          ///< The block number of the first
+				///   block in the file.
+      long numSamples;          ///< The number of samples stored in
+				///   the FITS file.
       Subint subintTable;       ///< Subint class containing table
-      ///   header keywords.
+				///   header keywords.
 
       std::string oldFilename;  ///< The filename of the old-format data.
Index: /trunk/src/tests/makeFakeData.cc
===================================================================
--- /trunk/src/tests/makeFakeData.cc	(revision 64)
+++ /trunk/src/tests/makeFakeData.cc	(revision 65)
@@ -11,5 +11,4 @@
 using namespace Block;
 using namespace FITS;
-using namespace std;
 
 int main()
@@ -17,5 +16,5 @@
   const int patternSize = 7
   int pattern[patternSize] = {1,1,2,3,5,8,13};
-  stringstream ss;
+  std::stringstream ss;
 
   // DO 1-BIT DATA.
@@ -27,8 +26,8 @@
     for(int j=0;j<pattern[i];j++) bindata1[place++] = true;
   }
-  for(int i=0;i<bindata1.size();i++) cout << bindata1[i];
-  cout << endl;
-  string binstring1 = binaryToString(bindata1);
-  cout << binstring1 <<endl;
+  for(int i=0;i<bindata1.size();i++) std::cout << bindata1[i];
+  std::cout << endl;
+  std::string binstring1 = binaryToString(bindata1);
+  std::cout << binstring1 <<endl;
 
   char *data = new char[Block::dataLength];
@@ -37,5 +36,5 @@
   ss.str("");
   ss << "cp data/dummy_header_s70_1bit.dat data/fakedata_s70_1bit.dat";
-  cerr << "Executing command: " << ss.str() << endl;
+  std::cerr << "Executing command: " << ss.str() << endl;
   system(ss.str().c_str());
   ofstream fdata;
@@ -44,5 +43,5 @@
   fdata.close();
 
-//   string fhead = "data/dummy_header_s70_1bit.dat";
+//   std::string fhead = "data/dummy_header_s70_1bit.dat";
 //   ifstream headfile(fhead.c_str());
 //   DataBlock block1(idSurvey(fhead));
@@ -51,11 +50,11 @@
 //   PSRFits fitsfile1;
 //   fitsfile1.initialise(block1);
-//   string name = fitsfile1.getFITSname();
+//   std::string name = fitsfile1.getFITSname();
 //   fitsfile1.writeSubintRow(block1);
 //   fitsfile1.closeFile();
 //   headfile.close();
-//   stringstream ss;
+//   std::stringstream ss;
 //   ss << "mv " << name << " data/fakedata_1bit.fits";
-//   cerr << "Executing command: " << ss.str() << endl;
+//   std::cerr << "Executing command: " << ss.str() << endl;
 //   system(ss.str().c_str());
 
@@ -69,6 +68,6 @@
     for(int j=0;j<pattern[i];j++) bindata2[place++] = (valuectr++)%3 + 1;
   }
-  for(int i=0;i<bindata2.size();i++) cout << bindata2[i];
-  cout << endl;
+  for(int i=0;i<bindata2.size();i++) std::cout << bindata2[i];
+  std::cout << endl;
   vector<bool> bindata2b(80);
   for(int i=0;i<40;i++){
@@ -76,8 +75,8 @@
     bindata2b[i*2+1] = bindata2[i]%2;
   }
-  for(int i=0;i<bindata2b.size();i++) cout << bindata2b[i];
-  cout << endl;
-  string binstring2 = binaryToString(bindata2b);
-  cout << binstring2 <<endl;
+  for(int i=0;i<bindata2b.size();i++) std::cout << bindata2b[i];
+  std::cout << endl;
+  std::string binstring2 = binaryToString(bindata2b);
+  std::cout << binstring2 <<endl;
 
   for(int i=0;i<bindata2.size();i++) data[i] = binstring2[i];
@@ -85,5 +84,5 @@
   ss.str("");
   ss << "cp data/dummy_header_s70_2bit.dat data/fakedata_s70_2bit.dat";
-  cerr << "Executing command: " << ss.str() << endl;
+  std::cerr << "Executing command: " << ss.str() << endl;
   system(ss.str().c_str());
   fdata.open("data/fakedata_s70_2bit.dat",ios_base::binary | ios::app);
@@ -108,6 +107,6 @@
   //------------------------------------------------------------
   // DO 4-BIT DATA.
-  string bindata4(40,'0');
-  string values4="123456789abcdef";
+  std::string bindata4(40,'0');
+  std::string values4="123456789abcdef";
   vector<int> bindata4int(40);
   place=0;
@@ -121,5 +120,5 @@
     for(int j=0;j<pattern[i];j++) bindata4int[intplace++] = (valuectr++)%15 + 1;
   }
-  cout << bindata4 << endl;
+  std::cout << bindata4 << endl;
   vector<bool> bindata4b(160);
   for(int i=0;i<40;i++){
@@ -130,8 +129,8 @@
     bindata4b[i*4+3] = bindata4int[i]%2;
   }
-  for(int i=0;i<bindata4b.size();i++) cout << bindata4b[i];
-  cout << endl;
-  string binstring4 = binaryToString(bindata4b);
-  cout << binstring4 <<endl;
+  for(int i=0;i<bindata4b.size();i++) std::cout << bindata4b[i];
+  std::cout << endl;
+  std::string binstring4 = binaryToString(bindata4b);
+  std::cout << binstring4 <<endl;
 
   for(int i=0;i<bindata4.size();i++) data[i] = binstring4[i];
@@ -139,5 +138,5 @@
   ss.str("");
   ss << "cp data/dummy_header_s70_4bit.dat data/fakedata_s70_4bit.dat";
-  cerr << "Executing command: " << ss.str() << endl;
+  std::cerr << "Executing command: " << ss.str() << endl;
   system(ss.str().c_str());
   fdata.open("data/fakedata_s70_4bit.dat",ios_base::binary | ios::app);
