#!/usr/bin/python from __future__ import print_function import os,sys import getopt import re # Set this to true if compiling within a master_tags checkout master_tag = True #master_tag = False def main(): """ Build and Install DiFX Usage: install-difx [options] Builds and installs mpifxcorr and associated tools. If invoked from the setup directory, the build will be within the source directory heirarchy, otherwise the build will be made within the current directory. By default install-difx will build and install the essential DiFX tools, but stop if any errors are encountered (e.g. if something fails to build). For the standard libraries and applications, build-difx will try to run autoreconf first. If this fails, step by step configuration will be tried. Options: -f --force Carry on regardless of any errors (all failed commands will be listed at the end.) -h --help Display this help message and quit. -v --verb Provide more chatter about progress. --mk5daemon Install mk5daemon (not installed by default) --perl Install some non-standard perl utilities --withmonitor Try to build the difx_monitor application (requires pgplot) --withfb Build filterbank (which requires pgplot) --withguiserver Build guiServer --withhops Build HOPS (which requires pgplot) --withm6support Build Mark6 support (which requires FUSE) --withmark6meta Build Mark6 metadata support --withpolconvert Build PolConvert (which requires CASA) --withpython Build Python bindings for certain libraries (requires ctypes) --withdatasim Build the datasim application (requires gsl) --noinstall Don't install, only build. --reconf Reconfigure step by step rather than with autoreconf --noconf Assume existing configuration is correct --clean Run make clean for all components --g77 Use g77, rather than gfortran --extraflags='flags' Add 'flags' to each of the CFLAGS, CXXFLAGS, and FFLAGS environment variables --makeflags='flags' Add 'flags' to each direct call of the make program, (so you can do make -j 4 to speed up compilation) --noipp Don't try and install ipp package config file, and also disable IPP acceleration of mpifxcorr --ipp Force installation of ipp package config file (default doesn't overwrite existing) --nodoc Don't build the documentation --cache='name' Speeds up the configuration using cache files 'name' and 'name-CXX' in the build directory --targ='...' Runs make ... on each of the components. The default is 'all'. --pristine Removes auto-built sources from source directories (and implies --clean) --doonly='...' Will only build the components specified in the list ..., where ... is a comma-separated list of sub-packages in ["perl", "difxio", "difxmessage", "mark5access", "vdifio", "calcif2", "difx2profile", "misc","vis2screen", "calcserver", "difx2fits", "vex2difx", "difx2mark4", "mpifxcorr", "difxfilterbank", "hops", "m6support", "polconvert", "guiServer", "mk5daemon", "difx_monitor", "datasim", "mark6meta", ] Remember, if you haven't signed up to the difx-users mailing list, please do so at https://listmgr.nrao.edu/mailman/listinfo/difx-users. """ pass ############################################################################### # Parse command line options and arguments # When adding an option or argument # * Make sure a default is set # * Make sure it is documented in main.__doc__ ############################################################################### try: opts, args = getopt.gnu_getopt(sys.argv[1:], "fhv", ["force", "help", "verb", "mk5daemon", "withmonitor", "withdatasim", "perl", "withfb", "withhops", "withm6support", "withguiserver", "withpython", "withpolconvert", "noinstall", "reconf", "noconf", "clean", "g77", "extraflags=", "makeflags=", "withmark6meta", "noipp", "ipp", "nodoc", "cache=", "targ=", "pristine", "doonly=" ]) except getopt.GetoptError as err: print(err) print(main.__doc__) sys.exit(2) if not 0 <= len(args) <= 0: print("Error: Wrong number of Arguments") print(main.__doc__) sys.exit(2) # set defaults force = False reconf = False noinstall = False gfortran = True extraflags = None MAKEcommand = "make " doclean = False target = 'all' verb = False cache = 'none' errors = [] noconf = False pristine = False dodoc = True dopybindings = False doipp = 0 orgcwd = os.getcwd() components = {"perl" :False, "difxio" :True, "difxmessage" :True, "dirlist" :True, "mark5access" :True, "mark6meta" :False, "codifio" :False, "vdifio" :True, "calcif2" :True, "difx2profile" :False, "misc" :True, "vis2screen" :True, "calcserver" :True, "difx2fits" :True, "vex2difx" :True, "difx2mark4" :True, "mpifxcorr" :True, "difxfilterbank":False, "hops" :False, "m6support" :False, "polconvert" :False, "guiServer" :False, "mk5daemon" :False, "difx_monitor" :False, "datasim" :False, "python" :True, } def setNormalComponentsFalse(): global components components["difxio"] = False components["difxmessage"] = False components["dirlist"] = False components["mark5access"] = False components["mark6meta"] = False components["codifio"] = False components["vdifio"] = False components["calcif2"] = False components["difx2profile"] = False components["misc"] = False components["vis2screen"] = False components["calcserver"] = False components["difx2fits"] = False components["vex2difx"] = False components["difx2mark4"] = False components["hops"] = False components["polconvert"] = False components["mpifxcorr"]= False components["python"]= False def setOnlyComponents(only): global components setNormalComponentsFalse() array = only.strip().split(',') if(len(array)==0): raise RuntimeError("Could not find components to build in only argument '%s'" % (only)) for a in array: c = a.strip() if(c in components): components[c] = True if(c == "python"): dopybindings = True else: raise RuntimeError("Unrecognized only component '%s'" % (a)) # read options if len(opts) > 0: for o, a in opts: if o in ("-f", "--force"): force = True if o in ("-h", "--help"): print(main.__doc__) sys.exit(2) if o in ("-v", "--verb"): verb = True if o == "--mk5daemon": components["mk5daemon"] = True if o == "--withmonitor": components["difx_monitor"] = True if o == "--withdatasim": components["datasim"] = True if o == "--perl": components["perl"] = True if o == "--withfb": components["difxfilterbank"] = True if o == "--withhops": components["hops"] = True if o == "--withm6support": components["m6support"] = True if o == "--withmark6meta": components["mark6meta"] = True if o == "--withpolconvert": components["polconvert"] = True if o == "--withguiserver": components["guiServer"] = True if o == "--withpython": dopybindings = True if o == "--noinstall": noinstall = True if o == "--reconf": reconf = True if o == "--noconf": noconf = True reconf = True if o == "--clean": doclean = True if o == "--nodoc": dodoc = False if o == "--g77": gfortran = False if o == "--extraflags": extraflags = a if o == "--makeflags": MAKEcommand += a + ' ' if o == "--ipp": doipp = 1 if o == "--noipp": doipp = -1 if o == "--cache": cache = a if o == "--targ": target = a noinstall = True if o == "--pristine": pristine = True doclean = True force = True if o == "--doonly": setOnlyComponents(a) ###### Get all relevant environment variables ########### print("************************************") print("Getting environmental variables") print("") difxroot = os.environ.get('DIFXROOT') if not difxroot: raise RuntimeError("DIFXROOT must be defined") if not re.match("/",difxroot): raise RuntimeError("DIFXROOT must be an absolute path") bindir = difxroot + '/bin/' libdir = difxroot + '/lib/' pkgdir = difxroot + '/lib/pkgconfig/' incdir = difxroot + '/include/' ipproot = os.environ.get('IPPROOT') ipplib32 = os.environ.get('IPPLIB32') ipplib64 = os.environ.get('IPPLIB64') mpicxx = os.environ.get('MPICXX') pgplotdir = os.environ.get('PGPLOTDIR') difxbits = os.environ.get('DIFXBITS') platform = sys.platform if(extraflags is not None): ###### Get all relevant environment variables ########### print("************************************") print("Setting environmental variables") print("") for ftype in ["CFLAGS", "CXXFLAGS", "FFLAGS"]: thisflags = extraflags if(ftype in os.environ): thisflags = os.environ[ftype] + ' ' + extraflags os.environ[ftype] = thisflags ##### Check that appropriate setup has been done ######## if not master_tag: if os.environ.get('DIFX_VERSION') == None: print("You must have already source'd setup.bash/setup.csh!") print("DIFX_VERSION was undefined - aborting compilation") raise RuntimeError("") difx_version = os.environ.get('DIFX_VERSION') if difx_version != "trunk": difx_version = "branches/" + difx_version else: difx_version = '' ##### OSX Specific changes if platform == "darwin": print("Using Darwin tools") LIBTOOLIZE = "glibtoolize" SHAREDPOSTFIX = "dylib" else: LIBTOOLIZE = "libtoolize" SHAREDPOSTFIX = "so" if gfortran: os.environ['USEGFORTRAN'] = 'yes' ###### Targets #################################################### # auto_compile(doreconf, dolibtoolize, doautoheader, ..., dompicxx) # calcif2 is a utility on truck but an application on master tags libtargets = [] if components["difxio"]: libtargets.append(["difxio", difx_version, True,True,True,False]) if components["difxmessage"]: libtargets.append(["difxmessage",difx_version, True,True,True,False]) if components["dirlist"]: libtargets.append(["dirlist", difx_version, True,True,True,False]) if components["mark5access"]: libtargets.append(["mark5access",difx_version, True,True,True,False]) if components["mark6meta"]: libtargets.append(["mark6meta", difx_version, True,True,True,False]) if components["python"]: libtargets.append(["python", difx_version, True,True,False,False]) if components["codifio"]: libtargets.append(["codifio", difx_version, True,True,True,False]) if components["vdifio"]: libtargets.append(["vdifio", difx_version, True,True,True,False]) utiltargets = [] if components["calcif2"] and not master_tag: utiltargets.append(["calcif2", difx_version, True,False,True,False]) if components["difx2profile"]: utiltargets.append([ "pulsar/difx2profile", difx_version, True,False,True,True]) if components["misc"]: utiltargets.append(["misc", difx_version, True,False,False,False]) if components["vis2screen"]: utiltargets.append(["vis2screen",difx_version, True,False,False,True]) apptargets = [] if components["calcif2"] and master_tag: apptargets.append(["calcif2", difx_version, True,False,True,False]) if components["calcserver"]: apptargets.append(["calcserver", difx_version, True,True,False,False]) if components["difx2fits"]: apptargets.append(["difx2fits", difx_version, True,False,True,False]) if components["vex2difx"]: apptargets.append(["vex2difx", difx_version, True,True,True,False]) if components["difx2mark4"]: apptargets.append(["difx2mark4", difx_version, True,True,True,False]) if components["difxfilterbank"]: apptargets.append(["difxfilterbank", difx_version, True,False,False,True]) if components["hops"]: apptargets.append(["hops", difx_version, True,True,True,False]) if components["m6support"]: apptargets.append(["m6support", difx_version, True,False,True,False]) if components["polconvert"]: apptargets.append(["polconvert", difx_version, True,False,True,False]) if components["guiServer"]: apptargets.append(["guiServer", difx_version, True,False,False,True]) if components["mk5daemon"]: apptargets.append(["mk5daemon", difx_version, True,False,True,False]) if components["difx_monitor"]: apptargets.append(["difx_monitor", difx_version, True,False,False,True]) if components["datasim"]: apptargets.append(["datasim", difx_version, True,True,True,False]) # a navigation aid def os_chdir(dir): os.chdir(dir) if verb: print("--> " + dir) print("==> " + os.getcwd()) print("") sys.stdout.flush() # work out build and source directories blddir = os.getcwd() setupdir = os.path.dirname(sys.argv[0]) if not setupdir: setupdir = "." os_chdir(setupdir) # master_tags/ has no setup directory if not master_tag: os_chdir("..") topdir = os.getcwd() # think a bit about pkgconfig path pkg_config_path = blddir + "/pkgconfig" pkg_config_path += ":" + os.environ.get('PKG_CONFIG_PATH') os.environ['PKG_CONFIG_PATH'] = pkg_config_path # source dir build starts in 'setup' of the svn tree and topdir of master_tag if blddir == topdir + '/setup': if verb: print(""); print("Building/Installing from the source directory") inbdir = False startdir = topdir + '/setup' elif master_tag and blddir == topdir: if verb: print(""); print("Building/Installing from the master tag source directory") inbdir = False startdir = topdir else: if verb: print(""); print("Building/Installing from the build directory:") inbdir = True if master_tag: startdir = topdir else: startdir = topdir + '/setup' # allow use of symbolic links in partial checkouts repldir = os.environ.get('DIFX_REPLDIR_PATH') if not repldir: repldir = 'no-repldir-to-replace-with-blddir' if verb: print(" Replace dir: " + repldir) if cache != 'none': cache = ' --cache-file=' + blddir + '/' + cache if verb: print(" Setup dir: " + setupdir) print(" Source dir: " + topdir) print(" Start dir: " + startdir) print(" Build dir: " + blddir) print(" PkgCfg path: " + os.environ.get('PKG_CONFIG_PATH')) print("") ###### Subroutine to run a command and raise error on non-zero return def run(cmd): if verb: print("Run(" + cmd + ") in " + os.getcwd()) sys.stdout.flush() if os.system(cmd): if force: errors.append(os.getcwd() + ' ' + cmd + " failed.") else: raise RuntimeError("Error running " + cmd + " in " + os.getcwd()) ###### Subroutine to do the compiling of an auto-tool ### def auto_compile(doreconf, dolibtoolize, doautoheader, prefix, dompicxx): thisdir = os.getcwd() # somewhere below topdir if (doreconf and (reconf or os.system("autoreconf -if"))): if noconf and os.path.exists('configure'): print("Re-using existing configuration") else: print("Reconfiguring step by step") if os.path.exists('m4'): run("aclocal -I m4") else: run("aclocal") if dolibtoolize: run(LIBTOOLIZE+" --copy --force") run("autoconf") if doautoheader: run("autoheader") run("automake -acf") if inbdir: cfp = thisdir blp = cfp.replace(topdir, blddir) blp = blp.replace(repldir, blddir) if not os.path.exists(blp): os.makedirs(blp) if verb: print("++> " + blp + " was created") os_chdir(blp) else: cfp = '.' if noconf and os.path.exists('config.status'): print("Re-using existing configuration") else: configstring = cfp + "/configure --prefix=" + prefix if cache != 'none': configstring += cache # configstring = cfp + "/configure --prefix=" + prefix + cache if dompicxx: if cache != 'none': configstring += "-CXX" configstring += " CXX=" + mpicxx if dopybindings and check_configureAC_has(cfp+"/configure.ac", "enable-python"): configstring += " --enable-python " if dopybindings and check_configureAC_has(cfp+"/configure.ac", "with-python"): configstring += " --with-python " run(configstring) run(MAKEcommand + target) if not noinstall: run(MAKEcommand + "install") os_chdir(thisdir) def check_configureAC_has(cfpath, key): try: for line in open(cfpath,'r'): if key in line: return True except: print ("Warning: could not check %s for %s\n.") return False ###### Subroutine to set up documentation area ########## def make_doc_area(basedocdir): if not os.path.exists(basedocdir): os.mkdir(basedocdir) print("cp -f %s/doco-index.html %s/index.html" % (startdir, basedocdir)) os.system("cp -f %s/doco-index.html %s/index.html" % (startdir, basedocdir)) ###### Work ########################################## # a few of these are (randomly) svn'd... autojunk = 'aclocal.m4 autom4te.cache compile config.guess' autojunk += ' config.h.in config.sub configure depcomp config.status' autojunk += ' install-sh ltmain.sh Makefile.in missing' # autojunk += ' COPYING INSTALL' if doclean: if components["perl"]: print("**** Cleaning vexlib") os_chdir("libraries/vex/"+difx_version+"/vexlib") run(MAKEcommand + "clean") os_chdir(topdir) os_chdir("libraries") thisdir = os.getcwd() for libtarget in libtargets: targetdir = libtarget[0] + '/' + libtarget[1] if os.path.exists(targetdir): print("") print("**** Cleaning "+targetdir) os_chdir(targetdir) run(MAKEcommand + "-k clean distclean") if pristine: run("rm -rf " + autojunk) else: print("") print("**** Skipping "+targetdir) print("") os_chdir(thisdir) os_chdir(topdir) os_chdir("applications") thisdir = os.getcwd() for apptarget in apptargets: targetdir = apptarget[0] + '/' + apptarget[1] if os.path.exists(targetdir): print("") print("**** Cleaning "+targetdir) os_chdir(targetdir) run(MAKEcommand + "-k clean distclean") if pristine: run("rm -rf " + autojunk) else: print("") print("**** Skipping "+targetdir) print("") os_chdir(thisdir) os_chdir(topdir) os_chdir("utilities") thisdir = os.getcwd() for utiltarget in utiltargets: if (utiltarget[1] == ''): targetdir = utiltarget[0] else: targetdir= utiltarget[1] + '/' + utiltarget[0] if os.path.exists(targetdir): print("") print("**** Cleaning "+targetdir) os_chdir(targetdir) run(MAKEcommand + "-k clean distclean") if pristine: run("rm -rf " + autojunk) else: print("") print("**** Skipping "+targetdir) print("") os_chdir(thisdir) targetdir = topdir + "/mpifxcorr/" + difx_version if os.path.exists(targetdir): print("") print("**** Cleaning "+targetdir) os_chdir(targetdir) run(MAKEcommand + "-k clean distclean") if pristine: run("rm -rf " + autojunk) else: print("") print("**** Skipping "+targetdir) print("") os_chdir(topdir) if pristine: run('find . -name Makefile.in -exec rm {} \;') sys.exit(0) # Clean exit ##### Make directories if required ###################### if not noinstall: print("************************************") print("Setting up directories") print("") if not os.path.exists(difxroot): os.mkdir(difxroot) if not os.path.exists(bindir): os.mkdir(bindir) if not os.path.exists(libdir): os.mkdir(libdir) if not os.path.exists(pkgdir): os.mkdir(pkgdir) if not os.path.exists(incdir): os.mkdir(incdir) ##### Install IPP package config file if appropriate ### if doipp<0: print("Not installing IPP .pc file") print("") else: if os.path.exists(pkgdir+"/ipp.pc") and doipp==0: print(pkgdir+"ipp.pc already exists") print("") else: print("Creating "+pkgdir+"ipp.pc") os_chdir(pkgdir) run(startdir+"/genipppc "+ipproot) os_chdir(topdir) print("") ##### Compile non-standard libraries #################### if components["perl"]: print("") print("************************************") print("Building vex") print("") os_chdir("libraries/vex/"+difx_version+"/vexlib") if platform == "darwin": run(MAKEcommand + "-f Makefile.osx") else: run(MAKEcommand) if not noinstall: run("mv -f libvex." + SHAREDPOSTFIX + " " + libdir) run("mv -f libvex.a " + libdir) os_chdir("../vexperl") run("perl Makefile.PL PREFIX=" + difxroot) run(MAKEcommand) if not noinstall: run(MAKEcommand + "install") os_chdir(topdir) print("") print("************************************") print("Building Astro Perl") print("") os_chdir("libraries/perl/"+difx_version+"/Astro") run("perl Makefile.PL PREFIX="+difxroot) run(MAKEcommand) if not noinstall: run(MAKEcommand + "install") os_chdir(topdir) print("") print("************************************") print("Building DIFX-Input Perl") print("") os.chdir("libraries/perl/"+difx_version+"/DIFX-Input") run("perl Makefile.PL PREFIX="+difxroot) run(MAKEcommand) if not noinstall: run(MAKEcommand + "install") os.chdir(topdir) ##### Make standard (autotool'd) libraries ############### os_chdir("libraries") thisdir = os.getcwd() for libtarget in libtargets: targetdir = libtarget[0] + '/' + libtarget[1] if os.path.exists(targetdir): print("") print("") print("************************************") print("Building ", targetdir) print("") os_chdir(targetdir) auto_compile(libtarget[2], libtarget[3], libtarget[4], difxroot, libtarget[5]) else: print("") print("******* Skipping "+targetdir) print("") os_chdir(thisdir) os_chdir(topdir) if components["mpifxcorr"]: ##### Make mpifxcorr ##################################### print("") print("************************************") print("Making mpifxcorr") print("") os_chdir("mpifxcorr/" + difx_version) thisdir = os.getcwd() if noconf and os.path.exists('configure'): print("Reusing mpifxcorr configuration") else: print("Reconfiguring mpifxcorr") run("aclocal") run("autoconf") run("autoheader") run("automake -acf") # the files added are all present, but that need not always be true.... if inbdir: cfp = thisdir blp = cfp.replace(topdir, blddir) blp = blp.replace(repldir, blddir) if not os.path.exists(blp): os.makedirs(blp) if verb: print("++> " + blp + " was created") os_chdir(blp) else: cfp = '.' if noconf and os.path.exists('config.status'): print("Re-using existing mpifxcorr configuration") else: configstring = cfp + "/configure CXX=" + mpicxx configstring += " --prefix=" + difxroot if doipp<0: configstring += ' --disable-ipp ' if cache != 'none': configstring += cache + "-CXX" run(configstring) run(MAKEcommand + target) if not noinstall: run(MAKEcommand + "install") os_chdir(topdir) ##### Make standard (autotool'd) applications ############ os_chdir("applications") thisdir = os.getcwd() for apptarget in apptargets: targetdir = apptarget[0] + '/' + apptarget[1] if os.path.exists(targetdir): print("") print("") print("************************************") print("Building ", apptarget[0]) print("") os_chdir(targetdir) auto_compile(apptarget[2], apptarget[3], apptarget[4], difxroot, apptarget[5]) else: print("") print("******* Skipping "+targetdir) print("") os_chdir(thisdir) os_chdir(topdir) ##### Make standard (autotool'd) utilities ############### os_chdir("utilities") thisdir = os.getcwd() for utiltarget in utiltargets: if (utiltarget[1] == ''): targetdir = utiltarget[0] else: targetdir= utiltarget[1] + '/' + utiltarget[0] if os.path.exists(targetdir): print("") print("") print("************************************") print("Building ", utiltarget[0]) os_chdir(targetdir) auto_compile(utiltarget[2], utiltarget[3], utiltarget[4], difxroot, utiltarget[5]) else: print("") print("******* Skipping "+targetdir) print("") os_chdir(thisdir) os_chdir(topdir) ##### Make documentation ################################# if(dodoc): basedocdir = difxroot + '/doc/' make_doc_area(basedocdir) os.chdir("mpifxcorr/" + difx_version) if os.system("which doxygen") != 0: print("No doxygen installed - skipping mpifxcorr documentation") else: os.system("doxygen Doxyfile") ##### Check for errors and list if present ############### if (force and not errors == []): print("") print("") print("************************************") print("WARNING: Errors were encountered during installation") print("") for error in errors: print(error) else: print("Done!")