diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index eb08576bb5..fb73bc6c7f 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -28,6 +28,8 @@ from easybuild.tools.filetools import run_cmd from easybuild.tools.systemtools import get_kernel_name, get_shared_lib_ext, get_platform_name +import easybuild.tools.environment as env + class GCC(Application): """ Self-contained build of GCC. @@ -265,13 +267,13 @@ def make(self): # register built GCC as compiler to use for stage 2/3 path = "%s/bin:%s" % (self.stage1installdir, os.getenv('PATH')) - os.putenv('PATH', path) + env.set('PATH', path) ld_lib_path = "%(dir)s/lib64:%(dir)s/lib:%(val)s" % { 'dir':self.stage1installdir, 'val':os.getenv('LD_LIBRARY_PATH') } - os.putenv('LD_LIBRARY_PATH', ld_lib_path) + env.set('LD_LIBRARY_PATH', ld_lib_path) # # STAGE 2: build GMP/PPL/CLooG for stage 3 @@ -371,7 +373,7 @@ def make(self): incpath = os.path.join(stage2prefix, 'include') cppflags = os.getenv('CPPFLAGS', '') - os.putenv('CPPFLAGS', "%s -L%s -I%s " % (cppflags, libpath, incpath)) + env.set('CPPFLAGS', "%s -L%s -I%s " % (cppflags, libpath, incpath)) # # STAGE 3: bootstrap build of final GCC (with PPL/CLooG support) diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 5db2579667..825881f75f 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -29,6 +29,8 @@ from easybuild.tools.filetools import run_cmd from easybuild.tools.modules import Modules +import easybuild.tools.environment as env + class Imkl(IntelBase): """ Class that can be used to install mkl @@ -174,10 +176,10 @@ def postproc(self): self.log.exception("Creating temporary directory failed") ## always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS - os.putenv('INSTALL_DIR', tmpbuild) - os.putenv('SPEC_OPT', opt) - os.putenv('COPTS', opt) - os.putenv('CFLAGS', opt) + env.set('INSTALL_DIR', tmpbuild) + env.set('SPEC_OPT', opt) + env.set('COPTS', opt) + env.set('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) @@ -293,9 +295,9 @@ def postproc(self): self.log.exception("Creating temporary directory failed") ## always set INSTALL_DIR, SPEC_OPT and CFLAGS - os.putenv('INSTALL_DIR', tmpbuild) - os.putenv('SPEC_OPT', opt) - os.putenv('CFLAGS', opt) + env.set('INSTALL_DIR', tmpbuild) + env.set('SPEC_OPT', opt) + env.set('CFLAGS', opt) try: intdir = os.path.join(interfacedir, i) diff --git a/easybuild/easyblocks/i/intelbase.py b/easybuild/easyblocks/i/intelbase.py index 43d54f816b..70c03ce5ef 100644 --- a/easybuild/easyblocks/i/intelbase.py +++ b/easybuild/easyblocks/i/intelbase.py @@ -22,6 +22,7 @@ import shutil from easybuild.framework.application import Application from easybuild.tools.filetools import run_cmd +import easybuild.tools.environment as env class IntelBase(Application): """ @@ -38,10 +39,10 @@ def __init__(self, *args, **kwargs): self.cfg.update({ 'license':[None,"License file path (default: None)"], 'license_activation':['license_server', "Indicates license activation type (default: 'license_server')"], - # 'usetmppath': + # 'usetmppath': # workaround for older SL5 version (5.5 and earlier) # used to be True, but False since SL5.6/SL6 - # disables TMP_PATH env and command line option + # disables TMP_PATH env and command line option 'usetmppath':[False, "Use temporary path for installation (default: False)"], 'm32':[False, "Enable 32-bit toolkit (default: False)"], }) @@ -71,7 +72,7 @@ def configure(self): self.log.error("Can't find license at %s" % self.license) ## set INTEL_LICENSE_FILE - os.environ["INTEL_LICENSE_FILE"] = self.license + env.set("INTEL_LICENSE_FILE", self.license) # clean home directory self.clean_homedir() @@ -113,14 +114,14 @@ def make_install(self): self.log.exception("Directory %s can't be created" % (tmpdir)) tmppathopt = '' if self.getcfg('usetmppath'): - os.putenv('TMP_PATH', tmpdir) + env.set('TMP_PATH', tmpdir) tmppathopt = "-t %s" % tmpdir ## set some extra env variables - os.environ['LOCAL_INSTALL_VERBOSE'] = '1' - os.environ['VERBOSE_MODE'] = '1' + env.set('LOCAL_INSTALL_VERBOSE','1') + env.set('VERBOSE_MODE', '1') - os.environ['INSTALL_PATH'] = self.installdir + env.set('INSTALL_PATH', self.installdir) ## perform installation cmd = "./install.sh %s -s %s" % (tmppathopt, silentcfg) @@ -136,4 +137,4 @@ def cleanup(self): Application.cleanup(self) - # no default sanity check, needs to be implemented by derived class \ No newline at end of file + # no default sanity check, needs to be implemented by derived class diff --git a/easybuild/easyblocks/m/mvapich2.py b/easybuild/easyblocks/m/mvapich2.py index a3b3c22996..6d08799024 100644 --- a/easybuild/easyblocks/m/mvapich2.py +++ b/easybuild/easyblocks/m/mvapich2.py @@ -20,6 +20,7 @@ ## import os from easybuild.framework.application import Application +import easybuild.tools.environment as env class MVAPICH2(Application): """ @@ -65,14 +66,14 @@ def configure(self): # enable Fortran 77/90 and C++ bindings add_configopts += '--enable-f77 --enable-fc --enable-cxx ' - # MVAPICH configure script complains when F90 or F90FLAGS are set, + # MVAPICH configure script complains when F90 or F90FLAGS are set, # they should be replaced with FC/FCFLAGS instead for (envvar, new_envvar) in [("F90", "FC"), ("F90FLAGS", "FCFLAGS")]: envvar_val = os.getenv(envvar) if envvar_val: if not os.getenv(new_envvar): - os.putenv(new_envvar, envvar_val) - os.putenv(envvar, '') + env.set(new_envvar, envvar_val) + env.set(envvar, '') else: self.log.error("Both %(ev)s and %(nev)s set, can I overwrite %(nev)s with %(ev)s (%(evv)s) ?" % { 'ev':envvar, @@ -100,7 +101,7 @@ def sanitycheck(self): """ if not self.getcfg('sanityCheckPaths'): - self.setcfg('sanityCheckPaths',{'files':["bin/%s" % x for x in ["mpicc", "mpicxx", "mpif77", + self.setcfg('sanityCheckPaths',{'files':["bin/%s" % x for x in ["mpicc", "mpicxx", "mpif77", "mpif90", "mpiexec.hydra"]] + ["lib/lib%s" % y for x in ["fmpich", "mpichcxx", "mpichf90", "mpich", "mpl", "opa"] @@ -110,4 +111,4 @@ def sanitycheck(self): self.log.info("Customized sanity check paths: %s"%self.getcfg('sanityCheckPaths')) - Application.sanitycheck(self) \ No newline at end of file + Application.sanitycheck(self) diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 47f74cce35..eea55414f7 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -21,6 +21,7 @@ import os from distutils.version import LooseVersion from easybuild.framework.application import Application +import easybuild.tools.environment as env class NetCDF(Application): """Support for building/installing netCDF""" @@ -40,7 +41,7 @@ def configure(self): # add -DgFortran to CPPFLAGS when building with GCC if self.tk.toolkit_comp_family() == "GCC": - os.environ['CPPFLAGS'] = "%s -DgFortran" % os.getenv('CPPFLAGS') + env.set('CPPFLAGS', "%s -DgFortran" % os.getenv('CPPFLAGS')) Application.configure(self) @@ -78,7 +79,7 @@ def set_netcdf_env_vars(log): if not netcdf: log.error("netCDF module not loaded?") else: - os.environ['NETCDF'] = netcdf + env.set('NETCDF', netcdf) log.debug("Set NETCDF to %s" % netcdf) netcdff = os.getenv('SOFTROOTNETCDFMINFORTRAN') netcdf_ver = os.getenv('SOFTVERSIONNETCDF') @@ -86,7 +87,7 @@ def set_netcdf_env_vars(log): if LooseVersion(netcdf_ver) >= LooseVersion("4.2"): log.error("netCDF v4.2 no longer supplies Fortran library, also need netCDF-Fortran") else: - os.environ['NETCDFF'] = netcdff + env.set('NETCDFF', netcdff) log.debug("Set NETCDFF to %s" % netcdff) def get_netcdf_module_set_cmds(log): @@ -101,4 +102,4 @@ def get_netcdf_module_set_cmds(log): txt += "setenv NETCDFF %s\n" % netcdff return txt else: - log.error("NETCDF environment variable not set?") \ No newline at end of file + log.error("NETCDF environment variable not set?") diff --git a/easybuild/easyblocks/n/netcdf_fortran.py b/easybuild/easyblocks/n/netcdf_fortran.py index dd50ec9bbd..d2303fe27d 100644 --- a/easybuild/easyblocks/n/netcdf_fortran.py +++ b/easybuild/easyblocks/n/netcdf_fortran.py @@ -20,6 +20,7 @@ ## import os from easybuild.framework.application import Application +import easybuild.tools.environment as env class NetCDF_Fortran(Application): """Support for building/installing the netCDF-Fortran library""" @@ -34,7 +35,7 @@ def configure(self): # add -DgFortran to CPPFLAGS when building with GCC if self.tk.toolkit_comp_family() == "GCC": - os.environ['CPPFLAGS'] = "%s -DgFortran" % os.getenv('CPPFLAGS') + env.set('CPPFLAGS', "%s -DgFortran" % os.getenv('CPPFLAGS')) Application.configure(self) @@ -55,4 +56,4 @@ def sanitycheck(self): self.log.info("Customized sanity check paths: %s"%self.getcfg('sanityCheckPaths')) - Application.sanitycheck(self) \ No newline at end of file + Application.sanitycheck(self) diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index f8a2e23990..90efd21291 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -29,6 +29,8 @@ from easybuild.tools.filetools import patch_perl_script_autoflush, run_cmd, run_cmd_qa, unpack from easybuild.easyblocks.n.netcdf import set_netcdf_env_vars, get_netcdf_module_set_cmds +import easybuild.tools.environment as env + class WPS(Application): """Support for building/installing WPS.""" @@ -53,7 +55,7 @@ def __init__(self): }) def configure(self): - """Configure build: + """Configure build: - set required environment variables (for netCDF, JasPer) - patch compile script and ungrib Makefile for non-default install paths of WRF and JasPer - run configure script and figure how to select desired build option @@ -91,8 +93,8 @@ def configure(self): jasper = os.getenv('SOFTROOTJASPER') jasperlibdir = os.path.join(jasper, "lib") if jasper: - os.environ['JASPERINC'] = os.path.join(jasper, "include") - os.environ['JASPERLIB'] = jasperlibdir + env.set('JASPERINC', os.path.join(jasper, "include")) + env.set('JASPERLIB', jasperlibdir) else: self.log.error("JasPer module not loaded?") @@ -337,4 +339,4 @@ def make_module_extra(self): txt += get_netcdf_module_set_cmds(self.log) - return txt \ No newline at end of file + return txt diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index a1a6eee516..665b1b8e65 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -26,6 +26,8 @@ from easybuild.tools.filetools import patch_perl_script_autoflush, run_cmd, run_cmd_qa from easybuild.easyblocks.n.netcdf import set_netcdf_env_vars, get_netcdf_module_set_cmds +import easybuild.tools.environment as env + class WRF(Application): """Support for building/installing WRF.""" @@ -33,7 +35,7 @@ def __init__(self,*args,**kwargs): """Add extra config options specific to WRF.""" Application.__init__(self, args,kwargs) - + self.build_in_installdir = True self.wrfsubdir = None @@ -47,8 +49,8 @@ def __init__(self,*args,**kwargs): 'runtest':[True, "Build and run WRF tests (default: True)."] }) - def configure(self): - """Configure build: + def configure(self): + """Configure build: - set some magic environment variables - run configure script - adjust configure.wrf file if needed @@ -70,7 +72,7 @@ def configure(self): if not (hdf5 or parallel_hdf5): self.log.error("Parallel HDF5 module not loaded?") else: - os.putenv('PHDF5', hdf5) + env.set('PHDF5', hdf5) else: self.log.info("HDF5 module not loaded, assuming that's OK...") @@ -78,8 +80,8 @@ def configure(self): jasper = os.getenv('SOFTROOTJASPER') jasperlibdir = os.path.join(jasper, "lib") if jasper: - os.environ['JASPERINC'] = os.path.join(jasper, "include") - os.environ['JASPERLIB'] = jasperlibdir + env.set('JASPERINC', os.path.join(jasper, "include")) + env.set('JASPERLIB', jasperlibdir) else: if os.getenv('JASPERINC') or os.getenv('JASPERLIB'): @@ -88,7 +90,7 @@ def configure(self): self.log.info("JasPer module not loaded, assuming that's OK...") # enable support for large file support in netCDF - os.putenv('WRFIO_NCD_LARGE_FILE_SUPPORT', '1') + env.set('WRFIO_NCD_LARGE_FILE_SUPPORT', '1') # patch arch/Config_new.pl script, so that run_cmd_qa receives all output to answer questions patch_perl_script_autoflush(os.path.join("arch", "Config_new.pl")) @@ -161,7 +163,7 @@ def configure(self): for envvar in ['CFLAGS', 'FFLAGS']: val = os.getenv(envvar) if '-O3' in val: - os.environ[envvar] = '%s -heap-arrays' % val + env.set(envvar, '%s -heap-arrays' % val) self.log.info("Updated %s to '%s'" % (envvar, os.getenv(envvar))) # replace -O3 with desired optimization options diff --git a/easybuild/framework/application.py b/easybuild/framework/application.py index f2dfbc71cf..2adb9d749e 100644 --- a/easybuild/framework/application.py +++ b/easybuild/framework/application.py @@ -32,6 +32,7 @@ import easybuild import easybuild.tools.config as config +import easybuild.tools.environment as env from easybuild.tools.build_log import EasyBuildError, initLogger, removeLogHandler,print_msg from easybuild.tools.config import source_path, buildPath, installPath from easybuild.tools.filetools import unpack, patch, run_cmd, convertName @@ -98,9 +99,6 @@ def __init__(self, name=None, version=None, newBuild=True, debug=False): # allow a post message to be set, which can be shown as last output self.postmsg = '' - # tempfile for the script which can be sourced - self.script_file = tempfile.NamedTemporaryFile() - # generic configuration parameters self.cfg = { 'name':[None, "Name of software"], @@ -848,11 +846,11 @@ def build(self): self.gen_installdir() self.make_builddir() - self.script_file.write("# EasyBuild version: %s for module %s/%s\n" % (easybuild.VERBOSE_VERSION, - self.name(), self.installversion)) - self.print_environ() + # reset tracked changes + env.reset_changes() + ## SOURCE print_msg("unpacking...", self.log) self.runstep('source', [self.unpack_src], skippable=True) @@ -893,6 +891,13 @@ def build(self): finally: self.runstep('cleanup', [self.cleanup]) + # write changes to the environment to logdir + logdir = os.path.join(self.installdir, config.logPath()) + if not os.path.isdir(logdir): + os.makedirs(logdir) + + env.write_changes(os.path.join(logdir, "easybuild-env-vars.sh")) + except StopException: pass @@ -920,13 +925,8 @@ def print_environ(self): mods_text = "\n".join(["module load %s/%s" % m for m in mods if m not in self.loaded_modules]) self.loaded_modules = mods - filter = ["_LMFILES_","LOADEDMODULES"] - env = copy.deepcopy(os.environ) - for key in filter: - env.pop(key, '') - changed = [(k,env[k]) for k in env if k not in self.orig_environ] for k in env: if k in self.orig_environ and env[k] != self.orig_environ[k]: @@ -937,7 +937,6 @@ def print_environ(self): text = "\n".join(['export %s="%s"' % change for change in changed]) unset_text = "\n".join(['unset %s' % key for key in unset]) - if mods: self.log.debug("Loaded modules:\n%s" % mods_text) if changed: @@ -945,12 +944,8 @@ def print_environ(self): if unset: self.log.debug("Removed from environment:\n%s" % unset_text) - if text or unset_text: - self.script_file.write("\n".join([text, unset_text])) - self.orig_environ = env - def postproc(self): """ Do some postprocessing @@ -987,7 +982,6 @@ def postproc(self): def cleanup(self): """ Cleanup leftover mess: remove/clean build directory - Move temporary files into log directory except when we're building in the installation directory, otherwise we remove the installation @@ -1007,19 +1001,6 @@ def cleanup(self): except OSError, err: self.log.exception("Cleaning up builddir %s failed: %s" % (self.builddir, err)) - logdir = os.path.join(self.installdir, config.logPath()) - actual_script_path = os.path.join(logdir, "easybuild-env-vars.sh") - - if not os.path.isdir(logdir): - os.makedirs(logdir) - - # move the temporary file to the actual destination - self.script_file.seek(0) - dest = open(actual_script_path, "w") - shutil.copyfileobj(self.script_file, dest) - dest.close() - self.script_file.close() - def sanitycheck(self): """ Do a sanity check on the installation diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py new file mode 100644 index 0000000000..44d07c84c8 --- /dev/null +++ b/easybuild/tools/environment.py @@ -0,0 +1,31 @@ +import os + +changes = {} + +def write_changes(filename): + """ + Write current changes to filename and reset environment afterwards + """ + script = open(filename,'w') + + for key in changes: + script.write('export %s="%s"\n' % (key, changes[key])) + + script.close() + reset_changes() + +def reset_changes(): + """ + Reset the changes tracked by this module + """ + global changes + changes = {} + +def set(key, value): + """ + put key in the environment with value + tracks added keys until write_changes has been called + """ + # os.putenv() is not necessary. os.environ will call this. + os.environ[key] = value + changes[key] = value diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ad8351d155..c28adf8532 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -33,6 +33,8 @@ from easybuild.tools.asyncprocess import Popen, PIPE, STDOUT, send_all, recv_some from easybuild.tools.build_log import getLog +import easybuild.tools.environment as env + log = getLog('fileTools') errorsFoundInLog = 0 @@ -568,7 +570,6 @@ def parse_cmd_output(cmd, stdouterr, ec, simple, log_all, log_ok, regexp): def modifyEnv(old, new): """ Compares 2 os.environ dumps. Adapts final environment. - - Assinging to os.environ doesn't seem to work, need to use os.putenv """ oldKeys = old.keys() newKeys = new.keys() @@ -578,12 +579,10 @@ def modifyEnv(old, new): ## hmm, smart checking with debug logging if not new[key] == old[key]: log.debug("Key in new environment found that is different from old one: %s (%s)" % (key, new[key])) - os.putenv(key, new[key]) - os.environ[key] = new[key] + env.set(key, new[key]) else: log.debug("Key in new environment found that is not in old one: %s (%s)" % (key, new[key])) - os.putenv(key, new[key]) - os.environ[key] = new[key] + env.set(key, new[key]) for key in oldKeys: if not key in newKeys: diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index a80f635b3b..52be1eee24 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -252,7 +252,7 @@ def dependencies_for(self, name, version): deps = [{'name':modname, 'version':modversion} for (modname, modversion) in mods] - # add dependencies of dependency modules only if they're not there yet + # add dependencies of dependency modules only if they're not there yet for moddepdeps in moddeps: for dep in moddepdeps: if not dep in deps: diff --git a/easybuild/tools/toolkit.py b/easybuild/tools/toolkit.py index e132dac1d4..89e4bdbaae 100644 --- a/easybuild/tools/toolkit.py +++ b/easybuild/tools/toolkit.py @@ -26,6 +26,8 @@ from easybuild.tools.modules import Modules, get_software_root from easybuild.tools import systemtools +import easybuild.tools.environment as env + log = getLog('Toolkit') class Toolkit: @@ -203,13 +205,13 @@ def _setVariables(self, dontset=None): continue log.debug("Setting environment variable %s to %s" % (key, val)) - os.environ[key] = val + env.set(key, val) # also set unique named variables that can be used in Makefiles # - so you can have 'CFLAGS = $(SOFTVARCFLAGS)' # -- 'CLFLAGS = $(CFLAGS)' gives '*** Recursive variable `CFLAGS' # references itself (eventually). Stop' error - os.environ["SOFTVAR%s" % key] = val + env.set("SOFTVAR%s" % key, val) def _getOptimalArchitecture(self): @@ -310,7 +312,7 @@ def prepareACML(self): if os.getenv('SOFTROOTGCC'): compiler = 'gfortran' elif os.getenv('SOFTROOTIFORT'): - compiler = 'ifort' + compiler = 'ifort' else: log.error("Don't know which compiler-specific subdir for ACML to use.") self.vars['LDFLAGS'] += " -L%(acml)s/%(comp)s64/lib/ " % { @@ -318,7 +320,7 @@ def prepareACML(self): 'comp':compiler, 'acml':os.environ['SOFTROOTACML'] } - self.vars['LIBBLAS'] = " -lacml_mv -lacml " #-lpthread" + self.vars['LIBBLAS'] = " -lacml_mv -lacml " #-lpthread" self.vars['LIBBLAS_MT'] = self.vars['LIBBLAS'] @@ -805,10 +807,10 @@ def mpi_cmd_for(self, cmd, nr_ranks): if mpi_type == "Intel": # set temporary dir for mdp - os.environ['I_MPI_MPD_TMPDIR'] = "/tmp" + env.set('I_MPI_MPD_TMPDIR', "/tmp") # set PBS_ENVIRONMENT, so that --file option for mpdboot isn't stripped away - os.environ['PBS_ENVIRONMENT'] = "PBS_BATCH_MPI" + env.set('PBS_ENVIRONMENT', "PBS_BATCH_MPI") # create mpdboot file fn = "/tmp/mpdboot"