Skip to content

generically find the arch folder and add top-level bin/lib symlinks in PDT & TAU easyblocks #2029

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 2 commits into from
May 19, 2020
Merged

generically find the arch folder and add top-level bin/lib symlinks in PDT & TAU easyblocks #2029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
98951e1
[PDT/TAU] Generically find the arch folder and add top-level bin/lib …
Flamefire Apr 16, 2020
e722ef1
Address review
Flamefire Apr 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 63 additions & 34 deletions easybuild/easyblocks/p/pdt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,37 +14,44 @@

@author Bernd Mohr (Juelich Supercomputing Centre)
@author Markus Geimer (Juelich Supercomputing Centre)
@author Alexander Grund (TU Dresden)
"""

import os

from easybuild.easyblocks.generic.configuremake import ConfigureMake
import easybuild.tools.toolchain as toolchain
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.run import run_cmd
from easybuild.tools.filetools import symlink


class EB_PDT(ConfigureMake):
"""Support for building/installing PDT."""
def find_arch_dir(install_dir):
"""
Find architecture directory inside the install tree.

def __init__(self, *args, **kwargs):
"""Initialisation of custom class variables for PDT."""
super(EB_PDT, self).__init__(*args, **kwargs)
For simplicity any top-level folder containing a bin and lib directory is collected
Raises an error if multiple or none matching folders are found
Returns the architecture specific folder
"""
arch_dirs = []
for entry in os.listdir(install_dir):
full_path = os.path.join(install_dir, entry)
# Only check directories which are not symlinks
# E.g. on x86_64 there are symlinks craycnl, mic_linux, ... to x86_64
if not os.path.isdir(full_path) or os.path.islink(full_path):
continue
if all(os.path.isdir(os.path.join(full_path, subdir)) for subdir in ['bin', 'lib']):
arch_dirs.append(full_path)

out, _ = run_cmd("uname -m", simple=False)
self.machine = out.strip()
self.log.info("Using '%s' as machine label", self.machine)
if not arch_dirs:
raise EasyBuildError('Architecture specific directory not found in %s', install_dir)
elif len(arch_dirs) != 1:
raise EasyBuildError('Found multiple architecture specific directories: %s', ', '.join(arch_dirs))
return arch_dirs[0]

def prepare_step(self, *args, **kwargs):
"""Custom prepare step for PDT."""
super(EB_PDT, self).prepare_step(*args, **kwargs)

# create install directory and make sure it does not get cleaned up again in the install step;
# the first configure iteration already puts things in place in the install directory,
# so that shouldn't get cleaned up afterwards...
self.log.info("Creating install dir %s before starting configure-build-install iterations", self.installdir)
self.make_installdir()
self.cfg['keeppreviousinstall'] = True
class EB_PDT(ConfigureMake):
"""Support for building/installing PDT."""

def configure_step(self):
"""Custom configuration procedure for PDT."""
Expand All @@ -57,36 +64,58 @@ def configure_step(self):
toolchain.SYSTEM: '-GNU',
toolchain.GCC: '-GNU',
toolchain.INTELCOMP: '-icpc',
toolchain.PGI: '-pgCC',
}
comp_fam = self.toolchain.comp_family()
if comp_fam in known_compilers:
try:
compiler_opt = known_compilers[comp_fam]
else:
except KeyError:
raise EasyBuildError("Compiler family not supported yet: %s" % comp_fam)
self.cfg.update('configopts', compiler_opt)

if self.toolchain.options['pic']:
self.cfg.update('-useropt=-fPIC')

# Configure creates required subfolders in installdir, so create first (but only once, during first iteration)
if self.iter_idx == 0:
super(EB_PDT, self).make_installdir()

super(EB_PDT, self).configure_step()

def build_step(self):
"""Custom build procedure for PDT."""
# The PDT build is triggered by 'make install', thus skip the 'make' step
"""Skip build step"""
# `make install` runs `make all` which runs `make clean`, so no point in doing a make first
pass

def make_installdir(self):
"""Skip creating installation directory, already done in configure step"""
pass

def install_step(self):
"""Create symlinks into arch-specific directories"""

if self.cfg['parallel']:
self.cfg.update('installopts', '-j %s' % self.cfg['parallel'])

super(EB_PDT, self).install_step()

# Link arch-specific directories into prefix
arch_dir = find_arch_dir(self.installdir)
self.log.info('Found %s as architecture specific directory. Creating symlinks...', arch_dir)
for subdir in ('bin', 'lib'):
src = os.path.join(arch_dir, subdir)
dst = os.path.join(self.installdir, subdir)
if os.path.lexists(dst):
self.log.info('Skipping creation of symlink %s as it already exists', dst)
else:
symlink(os.path.relpath(src, self.installdir), dst, use_abspath_source=False)

def sanity_check_step(self):
"""Custom sanity check for PDT."""
custom_paths = {
'files': [os.path.join(self.machine, 'bin', 'cparse'),
os.path.join(self.machine, 'include', 'pdb.h'),
os.path.join(self.machine, 'lib', 'libpdb.a')],
'files': [os.path.join('bin', 'cparse'),
os.path.join('include', 'pdb.h'),
os.path.join('lib', 'libpdb.a')],
'dirs': [],
}
super(EB_PDT, self).sanity_check_step(custom_paths=custom_paths)

def make_module_req_guess(self):
"""Custom guesses for environment variables (PATH, ...) for PDT."""
guesses = super(EB_PDT, self).make_module_req_guess()
guesses.update({
'PATH': [os.path.join(self.machine, 'bin')],
'LD_LIBRARY_PATH': [os.path.join(self.machine, 'lib')],
})
return guesses
55 changes: 28 additions & 27 deletions easybuild/easyblocks/t/tau.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@
import os

from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.easyblocks.pdt import find_arch_dir
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools import toolchain
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.filetools import mkdir
from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version
from easybuild.tools.run import run_cmd
from easybuild.tools.filetools import symlink
from easybuild.tools.modules import get_software_root, get_software_version
from easybuild.tools.systemtools import get_shared_lib_ext


Expand Down Expand Up @@ -65,10 +65,6 @@ def __init__(self, *args, **kwargs):
"""Initialize TAU easyblock."""
super(EB_TAU, self).__init__(*args, **kwargs)

out, _ = run_cmd("uname -m", simple=False)
self.machine = out.strip()
self.log.info("Using '%s' as machine label", self.machine)

self.variant_index = 0
self.cc, self.cxx, self.fortran = None, None, None
self.mpi_inc_dir, self.mpi_lib_dir = None, None
Expand Down Expand Up @@ -187,12 +183,6 @@ def prepare_step(self, *args, **kwargs):

self.variant_labels.append(variant_label)

# create install directory and make sure it does not get cleaned up again in the install step;
# the first configure iteration already puts things in place in the install directory,
# so that shouldn't get cleaned up afterwards...
self.log.info("Creating install dir %s before starting configure-build-install iterations", self.installdir)
super(EB_TAU, self).make_installdir()

def make_installdir(self):
"""Skip make install dir 'step', install dir is already created in prepare_step."""
pass
Expand Down Expand Up @@ -229,6 +219,10 @@ def configure_step(self):
for key in ['preconfigopts', 'configopts', 'prebuildopts', 'preinstallopts']:
self.log.debug("%s for TAU (variant index: %s): %s", key, self.variant_index, self.cfg[key])

# Configure creates required subfolders in installdir, so create first (but only once, during first iteration)
if self.iter_idx == 0:
super(EB_TAU, self).make_installdir()

super(EB_TAU, self).configure_step()

self.variant_index += 1
Expand All @@ -237,32 +231,39 @@ def build_step(self):
"""No custom build procedure for TAU."""
pass

def install_step(self):
"""Create symlinks into arch-specific directories"""
super(EB_TAU, self).install_step()
# Link arch-specific directories into prefix
arch_dir = find_arch_dir(self.installdir)
self.log.info('Found %s as architecture specific directory. Creating symlinks...', arch_dir)
for subdir in ('bin', 'lib'):
src = os.path.join(arch_dir, subdir)
dst = os.path.join(self.installdir, subdir)
if os.path.lexists(dst):
self.log.info('Skipping creation of symlink %s as it already exists', dst)
else:
symlink(os.path.relpath(src, self.installdir), dst, use_abspath_source=False)

def sanity_check_step(self):
"""Custom sanity check for TAU."""
custom_paths = {
'files': [os.path.join(self.machine, 'bin', 'pprof'), os.path.join('include', 'TAU.h'),
os.path.join(self.machine, 'lib', 'libTAU.%s' % get_shared_lib_ext())] +
[os.path.join(self.machine, 'lib', 'lib%s.a' % l) for l in self.variant_labels] +
[os.path.join(self.machine, 'lib', 'Makefile.' + l) for l in self.variant_labels],
'files':
[os.path.join('bin', 'pprof'), os.path.join('include', 'TAU.h'),
os.path.join('lib', 'libTAU.%s' % get_shared_lib_ext())] +
[os.path.join('lib', 'lib%s.a' % l) for l in self.variant_labels] +
[os.path.join('lib', 'Makefile.' + l) for l in self.variant_labels],
'dirs': [],
}
super(EB_TAU, self).sanity_check_step(custom_paths=custom_paths)

def make_module_req_guess(self):
"""Custom guesses for environment variables (PATH, ...) for TAU."""
guesses = super(EB_TAU, self).make_module_req_guess()
guesses.update({
'PATH': [os.path.join(self.machine, 'bin')],
})
return guesses

def make_module_extra(self):
"""Custom extra module file entries for TAU."""
txt = super(EB_TAU, self).make_module_extra()

txt += self.module_generator.prepend_paths('TAU_MF_DIR', os.path.join(self.machine, 'lib'))
txt += self.module_generator.prepend_paths('TAU_MF_DIR', 'lib')

tau_makefile = os.path.join(self.installdir, self.machine, 'lib', self.cfg['tau_makefile'])
tau_makefile = os.path.join(self.installdir, 'lib', self.cfg['tau_makefile'])
txt += self.module_generator.set_environment('TAU_MAKEFILE', tau_makefile)

# default measurement settings
Expand Down