|
| 1 | +## |
| 2 | +# Copyright 2009-2023 Ghent University |
| 3 | +# |
| 4 | +# This file is part of EasyBuild, |
| 5 | +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), |
| 6 | +# with support of Ghent University (http://ugent.be/hpc), |
| 7 | +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), |
| 8 | +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) |
| 9 | +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). |
| 10 | +# |
| 11 | +# https://github.com/easybuilders/easybuild |
| 12 | +# |
| 13 | +# EasyBuild is free software: you can redistribute it and/or modify |
| 14 | +# it under the terms of the GNU General Public License as published by |
| 15 | +# the Free Software Foundation v2. |
| 16 | +# |
| 17 | +# EasyBuild is distributed in the hope that it will be useful, |
| 18 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 19 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 20 | +# GNU General Public License for more details. |
| 21 | +# |
| 22 | +# You should have received a copy of the GNU General Public License |
| 23 | +# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. |
| 24 | +## |
| 25 | +""" |
| 26 | +EasyBuild support for installing Cargo packages (Rust lang package system) |
| 27 | +
|
| 28 | +@author: Mikael Oehman (Chalmers University of Technology) |
| 29 | +""" |
| 30 | + |
| 31 | +import os |
| 32 | + |
| 33 | +import easybuild.tools.environment as env |
| 34 | +from easybuild.tools.build_log import EasyBuildError |
| 35 | +from easybuild.framework.easyconfig import CUSTOM |
| 36 | +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock |
| 37 | +from easybuild.tools.filetools import extract_file, change_dir |
| 38 | +from easybuild.tools.run import run_cmd |
| 39 | +from easybuild.tools.config import build_option |
| 40 | +from easybuild.tools.filetools import write_file, compute_checksum |
| 41 | + |
| 42 | +CRATESIO_SOURCE = "https://crates.io/api/v1/crates" |
| 43 | + |
| 44 | + |
| 45 | +class Cargo(ExtensionEasyBlock): |
| 46 | + """Support for installing Cargo packages (Rust)""" |
| 47 | + |
| 48 | + @staticmethod |
| 49 | + def extra_options(extra_vars=None): |
| 50 | + """Define extra easyconfig parameters specific to Cargo""" |
| 51 | + extra_vars = ExtensionEasyBlock.extra_options(extra_vars) |
| 52 | + extra_vars.update({ |
| 53 | + 'enable_tests': [True, "Enable building of tests", CUSTOM], |
| 54 | + 'offline': [True, "Build offline", CUSTOM], |
| 55 | + 'lto': [None, "Override default LTO flag ('fat', 'thin', 'off')", CUSTOM], |
| 56 | + 'crates': [[], "List of (crate, version, [repo, rev]) tuples to use", CUSTOM], |
| 57 | + }) |
| 58 | + |
| 59 | + return extra_vars |
| 60 | + |
| 61 | + def __init__(self, *args, **kwargs): |
| 62 | + """Constructor for Cargo easyblock.""" |
| 63 | + super(Cargo, self).__init__(*args, **kwargs) |
| 64 | + self.cargo_home = os.path.join(self.builddir, '.cargo') |
| 65 | + env.setvar('CARGO_HOME', self.cargo_home) |
| 66 | + env.setvar('RUSTC', 'rustc') |
| 67 | + env.setvar('RUSTDOC', 'rustdoc') |
| 68 | + env.setvar('RUSTFMT', 'rustfmt') |
| 69 | + optarch = build_option('optarch') |
| 70 | + if not optarch: |
| 71 | + optarch = 'native' |
| 72 | + env.setvar('RUSTFLAGS', '-C target-cpu=%s' % optarch) |
| 73 | + env.setvar('RUST_LOG', 'DEBUG') |
| 74 | + env.setvar('RUST_BACKTRACE', '1') |
| 75 | + |
| 76 | + # Populate sources from "crates" list of tuples (only once) |
| 77 | + if self.cfg['crates']: |
| 78 | + # copy list of crates, so we can wipe 'crates' easyconfig parameter, |
| 79 | + # to avoid that creates are processed into 'sources' easyconfig parameter again |
| 80 | + # when easyblock is initialized again using same parsed easyconfig |
| 81 | + # (for example when check_sha256_checksums function is called, like in easyconfigs test suite) |
| 82 | + self.crates = self.cfg['crates'][:] |
| 83 | + sources = [] |
| 84 | + for crate_info in self.cfg['crates']: |
| 85 | + if len(crate_info) == 2: |
| 86 | + crate, version = crate_info |
| 87 | + sources.append({ |
| 88 | + 'download_filename': crate + '/' + version + '/download', |
| 89 | + 'filename': crate + '-' + version + '.tar.gz', |
| 90 | + 'source_urls': [CRATESIO_SOURCE], |
| 91 | + 'alt_location': 'crates.io', |
| 92 | + }) |
| 93 | + else: |
| 94 | + crate, version, repo, rev = crate_info |
| 95 | + url, repo_name_git = repo.rsplit('/', maxsplit=1) |
| 96 | + sources.append({ |
| 97 | + 'git_config': {'url': url, 'repo_name': repo_name_git[:-4], 'commit': rev}, |
| 98 | + 'filename': crate + '-' + version + '.tar.gz', |
| 99 | + 'source_urls': [CRATESIO_SOURCE], |
| 100 | + }) |
| 101 | + |
| 102 | + self.cfg.update('sources', sources) |
| 103 | + |
| 104 | + # set 'crates' easyconfig parameter to empty list to prevent re-processing into sources |
| 105 | + self.cfg['crates'] = [] |
| 106 | + |
| 107 | + def extract_step(self): |
| 108 | + """ |
| 109 | + Unpack the source files and populate them with required .cargo-checksum.json if offline |
| 110 | + """ |
| 111 | + if self.cfg['offline']: |
| 112 | + self.log.info("Setting vendored crates dir") |
| 113 | + # Replace crates-io with vendored sources using build dir wide toml file in CARGO_HOME |
| 114 | + # because the rust source subdirectories might differ with python packages |
| 115 | + config_toml = os.path.join(self.cargo_home, 'config.toml') |
| 116 | + write_file(config_toml, '[source.vendored-sources]\ndirectory = "%s"\n\n' % self.builddir, append=True) |
| 117 | + write_file(config_toml, '[source.crates-io]\nreplace-with = "vendored-sources"\n\n', append=True) |
| 118 | + |
| 119 | + # also vendor sources from other git sources (could be many crates for one git source) |
| 120 | + git_sources = set() |
| 121 | + for crate_info in self.crates: |
| 122 | + if len(crate_info) == 4: |
| 123 | + _, _, repo, rev = crate_info |
| 124 | + git_sources.add((repo, rev)) |
| 125 | + for repo, rev in git_sources: |
| 126 | + write_file(config_toml, '[source."%s"]\ngit = "%s"\nrev = "%s"\n' |
| 127 | + 'replace-with = "vendored-sources"\n\n' % (repo, repo, rev), append=True) |
| 128 | + |
| 129 | + # Use environment variable since it would also be passed along to builds triggered via python packages |
| 130 | + env.setvar('CARGO_NET_OFFLINE', 'true') |
| 131 | + |
| 132 | + # More work is needed here for git sources to work, especially those repos with multiple packages. |
| 133 | + for src in self.src: |
| 134 | + existing_dirs = set(os.listdir(self.builddir)) |
| 135 | + self.log.info("Unpacking source %s" % src['name']) |
| 136 | + srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'], |
| 137 | + extra_options=self.cfg['unpack_options'], change_into_dir=False) |
| 138 | + change_dir(srcdir) |
| 139 | + if srcdir: |
| 140 | + self.src[self.src.index(src)]['finalpath'] = srcdir |
| 141 | + else: |
| 142 | + raise EasyBuildError("Unpacking source %s failed", src['name']) |
| 143 | + |
| 144 | + # Create checksum file for all sources required by vendored crates.io sources |
| 145 | + new_dirs = set(os.listdir(self.builddir)) - existing_dirs |
| 146 | + if self.cfg['offline'] and len(new_dirs) == 1: |
| 147 | + cratedir = new_dirs.pop() |
| 148 | + self.log.info('creating .cargo-checksums.json file for : %s', cratedir) |
| 149 | + chksum = compute_checksum(src['path'], checksum_type='sha256') |
| 150 | + chkfile = os.path.join(self.builddir, cratedir, '.cargo-checksum.json') |
| 151 | + write_file(chkfile, '{"files":{},"package":"%s"}' % chksum) |
| 152 | + |
| 153 | + def configure_step(self): |
| 154 | + """Empty configuration step.""" |
| 155 | + pass |
| 156 | + |
| 157 | + @property |
| 158 | + def profile(self): |
| 159 | + return 'debug' if self.toolchain.options.get('debug', None) else 'release' |
| 160 | + |
| 161 | + def build_step(self): |
| 162 | + """Build with cargo""" |
| 163 | + parallel = '' |
| 164 | + if self.cfg['parallel']: |
| 165 | + parallel = "-j %s" % self.cfg['parallel'] |
| 166 | + |
| 167 | + tests = '' |
| 168 | + if self.cfg['enable_tests']: |
| 169 | + tests = "--tests" |
| 170 | + |
| 171 | + lto = '' |
| 172 | + if self.cfg['lto'] is not None: |
| 173 | + lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto']) |
| 174 | + |
| 175 | + run_cmd('rustc --print cfg', log_all=True, simple=True) # for tracking in log file |
| 176 | + cmd = ' '.join([ |
| 177 | + self.cfg['prebuildopts'], |
| 178 | + 'cargo build', |
| 179 | + '--profile=' + self.profile, |
| 180 | + lto, |
| 181 | + tests, |
| 182 | + parallel, |
| 183 | + self.cfg['buildopts'], |
| 184 | + ]) |
| 185 | + run_cmd(cmd, log_all=True, simple=True) |
| 186 | + |
| 187 | + def test_step(self): |
| 188 | + """Test with cargo""" |
| 189 | + if self.cfg['enable_tests']: |
| 190 | + cmd = ' '.join([ |
| 191 | + self.cfg['pretestopts'], |
| 192 | + 'cargo test', |
| 193 | + '--profile=' + self.profile, |
| 194 | + self.cfg['testopts'], |
| 195 | + ]) |
| 196 | + run_cmd(cmd, log_all=True, simple=True) |
| 197 | + |
| 198 | + def install_step(self): |
| 199 | + """Install with cargo""" |
| 200 | + cmd = ' '.join([ |
| 201 | + self.cfg['preinstallopts'], |
| 202 | + 'cargo install', |
| 203 | + '--profile=' + self.profile, |
| 204 | + '--root=' + self.installdir, |
| 205 | + '--path=.', |
| 206 | + self.cfg['installopts'], |
| 207 | + ]) |
| 208 | + run_cmd(cmd, log_all=True, simple=True) |
| 209 | + |
| 210 | + |
| 211 | +def generate_crate_list(sourcedir): |
| 212 | + """Helper for generating crate list""" |
| 213 | + import toml |
| 214 | + |
| 215 | + cargo_toml = toml.load(os.path.join(sourcedir, 'Cargo.toml')) |
| 216 | + cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock')) |
| 217 | + |
| 218 | + app_name = cargo_toml['package']['name'] |
| 219 | + deps = cargo_lock['package'] |
| 220 | + |
| 221 | + app_in_cratesio = False |
| 222 | + crates = [] |
| 223 | + other_crates = [] |
| 224 | + for dep in deps: |
| 225 | + name = dep['name'] |
| 226 | + version = dep['version'] |
| 227 | + if 'source' in dep: |
| 228 | + if name == app_name: |
| 229 | + app_in_cratesio = True # exclude app itself, needs to be first in crates list or taken from pypi |
| 230 | + else: |
| 231 | + if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index': |
| 232 | + crates.append((name, version)) |
| 233 | + else: |
| 234 | + # Lock file has revision#revision in the url for some reason. |
| 235 | + crates.append((name, version, dep['source'].rsplit('#', maxsplit=1)[0])) |
| 236 | + else: |
| 237 | + other_crates.append((name, version)) |
| 238 | + return app_in_cratesio, crates, other_crates |
| 239 | + |
| 240 | + |
| 241 | +if __name__ == '__main__': |
| 242 | + import sys |
| 243 | + app_in_cratesio, crates, other = generate_crate_list(sys.argv[1]) |
| 244 | + print(other) |
| 245 | + if app_in_cratesio or crates: |
| 246 | + print('crates = [') |
| 247 | + if app_in_cratesio: |
| 248 | + print(' (name, version),') |
| 249 | + for crate_info in crates: |
| 250 | + print(" ('" + "', '".join(crate_info) + "'),") |
| 251 | + print(']') |
0 commit comments